SQL generator: derive date-adjacent columns from pf.dim_period at baseline load
- col_meta gets dim_period_col field: maps a dimension column to its pf.dim_period counterpart (e.g. year -> cal_year, month -> cal_month) - When the date column is is_key of a dim_group and any sibling dimension has dim_period_col set, baseline and reference SQL JOIN pf.dim_period on the shifted date instead of copying raw source values - No dim_period config = identical SQL to before (fully backwards compatible) - Setup UI: period col input in col_meta editor, enabled for dimension columns with a dim_group set - Schema migration applied: dim_period_col text null on pf.col_meta Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cf9bdea9a8
commit
16c296d529
@ -25,7 +25,7 @@ function generateSQL(source, colMeta) {
|
|||||||
if (!dateCol) throw new Error('No date column defined in col_meta');
|
if (!dateCol) throw new Error('No date column defined in col_meta');
|
||||||
if (dims.length === 0) throw new Error('No dimension columns defined in col_meta');
|
if (dims.length === 0) throw new Error('No dimension columns defined in col_meta');
|
||||||
|
|
||||||
const srcTable = `${source.schema}.${source.tname}`;
|
const srcTable = `"${source.schema}"."${source.tname}"`;
|
||||||
const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean);
|
const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean);
|
||||||
const effectiveValue = dataCols.includes(valueCol) ? valueCol : null;
|
const effectiveValue = dataCols.includes(valueCol) ? valueCol : null;
|
||||||
const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null;
|
const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null;
|
||||||
@ -33,6 +33,19 @@ function generateSQL(source, colMeta) {
|
|||||||
const selectData = dataCols.map(q).join(', ');
|
const selectData = dataCols.map(q).join(', ');
|
||||||
const dimsJoined = dims.map(q).join(', ');
|
const dimsJoined = dims.map(q).join(', ');
|
||||||
|
|
||||||
|
// dim_period JOIN support: if the date column is the is_key of a dim_group,
|
||||||
|
// dimension siblings with dim_period_col set are derived from pf.dim_period
|
||||||
|
// instead of being copied raw from the source on baseline/reference load.
|
||||||
|
const dateKeyGroup = colMeta.find(c => c.role === 'date' && c.is_key && c.dim_group)?.dim_group;
|
||||||
|
const dimPeriodMap = new Map(
|
||||||
|
dateKeyGroup
|
||||||
|
? colMeta
|
||||||
|
.filter(c => c.role === 'dimension' && c.dim_group === dateKeyGroup && c.dim_period_col)
|
||||||
|
.map(c => [c.cname, c.dim_period_col])
|
||||||
|
: []
|
||||||
|
);
|
||||||
|
const hasDimPeriod = dimPeriodMap.size > 0;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
get_data: buildGetData(),
|
get_data: buildGetData(),
|
||||||
baseline: buildBaseline(),
|
baseline: buildBaseline(),
|
||||||
@ -47,10 +60,22 @@ function generateSQL(source, colMeta) {
|
|||||||
return `SELECT * FROM {{fc_table}}`;
|
return `SELECT * FROM {{fc_table}}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildLoadSelect(pfx) {
|
||||||
|
// pfx: table alias prefix ('s.' when joining dim_period, '' otherwise)
|
||||||
|
return dataCols.map(c => {
|
||||||
|
if (c === dateCol) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`;
|
||||||
|
if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`;
|
||||||
|
return `${pfx}${q(c)}`;
|
||||||
|
}).join(',\n ');
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildFromClause() {
|
||||||
|
if (!hasDimPeriod) return srcTable;
|
||||||
|
return `${srcTable} s\n JOIN pf.dim_period dp`
|
||||||
|
+ ` ON dp.drange @> (s.${q(dateCol)} + '{{date_offset}}'::interval)::date`;
|
||||||
|
}
|
||||||
|
|
||||||
function buildBaseline() {
|
function buildBaseline() {
|
||||||
const baselineSelect = dataCols.map(c =>
|
|
||||||
c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c)
|
|
||||||
).join(', ');
|
|
||||||
return `
|
return `
|
||||||
WITH
|
WITH
|
||||||
ilog AS (
|
ilog AS (
|
||||||
@ -60,8 +85,10 @@ ilog AS (
|
|||||||
)
|
)
|
||||||
,ins AS (
|
,ins AS (
|
||||||
INSERT INTO {{fc_table}} (${insertCols})
|
INSERT INTO {{fc_table}} (${insertCols})
|
||||||
SELECT ${baselineSelect}, 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
|
SELECT
|
||||||
FROM ${srcTable}
|
${buildLoadSelect(hasDimPeriod ? 's.' : '')},
|
||||||
|
'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
|
||||||
|
FROM ${buildFromClause()}
|
||||||
WHERE {{filter_clause}}
|
WHERE {{filter_clause}}
|
||||||
RETURNING *
|
RETURNING *
|
||||||
)
|
)
|
||||||
@ -69,9 +96,6 @@ SELECT count(*) AS rows_affected FROM ins`.trim();
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildReference() {
|
function buildReference() {
|
||||||
const referenceSelect = dataCols.map(c =>
|
|
||||||
c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c)
|
|
||||||
).join(', ');
|
|
||||||
return `
|
return `
|
||||||
WITH
|
WITH
|
||||||
ilog AS (
|
ilog AS (
|
||||||
@ -81,8 +105,10 @@ ilog AS (
|
|||||||
)
|
)
|
||||||
,ins AS (
|
,ins AS (
|
||||||
INSERT INTO {{fc_table}} (${insertCols})
|
INSERT INTO {{fc_table}} (${insertCols})
|
||||||
SELECT ${referenceSelect}, 'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
|
SELECT
|
||||||
FROM ${srcTable}
|
${buildLoadSelect(hasDimPeriod ? 's.' : '')},
|
||||||
|
'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
|
||||||
|
FROM ${buildFromClause()}
|
||||||
WHERE {{filter_clause}}
|
WHERE {{filter_clause}}
|
||||||
RETURNING *
|
RETURNING *
|
||||||
)
|
)
|
||||||
|
|||||||
@ -89,22 +89,24 @@ module.exports = function(pool) {
|
|||||||
await client.query('BEGIN');
|
await client.query('BEGIN');
|
||||||
for (const col of cols) {
|
for (const col of cols) {
|
||||||
await client.query(`
|
await client.query(`
|
||||||
INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, opos)
|
INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, dim_period_col, opos)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||||
ON CONFLICT (source_id, cname) DO UPDATE SET
|
ON CONFLICT (source_id, cname) DO UPDATE SET
|
||||||
label = EXCLUDED.label,
|
label = EXCLUDED.label,
|
||||||
role = EXCLUDED.role,
|
role = EXCLUDED.role,
|
||||||
is_key = EXCLUDED.is_key,
|
is_key = EXCLUDED.is_key,
|
||||||
dim_group = EXCLUDED.dim_group,
|
dim_group = EXCLUDED.dim_group,
|
||||||
opos = EXCLUDED.opos
|
dim_period_col = EXCLUDED.dim_period_col,
|
||||||
|
opos = EXCLUDED.opos
|
||||||
`, [
|
`, [
|
||||||
sourceId,
|
sourceId,
|
||||||
col.cname,
|
col.cname,
|
||||||
col.label || null,
|
col.label || null,
|
||||||
col.role || 'ignore',
|
col.role || 'ignore',
|
||||||
col.is_key || false,
|
col.is_key || false,
|
||||||
col.dim_group || null,
|
col.dim_group || null,
|
||||||
col.opos || null
|
col.dim_period_col || null,
|
||||||
|
col.opos || null
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
await client.query('COMMIT');
|
await client.query('COMMIT');
|
||||||
|
|||||||
@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS pf.source (
|
|||||||
-- backfill columns for existing installs
|
-- backfill columns for existing installs
|
||||||
ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb;
|
ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb;
|
||||||
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text;
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text;
|
||||||
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text;
|
||||||
|
|
||||||
-- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate
|
-- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate
|
||||||
|
|
||||||
|
|||||||
@ -282,6 +282,7 @@ export default function Setup({ refreshSources }) {
|
|||||||
<th className="px-3 py-1.5 font-medium">role</th>
|
<th className="px-3 py-1.5 font-medium">role</th>
|
||||||
<th className="px-3 py-1.5 font-medium text-center">key</th>
|
<th className="px-3 py-1.5 font-medium text-center">key</th>
|
||||||
<th className="px-3 py-1.5 font-medium">group</th>
|
<th className="px-3 py-1.5 font-medium">group</th>
|
||||||
|
<th className="px-3 py-1.5 font-medium">period col</th>
|
||||||
<th className="px-3 py-1.5 font-medium">label</th>
|
<th className="px-3 py-1.5 font-medium">label</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@ -317,6 +318,16 @@ export default function Setup({ refreshSources }) {
|
|||||||
className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default"
|
className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default"
|
||||||
/>
|
/>
|
||||||
</td>
|
</td>
|
||||||
|
<td className="px-3 py-1.5">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={col.dim_period_col || ''}
|
||||||
|
onChange={e => updateCol(i, 'dim_period_col', e.target.value || null)}
|
||||||
|
placeholder="—"
|
||||||
|
disabled={col.role !== 'dimension' || !col.dim_group}
|
||||||
|
className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default font-mono text-xs"
|
||||||
|
/>
|
||||||
|
</td>
|
||||||
<td className="px-3 py-1.5">
|
<td className="px-3 py-1.5">
|
||||||
<input
|
<input
|
||||||
type="text"
|
type="text"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user