From 16c296d52904bc9b9335d3cf9acd370536774b86 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 23 May 2026 10:06:05 -0400 Subject: [PATCH] SQL generator: derive date-adjacent columns from pf.dim_period at baseline load - col_meta gets dim_period_col field: maps a dimension column to its pf.dim_period counterpart (e.g. year -> cal_year, month -> cal_month) - When the date column is is_key of a dim_group and any sibling dimension has dim_period_col set, baseline and reference SQL JOIN pf.dim_period on the shifted date instead of copying raw source values - No dim_period config = identical SQL to before (fully backwards compatible) - Setup UI: period col input in col_meta editor, enabled for dimension columns with a dim_group set - Schema migration applied: dim_period_col text null on pf.col_meta Co-Authored-By: Claude Sonnet 4.6 --- lib/sql_generator.js | 48 +++++++++++++++++++++++++++++++---------- routes/sources.js | 26 +++++++++++----------- setup_sql/01_schema.sql | 1 + ui/src/views/Setup.jsx | 11 ++++++++++ 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 01c485c..7794cbc 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -25,7 +25,7 @@ function generateSQL(source, colMeta) { if (!dateCol) throw new Error('No date column defined in col_meta'); if (dims.length === 0) throw new Error('No dimension columns defined in col_meta'); - const srcTable = `${source.schema}.${source.tname}`; + const srcTable = `"${source.schema}"."${source.tname}"`; const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean); const effectiveValue = dataCols.includes(valueCol) ? valueCol : null; const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null; @@ -33,6 +33,19 @@ function generateSQL(source, colMeta) { const selectData = dataCols.map(q).join(', '); const dimsJoined = dims.map(q).join(', '); + // dim_period JOIN support: if the date column is the is_key of a dim_group, + // dimension siblings with dim_period_col set are derived from pf.dim_period + // instead of being copied raw from the source on baseline/reference load. + const dateKeyGroup = colMeta.find(c => c.role === 'date' && c.is_key && c.dim_group)?.dim_group; + const dimPeriodMap = new Map( + dateKeyGroup + ? colMeta + .filter(c => c.role === 'dimension' && c.dim_group === dateKeyGroup && c.dim_period_col) + .map(c => [c.cname, c.dim_period_col]) + : [] + ); + const hasDimPeriod = dimPeriodMap.size > 0; + return { get_data: buildGetData(), baseline: buildBaseline(), @@ -47,10 +60,22 @@ function generateSQL(source, colMeta) { return `SELECT * FROM {{fc_table}}`; } + function buildLoadSelect(pfx) { + // pfx: table alias prefix ('s.' when joining dim_period, '' otherwise) + return dataCols.map(c => { + if (c === dateCol) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`; + if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`; + return `${pfx}${q(c)}`; + }).join(',\n '); + } + + function buildFromClause() { + if (!hasDimPeriod) return srcTable; + return `${srcTable} s\n JOIN pf.dim_period dp` + + ` ON dp.drange @> (s.${q(dateCol)} + '{{date_offset}}'::interval)::date`; + } + function buildBaseline() { - const baselineSelect = dataCols.map(c => - c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c) - ).join(', '); return ` WITH ilog AS ( @@ -60,8 +85,10 @@ ilog AS ( ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${baselineSelect}, 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() - FROM ${srcTable} + SELECT + ${buildLoadSelect(hasDimPeriod ? 's.' : '')}, + 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM ${buildFromClause()} WHERE {{filter_clause}} RETURNING * ) @@ -69,9 +96,6 @@ SELECT count(*) AS rows_affected FROM ins`.trim(); } function buildReference() { - const referenceSelect = dataCols.map(c => - c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c) - ).join(', '); return ` WITH ilog AS ( @@ -81,8 +105,10 @@ ilog AS ( ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${referenceSelect}, 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() - FROM ${srcTable} + SELECT + ${buildLoadSelect(hasDimPeriod ? 's.' : '')}, + 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM ${buildFromClause()} WHERE {{filter_clause}} RETURNING * ) diff --git a/routes/sources.js b/routes/sources.js index 5b2352e..d1c1157 100644 --- a/routes/sources.js +++ b/routes/sources.js @@ -89,22 +89,24 @@ module.exports = function(pool) { await client.query('BEGIN'); for (const col of cols) { await client.query(` - INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, opos) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, dim_period_col, opos) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (source_id, cname) DO UPDATE SET - label = EXCLUDED.label, - role = EXCLUDED.role, - is_key = EXCLUDED.is_key, - dim_group = EXCLUDED.dim_group, - opos = EXCLUDED.opos + label = EXCLUDED.label, + role = EXCLUDED.role, + is_key = EXCLUDED.is_key, + dim_group = EXCLUDED.dim_group, + dim_period_col = EXCLUDED.dim_period_col, + opos = EXCLUDED.opos `, [ sourceId, col.cname, - col.label || null, - col.role || 'ignore', - col.is_key || false, - col.dim_group || null, - col.opos || null + col.label || null, + col.role || 'ignore', + col.is_key || false, + col.dim_group || null, + col.dim_period_col || null, + col.opos || null ]); } await client.query('COMMIT'); diff --git a/setup_sql/01_schema.sql b/setup_sql/01_schema.sql index dfd677d..b555002 100644 --- a/setup_sql/01_schema.sql +++ b/setup_sql/01_schema.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS pf.source ( -- backfill columns for existing installs ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb; ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text; +ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text; -- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate diff --git a/ui/src/views/Setup.jsx b/ui/src/views/Setup.jsx index edac978..a9999b4 100644 --- a/ui/src/views/Setup.jsx +++ b/ui/src/views/Setup.jsx @@ -282,6 +282,7 @@ export default function Setup({ refreshSources }) { role key group + period col label @@ -317,6 +318,16 @@ export default function Setup({ refreshSources }) { className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default" /> + + updateCol(i, 'dim_period_col', e.target.value || null)} + placeholder="—" + disabled={col.role !== 'dimension' || !col.dim_group} + className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default font-mono text-xs" + /> +