pt/pf_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Carry every date and value column on baseline and reference loads

Browse Source 
The generator picked columns with find(), so a source with more than one
date or value column silently loaded only the first of each. The version's
DDL is built from col_meta separately, so the extra columns existed in the
forecast table and stayed null for its whole life -- gs.osm_skinny lost
sdate and stdcost_usd across 295k rows that way.

Loads copy the source row wholesale, so they now carry all of them, with
the offset shifting every date column together. The adjustment operations
are genuinely single-measure -- scale distributes one {{value_incr}} -- so
they keep the narrow column list until that has a defined meaning.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-09-16 17:42:14 -04:00
parent e31a60b18b
commit 0e87f22e24
1 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
lib/sql_generator.js
View File

@ -16,9 +16,21 @@ function generateSQL(source, colMeta) {
.sort((a, b) => (a.opos || 0) - (b.opos || 0)) .sort((a, b) => (a.opos || 0) - (b.opos || 0))
.map(c => c.cname); .map(c => c.cname);
const valueCol = colMeta.find(c => c.role === 'value')?.cname; // Every column of each measure/date role, in col_meta order. Loads carry all of
const unitsCol = colMeta.find(c => c.role === 'units')?.cname; // them; the adjustment operations are single-measure (scale distributes one
const dateCol = colMeta.find(c => c.role === 'date')?.cname; // {{value_incr}}) and use only the first of each, below.
const byRole = role => colMeta
.filter(c => c.role === role)
.sort((a, b) => (a.opos || 0) - (b.opos || 0))
.map(c => c.cname);
const valueCols = byRole('value');
const unitsCols = byRole('units');
const dateCols = byRole('date');
const valueCol = valueCols[0];
const unitsCol = unitsCols[0];
const dateCol = dateCols[0];
if (!valueCol) throw new Error('No value column defined in col_meta'); if (!valueCol) throw new Error('No value column defined in col_meta');
if (!dateCol) throw new Error('No date column defined in col_meta'); if (!dateCol) throw new Error('No date column defined in col_meta');
@ -32,6 +44,13 @@ function generateSQL(source, colMeta) {
const selectData = dataCols.map(q).join(', '); const selectData = dataCols.map(q).join(', ');
const dimsJoined = dims.map(q).join(', '); const dimsJoined = dims.map(q).join(', ');
// Baseline and reference copy the source row wholesale, so they carry every
// measure and every date — not just the primary one the operations act on.
// Dropping the others would leave those columns null for the life of the version.
const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols];
const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
const dateColSet = new Set(dateCols);
// dim_period JOIN support: if the date column is the is_key of a dim_group, // dim_period JOIN support: if the date column is the is_key of a dim_group,
// dimension siblings with dim_period_col set are derived from pf.dim_period // dimension siblings with dim_period_col set are derived from pf.dim_period
// instead of being copied raw from the source on baseline/reference load. // instead of being copied raw from the source on baseline/reference load.
@ -61,8 +80,9 @@ function generateSQL(source, colMeta) {
function buildLoadSelect(pfx) { function buildLoadSelect(pfx) {
// pfx: table alias prefix ('s.' when joining dim_period, '' otherwise) // pfx: table alias prefix ('s.' when joining dim_period, '' otherwise)
return dataCols.map(c => { // The offset shifts every date column, so order date and ship date stay in step.
if (c === dateCol) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`; return loadCols.map(c => {
if (dateColSet.has(c)) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`;
if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`; if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`;
return `${pfx}${q(c)}`; return `${pfx}${q(c)}`;
}).join(',\n '); }).join(',\n ');
@ -83,7 +103,7 @@ ilog AS (
RETURNING id RETURNING id
) )
,ins AS ( ,ins AS (
INSERT INTO {{fc_table}} (${insertCols}) INSERT INTO {{fc_table}} (${loadInsertCols})
SELECT SELECT
${buildLoadSelect(hasDimPeriod ? 's.' : '')}, ${buildLoadSelect(hasDimPeriod ? 's.' : '')},
'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
@ -103,7 +123,7 @@ ilog AS (
RETURNING id RETURNING id
) )
,ins AS ( ,ins AS (
INSERT INTO {{fc_table}} (${insertCols}) INSERT INTO {{fc_table}} (${loadInsertCols})
SELECT SELECT
${buildLoadSelect(hasDimPeriod ? 's.' : '')}, ${buildLoadSelect(hasDimPeriod ? 's.' : '')},
'reference', (SELECT id FROM ilog), '{{pf_user}}', now() 'reference', (SELECT id FROM ilog), '{{pf_user}}', now()