From 19e52fa3fee1d77b7af32189336e10c42e411b67 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Mon, 21 Sep 2026 09:19:09 -0400 Subject: [PATCH] Carry every date column through an adjustment Scale and recode wrote dataCols, which held dateCols[0] and nothing else. On a source with order, requested and ship dates that means every scale and recode row was written with rdate and sdate_e null -- not stale, absent -- while the period dimensions derived from them (rseas, smon_e) were copied with the other dimensions and so read perfectly, which is what kept it out of sight. Both columns are role 'date' and therefore sliceable, so an adjustment scoped to a ship date matched no adjustment row that had ever been written, and anything dating the forecast by ship date dropped every adjustment without saying so. Clone already carried them all, for the date_offset it has to shift; the loads always did. The GROUP BY widens with the select list. On osm_skinny that is 971,185 groups against 971,352 -- ordnum and ordline already make a row nearly unique -- so the row count is unaffected in practice. Existing rows are not repaired; see Known issues. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 19 +++++++++++++++++++ lib/sql_generator.js | 32 ++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5fd6504..9fdb323 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -408,6 +408,18 @@ Turning a region back into slices re-derives, per cell, the same filters Perspec All three operations follow the same structure: insert a `pf.log` row in a CTE, then insert forecast rows referencing its id. `{{where_clause}}` is built from the slice; `{{exclude_clause}}` blocks `exclude_iters` rows. +**Every date column travels, not just the first.** `dataCols` is +`[...dims, ...dateCols, value, units]`. It used to be `dateCols[0]`, so on a +source with order, requested and ship dates, scale and recode wrote rows whose +`rdate` and `sdate_e` were **null** — and because the derived period dimensions +(`rseas`, `smon_e`) are dimensions and were copied with the rest, the labels +read correctly while the dates under them were gone. Both columns are +`role = 'date'` and therefore sliceable, so an adjustment scoped to a ship date +matched no adjustment row that had ever been written. Clone and the loads were +always right. The wider GROUP BY costs almost nothing on `osm_skinny` +(971,185 → 971,352 groups) because `ordnum`/`ordline` already make a row +near-unique. + - **Scale** — distributes `value_incr`/`units_incr` proportionally across rows in the slice using window functions - **Recode** — inserts negative rows (zero out original) + positive rows with `{{set_clause}}` dimension overrides; both share the same logid - **Clone** — copies the slice with `{{set_clause}}` overrides and `{{scale_factor}}` multiplier; original untouched @@ -539,6 +551,13 @@ column. ## Known issues / active work +- **Rows written before 2026-09-21 have null `rdate`/`sdate_e`.** The generator + fix above changes what gets written from here on; it does not repair what is + there. On `fc_osm_skinny_29` that is ~270k scale and recode rows. A backfill + would have to re-derive each from the baseline rows at the same coordinate, + which is possible but is inference, not recovery — deleting and replaying + those log entries is the honest repair + - **Zero-row operations report success.** Scale refuses with "Nothing to scale…" when its slice matches nothing; recode and clone commit an empty log entry and return `rows_affected: 0`. A recode of a rep whose rows are all diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 901fd0f..56d9817 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -198,19 +198,31 @@ function generateSQL(source, colMeta) { const valueCol = valueCols[0]; const unitsCol = unitsCols[0]; - const dateCol = dateCols[0]; if (!valueCol) throw new Error('No value column defined in col_meta'); - if (!dateCol) throw new Error('No date column defined in col_meta'); + if (!dateCols.length) throw new Error('No date column defined in col_meta'); if (dims.length === 0) throw new Error('No dimension columns defined in col_meta'); const srcTable = `"${source.schema}"."${source.tname}"`; - const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean); + // Every date column, not just the primary one. Scale and recode used to + // carry dateCols[0] alone, so a source with order, requested and ship dates + // wrote adjustment rows whose rdate and sdate_e were *null* -- not stale, + // absent. Those columns are role 'date' and therefore sliceable, so a later + // adjustment scoped to a ship date matched no adjustment row ever written, + // and any reader that dated the forecast by ship date lost every + // adjustment silently. The derived period dimensions (rseas, smon_e) were + // fine throughout, being dimensions and copied with the rest -- which is + // what made it invisible: the labels were right and the dates underneath + // them were gone. + const dataCols = [...dims, ...dateCols, valueCol, unitsCol].filter(Boolean); const effectiveValue = dataCols.includes(valueCol) ? valueCol : null; const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null; const insertCols = [...dataCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', '); const selectData = dataCols.map(q).join(', '); const dimsJoined = dims.map(q).join(', '); + // The date columns as a select list, in the same order dataCols lists them, + // so it lines up with insertCols positionally. + const datesJoined = dateCols.map(q).join(', '); // Baseline and reference copy the source row wholesale, so they carry every // measure and every date — not just the primary one the operations act on. @@ -410,7 +422,7 @@ SELECT count(*) AS rows_affected, (SELECT id FROM ilog) AS log_id FROM ins`.trim // comes to the same figure the ungrouped window produced. const baseSelectParts = [ ...dimsJoined ? [dimsJoined] : [], - q(dateCol), + datesJoined, effectiveValue ? `sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : null, effectiveUnits ? `sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null, effectiveValue ? `sum(sum(${q(effectiveValue)})) OVER () AS total_value` : null, @@ -430,12 +442,12 @@ ilog AS ( WHERE {{where_clause}} {{exclude_clause}} GROUP BY - ${groupCols([...dims, dateCol])} + ${groupCols([...dims, ...dateCols])} ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) SELECT - ${[dimsJoined, q(dateCol), ...(effectiveValue ? [vSel] : []), ...(effectiveUnits ? [uSel] : [])].join(',\n ')}, + ${[dimsJoined, datesJoined, ...(effectiveValue ? [vSel] : []), ...(effectiveUnits ? [uSel] : [])].join(',\n ')}, 'scale', (SELECT id FROM ilog), '{{pf_user}}', now() FROM base RETURNING * @@ -454,23 +466,23 @@ ilog AS ( ,src AS ( SELECT ${dimsJoined}, - ${q(dateCol)}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} + ${datesJoined}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} GROUP BY - ${groupCols([...dims, dateCol])} + ${groupCols([...dims, ...dateCols])} ) ,neg AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${dimsJoined}, ${q(dateCol)}, ${effectiveValue ? `-${q(effectiveValue)}` : '0'}${effectiveUnits ? `, -${q(effectiveUnits)}` : ''}, + SELECT ${dimsJoined}, ${datesJoined}, ${effectiveValue ? `-${q(effectiveValue)}` : '0'}${effectiveUnits ? `, -${q(effectiveUnits)}` : ''}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() FROM src RETURNING * ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT {{set_clause}}, ${q(dateCol)}, ${effectiveValue ? q(effectiveValue) : '0'}${effectiveUnits ? `, ${q(effectiveUnits)}` : ''}, + SELECT {{set_clause}}, ${datesJoined}, ${effectiveValue ? q(effectiveValue) : '0'}${effectiveUnits ? `, ${q(effectiveUnits)}` : ''}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() FROM src RETURNING *