diff --git a/CLAUDE.md b/CLAUDE.md index 5fd6504..9fdb323 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -408,6 +408,18 @@ Turning a region back into slices re-derives, per cell, the same filters Perspec All three operations follow the same structure: insert a `pf.log` row in a CTE, then insert forecast rows referencing its id. `{{where_clause}}` is built from the slice; `{{exclude_clause}}` blocks `exclude_iters` rows. +**Every date column travels, not just the first.** `dataCols` is +`[...dims, ...dateCols, value, units]`. It used to be `dateCols[0]`, so on a +source with order, requested and ship dates, scale and recode wrote rows whose +`rdate` and `sdate_e` were **null** — and because the derived period dimensions +(`rseas`, `smon_e`) are dimensions and were copied with the rest, the labels +read correctly while the dates under them were gone. Both columns are +`role = 'date'` and therefore sliceable, so an adjustment scoped to a ship date +matched no adjustment row that had ever been written. Clone and the loads were +always right. The wider GROUP BY costs almost nothing on `osm_skinny` +(971,185 → 971,352 groups) because `ordnum`/`ordline` already make a row +near-unique. + - **Scale** — distributes `value_incr`/`units_incr` proportionally across rows in the slice using window functions - **Recode** — inserts negative rows (zero out original) + positive rows with `{{set_clause}}` dimension overrides; both share the same logid - **Clone** — copies the slice with `{{set_clause}}` overrides and `{{scale_factor}}` multiplier; original untouched @@ -539,6 +551,13 @@ column. ## Known issues / active work +- **Rows written before 2026-09-21 have null `rdate`/`sdate_e`.** The generator + fix above changes what gets written from here on; it does not repair what is + there. On `fc_osm_skinny_29` that is ~270k scale and recode rows. A backfill + would have to re-derive each from the baseline rows at the same coordinate, + which is possible but is inference, not recovery — deleting and replaying + those log entries is the honest repair + - **Zero-row operations report success.** Scale refuses with "Nothing to scale…" when its slice matches nothing; recode and clone commit an empty log entry and return `rows_affected: 0`. A recode of a rep whose rows are all diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 901fd0f..56d9817 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -198,19 +198,31 @@ function generateSQL(source, colMeta) { const valueCol = valueCols[0]; const unitsCol = unitsCols[0]; - const dateCol = dateCols[0]; if (!valueCol) throw new Error('No value column defined in col_meta'); - if (!dateCol) throw new Error('No date column defined in col_meta'); + if (!dateCols.length) throw new Error('No date column defined in col_meta'); if (dims.length === 0) throw new Error('No dimension columns defined in col_meta'); const srcTable = `"${source.schema}"."${source.tname}"`; - const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean); + // Every date column, not just the primary one. Scale and recode used to + // carry dateCols[0] alone, so a source with order, requested and ship dates + // wrote adjustment rows whose rdate and sdate_e were *null* -- not stale, + // absent. Those columns are role 'date' and therefore sliceable, so a later + // adjustment scoped to a ship date matched no adjustment row ever written, + // and any reader that dated the forecast by ship date lost every + // adjustment silently. The derived period dimensions (rseas, smon_e) were + // fine throughout, being dimensions and copied with the rest -- which is + // what made it invisible: the labels were right and the dates underneath + // them were gone. + const dataCols = [...dims, ...dateCols, valueCol, unitsCol].filter(Boolean); const effectiveValue = dataCols.includes(valueCol) ? valueCol : null; const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null; const insertCols = [...dataCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', '); const selectData = dataCols.map(q).join(', '); const dimsJoined = dims.map(q).join(', '); + // The date columns as a select list, in the same order dataCols lists them, + // so it lines up with insertCols positionally. + const datesJoined = dateCols.map(q).join(', '); // Baseline and reference copy the source row wholesale, so they carry every // measure and every date — not just the primary one the operations act on. @@ -410,7 +422,7 @@ SELECT count(*) AS rows_affected, (SELECT id FROM ilog) AS log_id FROM ins`.trim // comes to the same figure the ungrouped window produced. const baseSelectParts = [ ...dimsJoined ? [dimsJoined] : [], - q(dateCol), + datesJoined, effectiveValue ? `sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : null, effectiveUnits ? `sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null, effectiveValue ? `sum(sum(${q(effectiveValue)})) OVER () AS total_value` : null, @@ -430,12 +442,12 @@ ilog AS ( WHERE {{where_clause}} {{exclude_clause}} GROUP BY - ${groupCols([...dims, dateCol])} + ${groupCols([...dims, ...dateCols])} ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) SELECT - ${[dimsJoined, q(dateCol), ...(effectiveValue ? [vSel] : []), ...(effectiveUnits ? [uSel] : [])].join(',\n ')}, + ${[dimsJoined, datesJoined, ...(effectiveValue ? [vSel] : []), ...(effectiveUnits ? [uSel] : [])].join(',\n ')}, 'scale', (SELECT id FROM ilog), '{{pf_user}}', now() FROM base RETURNING * @@ -454,23 +466,23 @@ ilog AS ( ,src AS ( SELECT ${dimsJoined}, - ${q(dateCol)}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} + ${datesJoined}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} GROUP BY - ${groupCols([...dims, dateCol])} + ${groupCols([...dims, ...dateCols])} ) ,neg AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${dimsJoined}, ${q(dateCol)}, ${effectiveValue ? `-${q(effectiveValue)}` : '0'}${effectiveUnits ? `, -${q(effectiveUnits)}` : ''}, + SELECT ${dimsJoined}, ${datesJoined}, ${effectiveValue ? `-${q(effectiveValue)}` : '0'}${effectiveUnits ? `, -${q(effectiveUnits)}` : ''}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() FROM src RETURNING * ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT {{set_clause}}, ${q(dateCol)}, ${effectiveValue ? q(effectiveValue) : '0'}${effectiveUnits ? `, ${q(effectiveUnits)}` : ''}, + SELECT {{set_clause}}, ${datesJoined}, ${effectiveValue ? q(effectiveValue) : '0'}${effectiveUnits ? `, ${q(effectiveUnits)}` : ''}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() FROM src RETURNING *