diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 3d655b9..62bd0b3 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -224,6 +224,22 @@ function generateSQL(source, colMeta) { const cloneCols = [...dims, ...dateCols, effectiveValue, effectiveUnits].filter(Boolean); const cloneInsertCols = [...cloneCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', '); + // An adjustment writes one row per *coordinate* it touches, not one per row + // it reads. + // + // Reading rows one-for-one meant every operation inherited the row count of + // everything before it: the baseline's rows plus every prior adjustment's + // rows at the same coordinate, so the table grew super-linearly with how + // much work had been done on it. Eight Pull Forward entries and the next + // scale over the same slice writes nine times what it needs to. + // + // Collapsing is over pf_logid and pf_iter only -- every stored dimension and + // date stays in the GROUP BY -- so no column goes null and nothing becomes + // unsliceable later. The distribution maths is untouched either way, since + // the window sums see the same totals whether or not the rows underneath + // them have been added up first. + const groupCols = (cols) => cols.map(q).join(',\n '); + const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols]; const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', '); const dateColSet = new Set(dateCols); @@ -388,13 +404,16 @@ SELECT count(*) AS rows_affected FROM ins`.trim(); const uSel = effectiveUnits ? `round((${q(effectiveUnits)} / NULLIF(total_units, 0)) * {{units_incr}}, 5)` : `0`; + // sum(sum(x)) OVER () is the aggregate of the aggregates: the window runs + // after the GROUP BY, so the total is over collapsed coordinates and + // comes to the same figure the ungrouped window produced. const baseSelectParts = [ ...dimsJoined ? [dimsJoined] : [], q(dateCol), - effectiveValue ? q(effectiveValue) : null, - effectiveUnits ? q(effectiveUnits) : null, - effectiveValue ? `sum(${q(effectiveValue)}) OVER () AS total_value` : null, - effectiveUnits ? `sum(${q(effectiveUnits)}) OVER () AS total_units` : null + effectiveValue ? `sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : null, + effectiveUnits ? `sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null, + effectiveValue ? `sum(sum(${q(effectiveValue)})) OVER () AS total_value` : null, + effectiveUnits ? `sum(sum(${q(effectiveUnits)})) OVER () AS total_units` : null ].filter(Boolean).join(',\n '); return ` WITH @@ -409,6 +428,8 @@ ilog AS ( FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} + GROUP BY + ${groupCols([...dims, dateCol])} ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) @@ -430,10 +451,14 @@ ilog AS ( RETURNING id ) ,src AS ( - SELECT ${selectData} + SELECT + ${dimsJoined}, + ${q(dateCol)}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} + GROUP BY + ${groupCols([...dims, dateCol])} ) ,neg AS ( INSERT INTO {{fc_table}} (${insertCols}) @@ -480,9 +505,15 @@ ilog AS ( SELECT ${select}, 'clone', (SELECT id FROM ilog), '{{pf_user}}', now() - FROM {{fc_table}} s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''} - WHERE {{where_clause}} - {{exclude_clause}} + FROM ( + SELECT + ${groupCols([...dims, ...dateCols])}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''} + FROM {{fc_table}} + WHERE {{where_clause}} + {{exclude_clause}} + GROUP BY + ${groupCols([...dims, ...dateCols])} + ) s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''} RETURNING * ) ${opTail('ins')}`.trim();