Write one adjustment row per coordinate, not per row read
Every operation inherited the row count of everything before it. Scale read the baseline's rows plus every prior adjustment's rows sitting at the same dimensional coordinate, and wrote a delta for each -- so eight Pull Forward entries meant the next scale over that slice wrote nine rows where one would do, and the table grew super-linearly with how much work had been done on it. The base sets are grouped now: scale's `base`, recode's `src`, and clone's source, each by every stored dimension and date, summing the measures. The collapse is over pf_logid and pf_iter alone, so no column goes null and nothing becomes unsliceable by a later operation -- which is the trap in collapsing to the display grain instead, where the non-grain dimensions would have to be null and the next slice naming one would silently miss these rows. The maths is unchanged. Scale's proportional split needs the total over the pool, and sum(sum(x)) OVER () gives the same figure over collapsed coordinates that sum(x) OVER () gave over raw ones -- the window runs after the GROUP BY. Verified on a real slice: 3,989 rows collapse to 3,187 at 1,303,545.75 either way. Across the version's existing adjustments it is 149,458 rows against 124,474, and that understates it, since the point is that the next layer no longer multiplies the last. All three templates planned against the live table before committing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
74fd030638
commit
f70de94e61
@ -224,6 +224,22 @@ function generateSQL(source, colMeta) {
|
|||||||
const cloneCols = [...dims, ...dateCols, effectiveValue, effectiveUnits].filter(Boolean);
|
const cloneCols = [...dims, ...dateCols, effectiveValue, effectiveUnits].filter(Boolean);
|
||||||
const cloneInsertCols = [...cloneCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
const cloneInsertCols = [...cloneCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
||||||
|
|
||||||
|
// An adjustment writes one row per *coordinate* it touches, not one per row
|
||||||
|
// it reads.
|
||||||
|
//
|
||||||
|
// Reading rows one-for-one meant every operation inherited the row count of
|
||||||
|
// everything before it: the baseline's rows plus every prior adjustment's
|
||||||
|
// rows at the same coordinate, so the table grew super-linearly with how
|
||||||
|
// much work had been done on it. Eight Pull Forward entries and the next
|
||||||
|
// scale over the same slice writes nine times what it needs to.
|
||||||
|
//
|
||||||
|
// Collapsing is over pf_logid and pf_iter only -- every stored dimension and
|
||||||
|
// date stays in the GROUP BY -- so no column goes null and nothing becomes
|
||||||
|
// unsliceable later. The distribution maths is untouched either way, since
|
||||||
|
// the window sums see the same totals whether or not the rows underneath
|
||||||
|
// them have been added up first.
|
||||||
|
const groupCols = (cols) => cols.map(q).join(',\n ');
|
||||||
|
|
||||||
const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols];
|
const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols];
|
||||||
const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
||||||
const dateColSet = new Set(dateCols);
|
const dateColSet = new Set(dateCols);
|
||||||
@ -388,13 +404,16 @@ SELECT count(*) AS rows_affected FROM ins`.trim();
|
|||||||
const uSel = effectiveUnits
|
const uSel = effectiveUnits
|
||||||
? `round((${q(effectiveUnits)} / NULLIF(total_units, 0)) * {{units_incr}}, 5)`
|
? `round((${q(effectiveUnits)} / NULLIF(total_units, 0)) * {{units_incr}}, 5)`
|
||||||
: `0`;
|
: `0`;
|
||||||
|
// sum(sum(x)) OVER () is the aggregate of the aggregates: the window runs
|
||||||
|
// after the GROUP BY, so the total is over collapsed coordinates and
|
||||||
|
// comes to the same figure the ungrouped window produced.
|
||||||
const baseSelectParts = [
|
const baseSelectParts = [
|
||||||
...dimsJoined ? [dimsJoined] : [],
|
...dimsJoined ? [dimsJoined] : [],
|
||||||
q(dateCol),
|
q(dateCol),
|
||||||
effectiveValue ? q(effectiveValue) : null,
|
effectiveValue ? `sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : null,
|
||||||
effectiveUnits ? q(effectiveUnits) : null,
|
effectiveUnits ? `sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null,
|
||||||
effectiveValue ? `sum(${q(effectiveValue)}) OVER () AS total_value` : null,
|
effectiveValue ? `sum(sum(${q(effectiveValue)})) OVER () AS total_value` : null,
|
||||||
effectiveUnits ? `sum(${q(effectiveUnits)}) OVER () AS total_units` : null
|
effectiveUnits ? `sum(sum(${q(effectiveUnits)})) OVER () AS total_units` : null
|
||||||
].filter(Boolean).join(',\n ');
|
].filter(Boolean).join(',\n ');
|
||||||
return `
|
return `
|
||||||
WITH
|
WITH
|
||||||
@ -409,6 +428,8 @@ ilog AS (
|
|||||||
FROM {{fc_table}}
|
FROM {{fc_table}}
|
||||||
WHERE {{where_clause}}
|
WHERE {{where_clause}}
|
||||||
{{exclude_clause}}
|
{{exclude_clause}}
|
||||||
|
GROUP BY
|
||||||
|
${groupCols([...dims, dateCol])}
|
||||||
)
|
)
|
||||||
,ins AS (
|
,ins AS (
|
||||||
INSERT INTO {{fc_table}} (${insertCols})
|
INSERT INTO {{fc_table}} (${insertCols})
|
||||||
@ -430,10 +451,14 @@ ilog AS (
|
|||||||
RETURNING id
|
RETURNING id
|
||||||
)
|
)
|
||||||
,src AS (
|
,src AS (
|
||||||
SELECT ${selectData}
|
SELECT
|
||||||
|
${dimsJoined},
|
||||||
|
${q(dateCol)}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''}
|
||||||
FROM {{fc_table}}
|
FROM {{fc_table}}
|
||||||
WHERE {{where_clause}}
|
WHERE {{where_clause}}
|
||||||
{{exclude_clause}}
|
{{exclude_clause}}
|
||||||
|
GROUP BY
|
||||||
|
${groupCols([...dims, dateCol])}
|
||||||
)
|
)
|
||||||
,neg AS (
|
,neg AS (
|
||||||
INSERT INTO {{fc_table}} (${insertCols})
|
INSERT INTO {{fc_table}} (${insertCols})
|
||||||
@ -480,9 +505,15 @@ ilog AS (
|
|||||||
SELECT
|
SELECT
|
||||||
${select},
|
${select},
|
||||||
'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
|
'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
|
||||||
FROM {{fc_table}} s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''}
|
FROM (
|
||||||
|
SELECT
|
||||||
|
${groupCols([...dims, ...dateCols])}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''}
|
||||||
|
FROM {{fc_table}}
|
||||||
WHERE {{where_clause}}
|
WHERE {{where_clause}}
|
||||||
{{exclude_clause}}
|
{{exclude_clause}}
|
||||||
|
GROUP BY
|
||||||
|
${groupCols([...dims, ...dateCols])}
|
||||||
|
) s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''}
|
||||||
RETURNING *
|
RETURNING *
|
||||||
)
|
)
|
||||||
${opTail('ins')}`.trim();
|
${opTail('ins')}`.trim();
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user