Paul Trowbridge
2337951b65
pf_segment and pf_bucket were resolved at read time through a chain of fallbacks (label, tag, note, the version's unlabeled_load; the version's adjustment defaults), repeated in /agg, the /data cursor and the decorate proc, so nothing but the pivot could say what bucket a row was in. Now they are written onto pf.log when the entry is created: a load's label is required and its bucket defaults to the label; an adjustment is stamped with the version's adjustment_segment/adjustment_bucket, else the built-ins. Reads become l.label / l.bucket. A log_named CHECK guarantees both are present, and 01_schema.sql backfills existing entries with the old read-time rule so no displayed name changes. The Baseline page's "Fallback names" become "Defaults for new adjustments"; unlabeled_load is no longer read. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
816 lines
37 KiB
JavaScript
816 lines
37 KiB
JavaScript
// Generates operation SQL for a source table, baking in column names from col_meta.
|
|
// Runtime values are left as {{token}} substitution points.
|
|
//
|
|
// Columns flagged col_meta.in_grain define a display grain. When one is set the
|
|
// initial load (get_agg) and every operation return rows pre-aggregated to that
|
|
// grain and keyed on pf_gkey, instead of raw forecast rows keyed on pf_id.
|
|
//
|
|
// Tokens baked in at generation time: column names, source schema.table
|
|
// Tokens substituted at request time: {{fc_table}}, {{where_clause}}, {{exclude_clause}},
|
|
// {{version_id}}, {{logid}}, {{pf_user}}, {{note}},
|
|
// {{label}}, {{bucket}}, {{tag}}, {{territory_clause}},
|
|
// {{params}}, {{slice}}, {{date_from}}, {{date_to}},
|
|
// {{value_incr}}, {{units_incr}}, {{set_clause}}, {{scale_factor}}
|
|
|
|
// What the pivot shows for a row's segment and its bucket.
|
|
//
|
|
// Both are STORED on the row's pf.log entry -- l.label and l.bucket -- and read
|
|
// back as plain columns. Every entry carries both: pf.log's log_named CHECK
|
|
// refuses one without, so a report can join pf.log and select them with no rule
|
|
// of its own.
|
|
//
|
|
// They used to be derived at read time, through a chain of fallbacks (a load's
|
|
// label, else its tag, else its note, else the version's unlabeled_load; an
|
|
// adjustment's label, else the version's adjustment_segment, else a constant).
|
|
// That made the pivot the only reader that knew what bucket a row was in: the
|
|
// same CASE was repeated in /agg, the /data cursor and the decorate proc in
|
|
// /opt/forecast, and anything querying pf directly had to copy it again. It also
|
|
// meant editing a version's fallback name silently renamed every past adjustment.
|
|
//
|
|
// The ordering prefix is part of the stored text, not computed here. Perspective
|
|
// orders column groups by the value string, so "01 - Actual" is the only way an
|
|
// arbitrary order can be expressed -- and l.label is where a person types it.
|
|
//
|
|
// ---------------------------------------------------------------------------
|
|
// ADJUSTMENT DEFAULTS -- what a scale/recode/clone is stamped with.
|
|
//
|
|
// An adjustment has no label or bucket input, so the route stamps these onto its
|
|
// log entry when it is written: the version's own adjustment_segment /
|
|
// adjustment_bucket when set, else the constants below. Resolved once, at write
|
|
// time -- changing a version's default afterwards names new adjustments, not old
|
|
// ones. Exported so the operation routes stamp the rows they push back to the
|
|
// pivot with the same values they wrote.
|
|
//
|
|
// A load has no default: its label is required, and its bucket is its label
|
|
// unless one is given (the routes enforce both).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// An adjustment with no label of its own. The 99 keeps it after every numbered
|
|
// segment -- ordering is string ordering, so this only works while the loads
|
|
// carry 01-0n.
|
|
const ADJUSTMENT_SEGMENT = '99 - Adjustments';
|
|
|
|
// What an adjustment counts toward. Prefixed to match the segments it adjusts,
|
|
// so the adjustments sit in the same column as the rows they adjust.
|
|
const ADJUSTMENT_BUCKET = '04 - Forecast';
|
|
|
|
const SEGMENT_EXPR = `l.label`;
|
|
const BUCKET_EXPR = `l.bucket`;
|
|
|
|
const NOTE_EXPR = `CASE WHEN l.operation IN ('baseline','reference')
|
|
THEN NULL
|
|
ELSE COALESCE(NULLIF(l.tag, ''), NULLIF(l.note, ''))
|
|
END`;
|
|
|
|
// Every pf.log column the expressions above read, for /agg's GROUP BY: they
|
|
// are functionally dependent on pf_logid, which is in the grain, but Postgres
|
|
// will not infer that.
|
|
const LABEL_GROUP_COLS = ['l.operation', 'l.label', 'l.tag', 'l.note', 'l.bucket'];
|
|
|
|
// wrap a column name in double quotes for safe use in SQL
|
|
function q(name) { return `"${name}"`; }
|
|
|
|
// The display grain: dimension/date columns flagged in_grain, plus pf_iter and
|
|
// pf_logid which are always part of it. Returns null when nothing is flagged —
|
|
// that is raw-row mode, where operations return whole rows and the client
|
|
// indexes on pf_id (the pre-grain behaviour).
|
|
//
|
|
// Keeping pf_logid in the grain is what makes the append model work: each
|
|
// operation's contribution stays a distinct row, so table.update() accumulates
|
|
// rather than replacing a bucket total, and undo can remove exactly that
|
|
// operation's rows.
|
|
function grainOf(colMeta) {
|
|
const cols = colMeta
|
|
.filter(c => c.in_grain && (c.role === 'dimension' || c.role === 'date'))
|
|
.sort((a, b) => (a.opos || 0) - (b.opos || 0))
|
|
.map(c => c.cname);
|
|
if (cols.length === 0) return null;
|
|
|
|
// pf_gkey must be unique per grain tuple. chr(31) (unit separator) joins the
|
|
// parts and chr(30) stands in for NULL, so ('a', NULL) cannot collide with
|
|
// (NULL, 'a') and a NULL stays distinct from an empty string — a collision
|
|
// would silently merge two groups into one indexed row.
|
|
//
|
|
// md5 of that, rather than the concatenation itself, because the key is an
|
|
// opaque handle -- nothing reads it but table.update() and table.remove().
|
|
// The raw form averaged 233 chars on a 24-column grain and, being unique per
|
|
// row, defeated Arrow's dictionary encoding: 65.6 MB of a 109 MB payload,
|
|
// more than every other column combined. 128 bits keeps collisions unreachable.
|
|
const key = (pfx = '') => `md5(concat_ws(chr(31), ${[
|
|
...cols.map(c => `COALESCE(${pfx}${q(c)}::text, chr(30))`),
|
|
`${pfx}pf_iter`,
|
|
`${pfx}pf_logid::text`
|
|
].join(', ')}))`;
|
|
|
|
const groupCols = (pfx = '') => [...cols.map(c => `${pfx}${q(c)}`), `${pfx}pf_iter`, `${pfx}pf_logid`];
|
|
|
|
return { cols, key, groupCols };
|
|
}
|
|
|
|
// Date columns that anchor a dim_group, each with the dimensions derived from
|
|
// pf.dim_period for it. Shared so the generator and the routes agree on both the
|
|
// membership and the join aliases -- the same reason grainOf is a single function.
|
|
function dateGroupsOf(colMeta) {
|
|
return colMeta
|
|
.filter(c => c.role === 'date' && c.is_key && c.dim_group)
|
|
.map((keyCol, i) => ({
|
|
alias: `dp${i + 1}`,
|
|
dateCol: keyCol.cname,
|
|
group: keyCol.dim_group,
|
|
derived: colMeta
|
|
.filter(c => c.role === 'dimension'
|
|
&& c.dim_group === keyCol.dim_group
|
|
&& c.dim_period_col)
|
|
.map(c => ({ cname: c.cname, periodCol: c.dim_period_col })),
|
|
}))
|
|
.filter(g => g.derived.length > 0);
|
|
}
|
|
|
|
// cname -> which join it comes from and which of its columns
|
|
function dimPeriodMapOf(dateGroups) {
|
|
return new Map(
|
|
dateGroups.flatMap(g => g.derived.map(d => [d.cname, { alias: g.alias, periodCol: d.periodCol }]))
|
|
);
|
|
}
|
|
|
|
// The joins themselves, against a date that has already had {{date_offset}}
|
|
// applied. LEFT because a null date -- an order not yet shipped has no ship date
|
|
// -- must leave the period columns empty rather than drop the row.
|
|
function dimPeriodJoins(dateGroups, alias = 's') {
|
|
return dateGroups.map(g =>
|
|
`\n LEFT JOIN pf.dim_period ${g.alias}`
|
|
+ ` ON ${g.alias}.drange @> (${alias}."${g.dateCol}" + '{{date_offset}}'::interval)::date`
|
|
).join('');
|
|
}
|
|
|
|
function generateSQL(source, colMeta) {
|
|
const dims = colMeta
|
|
.filter(c => c.role === 'dimension')
|
|
.sort((a, b) => (a.opos || 0) - (b.opos || 0))
|
|
.map(c => c.cname);
|
|
|
|
// Every column of each measure/date role, in col_meta order. Loads carry all of
|
|
// them; the adjustment operations are single-measure (scale distributes one
|
|
// {{value_incr}}) and use only the first of each, below.
|
|
const byRole = role => colMeta
|
|
.filter(c => c.role === role)
|
|
.sort((a, b) => (a.opos || 0) - (b.opos || 0))
|
|
.map(c => c.cname);
|
|
|
|
const valueCols = byRole('value');
|
|
const unitsCols = byRole('units');
|
|
const dateCols = byRole('date');
|
|
|
|
const valueCol = valueCols[0];
|
|
const unitsCol = unitsCols[0];
|
|
|
|
if (!valueCol) throw new Error('No value column defined in col_meta');
|
|
if (!dateCols.length) throw new Error('No date column defined in col_meta');
|
|
if (dims.length === 0) throw new Error('No dimension columns defined in col_meta');
|
|
|
|
const srcTable = `"${source.schema}"."${source.tname}"`;
|
|
// Every date column, not just the primary one. Scale and recode used to
|
|
// carry dateCols[0] alone, so a source with order, requested and ship dates
|
|
// wrote adjustment rows whose rdate and sdate_e were *null* -- not stale,
|
|
// absent. Those columns are role 'date' and therefore sliceable, so a later
|
|
// adjustment scoped to a ship date matched no adjustment row ever written,
|
|
// and any reader that dated the forecast by ship date lost every
|
|
// adjustment silently. The derived period dimensions (rseas, smon_e) were
|
|
// fine throughout, being dimensions and copied with the rest -- which is
|
|
// what made it invisible: the labels were right and the dates underneath
|
|
// them were gone.
|
|
const dataCols = [...dims, ...dateCols, valueCol, unitsCol].filter(Boolean);
|
|
const effectiveValue = dataCols.includes(valueCol) ? valueCol : null;
|
|
const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null;
|
|
const insertCols = [...dataCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
|
const selectData = dataCols.map(q).join(', ');
|
|
const dimsJoined = dims.map(q).join(', ');
|
|
// The date columns as a select list, in the same order dataCols lists them,
|
|
// so it lines up with insertCols positionally.
|
|
const datesJoined = dateCols.map(q).join(', ');
|
|
|
|
// Baseline and reference copy the source row wholesale, so they carry every
|
|
// measure and every date — not just the primary one the operations act on.
|
|
// Dropping the others would leave those columns null for the life of the version.
|
|
// Clone carries every date column, not just the primary one, because it is the
|
|
// operation that moves rows through time: {{date_offset}} shifts them all
|
|
// together, and the period dimensions are re-derived from pf.dim_period against
|
|
// the shifted dates rather than copied from the row being cloned. Cloning last
|
|
// year's mix forward a year otherwise produces rows dated 2027 still labelled
|
|
// with 2026's periods.
|
|
const cloneCols = [...dims, ...dateCols, effectiveValue, effectiveUnits].filter(Boolean);
|
|
const cloneInsertCols = [...cloneCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
|
|
|
// An adjustment writes one row per *coordinate* it touches, not one per row
|
|
// it reads.
|
|
//
|
|
// Reading rows one-for-one meant every operation inherited the row count of
|
|
// everything before it: the baseline's rows plus every prior adjustment's
|
|
// rows at the same coordinate, so the table grew super-linearly with how
|
|
// much work had been done on it. Eight Pull Forward entries and the next
|
|
// scale over the same slice writes nine times what it needs to.
|
|
//
|
|
// Collapsing is over pf_logid and pf_iter only -- every stored dimension and
|
|
// date stays in the GROUP BY -- so no column goes null and nothing becomes
|
|
// unsliceable later. The distribution maths is untouched either way, since
|
|
// the window sums see the same totals whether or not the rows underneath
|
|
// them have been added up first.
|
|
const groupCols = (cols) => cols.map(q).join(',\n ');
|
|
|
|
const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols];
|
|
const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
|
const dateColSet = new Set(dateCols);
|
|
|
|
// dim_period JOIN support: a date column that is the is_key of a dim_group
|
|
// anchors that group, and dimension siblings with dim_period_col set are
|
|
// derived from pf.dim_period instead of copied raw. Derivation is against the
|
|
// date *after* {{date_offset}}, which is the whole point -- shift a baseline
|
|
// forward a year and its period columns follow, rather than still naming the
|
|
// year it came from.
|
|
//
|
|
// Every such group, not just the first. This used to be a find(), so a source
|
|
// with order, requested and ship date groups derived the order one and copied
|
|
// the other two raw -- shifted dates against unshifted period labels.
|
|
const dateGroups = dateGroupsOf(colMeta);
|
|
const dimPeriodMap = dimPeriodMapOf(dateGroups);
|
|
const hasDimPeriod = dimPeriodMap.size > 0;
|
|
|
|
// display grain — when set, initial load and operations both return rows
|
|
// pre-aggregated to it instead of raw forecast rows
|
|
const grain = grainOf(colMeta);
|
|
// A flag on anything other than a dimension/date column is ignored by grainOf,
|
|
// which is what we want — the role change is the source of truth, not a stale flag.
|
|
if (grain) {
|
|
const missing = grain.cols.filter(c => !dataCols.includes(c));
|
|
if (missing.length > 0) {
|
|
throw new Error(
|
|
`Grain columns are never populated in the forecast table: ${missing.join(', ')}`
|
|
);
|
|
}
|
|
if (!effectiveValue && !effectiveUnits) {
|
|
throw new Error('A grain requires at least one value or units column to aggregate');
|
|
}
|
|
}
|
|
|
|
return {
|
|
get_data: buildGetData(),
|
|
...(grain ? { get_agg: buildGetAgg() } : {}),
|
|
baseline: buildBaseline(),
|
|
reference: buildReference(),
|
|
scale: buildScale(),
|
|
recode: buildRecode(),
|
|
clone: buildClone(),
|
|
undo: buildUndo()
|
|
};
|
|
|
|
function buildGetData() {
|
|
return `SELECT * FROM {{fc_table}}`;
|
|
}
|
|
|
|
// Aggregate the whole forecast table to the display grain. This is the initial
|
|
// load for grain sources — the client loads the result into a native Perspective
|
|
// table indexed on pf_gkey and its view sums across these rows, exactly as an
|
|
// Excel pivot cache sums its data tab.
|
|
function buildGetAgg() {
|
|
// pf_logid is part of the grain, so joining pf.log adds no rows — each group
|
|
// already belongs to exactly one log entry. Without this the segment labels
|
|
// that /data surfaces would vanish the moment a source declares a grain.
|
|
return `
|
|
SELECT
|
|
${grainSelect('t.')}
|
|
,${SEGMENT_EXPR} AS pf_segment
|
|
,${BUCKET_EXPR} AS pf_bucket
|
|
,${NOTE_EXPR} AS pf_note
|
|
,l.operation AS pf_op
|
|
FROM {{fc_table}} t
|
|
LEFT JOIN pf.log l
|
|
ON l.id = t.pf_logid
|
|
WHERE {{territory_clause}}
|
|
GROUP BY
|
|
${grain.groupCols('t.').join('\n ,')}
|
|
,${LABEL_GROUP_COLS.join('\n ,')}`.trim();
|
|
}
|
|
|
|
// grain columns + pf_gkey + summed measures, in the leading-comma style the
|
|
// rest of the generated SQL uses
|
|
function grainSelect(pfx = '') {
|
|
return [
|
|
...grain.groupCols(pfx),
|
|
`${grain.key(pfx)} AS pf_gkey`,
|
|
effectiveValue ? `SUM(${pfx}${q(effectiveValue)}) AS ${q(effectiveValue)}` : null,
|
|
effectiveUnits ? `SUM(${pfx}${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null
|
|
].filter(Boolean).join('\n ,');
|
|
}
|
|
|
|
// Tail of an operation statement: in grain mode the inserted rows come back
|
|
// aggregated to grain (the client appends them and lets the view re-sum);
|
|
// otherwise whole rows come back as before.
|
|
function opTail(cte) {
|
|
if (!grain) return `SELECT * FROM ${cte}`;
|
|
return `
|
|
SELECT
|
|
${grainSelect()}
|
|
FROM ${cte}
|
|
GROUP BY
|
|
${grain.groupCols().join('\n ,')}`.trim();
|
|
}
|
|
|
|
function buildLoadSelect(pfx) {
|
|
// pfx: table alias prefix ('s.' when joining dim_period, '' otherwise)
|
|
// The offset shifts every date column, so order date and ship date stay in step.
|
|
return loadCols.map(c => {
|
|
if (dateColSet.has(c)) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`;
|
|
if (dimPeriodMap.has(c)) {
|
|
const { alias, periodCol } = dimPeriodMap.get(c);
|
|
return `${alias}.${q(periodCol)} AS ${q(c)}`;
|
|
}
|
|
return `${pfx}${q(c)}`;
|
|
}).join(',\n ');
|
|
}
|
|
|
|
function buildFromClause() {
|
|
if (!hasDimPeriod) return srcTable;
|
|
return srcTable + ' s' + dimPeriodJoins(dateGroups);
|
|
}
|
|
|
|
function buildBaseline() {
|
|
return `
|
|
WITH
|
|
ilog AS (
|
|
INSERT INTO pf.log (version_id, pf_user, operation, slice, params, note, label, bucket, tag)
|
|
VALUES ({{version_id}}, '{{pf_user}}', 'baseline', NULL, '{{params}}'::jsonb, '{{note}}',
|
|
NULLIF('{{label}}', ''), NULLIF('{{bucket}}', ''), NULLIF('{{tag}}', ''))
|
|
RETURNING id
|
|
)
|
|
,ins AS (
|
|
INSERT INTO {{fc_table}} (${loadInsertCols})
|
|
SELECT
|
|
${buildLoadSelect(hasDimPeriod ? 's.' : '')},
|
|
'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM ${buildFromClause()}
|
|
WHERE {{filter_clause}}
|
|
RETURNING *
|
|
)
|
|
SELECT count(*) AS rows_affected, (SELECT id FROM ilog) AS log_id FROM ins`.trim();
|
|
}
|
|
|
|
function buildReference() {
|
|
return `
|
|
WITH
|
|
ilog AS (
|
|
INSERT INTO pf.log (version_id, pf_user, operation, slice, params, note, label, bucket, tag)
|
|
VALUES ({{version_id}}, '{{pf_user}}', 'reference', NULL, '{{params}}'::jsonb, '{{note}}',
|
|
NULLIF('{{label}}', ''), NULLIF('{{bucket}}', ''), NULLIF('{{tag}}', ''))
|
|
RETURNING id
|
|
)
|
|
,ins AS (
|
|
INSERT INTO {{fc_table}} (${loadInsertCols})
|
|
SELECT
|
|
${buildLoadSelect(hasDimPeriod ? 's.' : '')},
|
|
'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM ${buildFromClause()}
|
|
WHERE {{filter_clause}}
|
|
RETURNING *
|
|
)
|
|
SELECT count(*) AS rows_affected, (SELECT id FROM ilog) AS log_id FROM ins`.trim();
|
|
}
|
|
|
|
function buildScale() {
|
|
const vSel = effectiveValue
|
|
? `round((${q(effectiveValue)} / NULLIF(total_value, 0)) * {{value_incr}}, 2)`
|
|
: `0`;
|
|
const uSel = effectiveUnits
|
|
? `round((${q(effectiveUnits)} / NULLIF(total_units, 0)) * {{units_incr}}, 5)`
|
|
: `0`;
|
|
// sum(sum(x)) OVER () is the aggregate of the aggregates: the window runs
|
|
// after the GROUP BY, so the total is over collapsed coordinates and
|
|
// comes to the same figure the ungrouped window produced.
|
|
const baseSelectParts = [
|
|
...dimsJoined ? [dimsJoined] : [],
|
|
datesJoined,
|
|
effectiveValue ? `sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : null,
|
|
effectiveUnits ? `sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : null,
|
|
effectiveValue ? `sum(sum(${q(effectiveValue)})) OVER () AS total_value` : null,
|
|
effectiveUnits ? `sum(sum(${q(effectiveUnits)})) OVER () AS total_units` : null
|
|
].filter(Boolean).join(',\n ');
|
|
return `
|
|
WITH
|
|
ilog AS (
|
|
INSERT INTO pf.log (version_id, pf_user, operation, slice, params, note, label, bucket)
|
|
VALUES ({{version_id}}, '{{pf_user}}', 'scale', '{{slice}}'::jsonb, '{{params}}'::jsonb, '{{note}}',
|
|
NULLIF('{{label}}', ''), NULLIF('{{bucket}}', ''))
|
|
RETURNING id
|
|
)
|
|
,base AS (
|
|
SELECT
|
|
${baseSelectParts}
|
|
FROM {{fc_table}}
|
|
WHERE {{where_clause}}
|
|
{{exclude_clause}}
|
|
GROUP BY
|
|
${groupCols([...dims, ...dateCols])}
|
|
)
|
|
,ins AS (
|
|
INSERT INTO {{fc_table}} (${insertCols})
|
|
SELECT
|
|
${[dimsJoined, datesJoined, ...(effectiveValue ? [vSel] : []), ...(effectiveUnits ? [uSel] : [])].join(',\n ')},
|
|
'scale', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM base
|
|
RETURNING *
|
|
)
|
|
${opTail('ins')}`.trim();
|
|
}
|
|
|
|
function buildRecode() {
|
|
return `
|
|
WITH
|
|
ilog AS (
|
|
INSERT INTO pf.log (version_id, pf_user, operation, slice, params, note, label, bucket)
|
|
VALUES ({{version_id}}, '{{pf_user}}', 'recode', '{{slice}}'::jsonb, '{{params}}'::jsonb, '{{note}}',
|
|
NULLIF('{{label}}', ''), NULLIF('{{bucket}}', ''))
|
|
RETURNING id
|
|
)
|
|
,src AS (
|
|
SELECT
|
|
${dimsJoined},
|
|
${datesJoined}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''}
|
|
FROM {{fc_table}}
|
|
WHERE {{where_clause}}
|
|
{{exclude_clause}}
|
|
GROUP BY
|
|
${groupCols([...dims, ...dateCols])}
|
|
)
|
|
,neg AS (
|
|
INSERT INTO {{fc_table}} (${insertCols})
|
|
SELECT ${dimsJoined}, ${datesJoined}, ${effectiveValue ? `-${q(effectiveValue)}` : '0'}${effectiveUnits ? `, -${q(effectiveUnits)}` : ''},
|
|
'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM src
|
|
RETURNING *
|
|
)
|
|
,ins AS (
|
|
INSERT INTO {{fc_table}} (${insertCols})
|
|
SELECT {{set_clause}}, ${datesJoined}, ${effectiveValue ? q(effectiveValue) : '0'}${effectiveUnits ? `, ${q(effectiveUnits)}` : ''},
|
|
'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM src
|
|
RETURNING *
|
|
)
|
|
${grain ? `,allrows AS (
|
|
SELECT * FROM neg
|
|
UNION ALL
|
|
SELECT * FROM ins
|
|
)
|
|
${opTail('allrows')}` : 'SELECT * FROM neg UNION ALL SELECT * FROM ins'}`.trim();
|
|
}
|
|
|
|
function buildClone() {
|
|
const select = [
|
|
// dims: whatever {{set_clause}} resolves them to. The route builds it,
|
|
// and substitutes the dim_period expression for any derived dimension
|
|
// the caller has not overridden outright.
|
|
'{{set_clause}}',
|
|
...dateCols.map(c => `(s.${q(c)} + '{{date_offset}}'::interval)::date`),
|
|
effectiveValue ? `round(s.${q(effectiveValue)} * {{scale_factor}}, 2)` : null,
|
|
effectiveUnits ? `round(s.${q(effectiveUnits)} * {{scale_factor}}, 5)` : null,
|
|
].filter(Boolean).join(',\n ');
|
|
|
|
return `
|
|
WITH
|
|
ilog AS (
|
|
INSERT INTO pf.log (version_id, pf_user, operation, slice, params, note, label, bucket)
|
|
VALUES ({{version_id}}, '{{pf_user}}', 'clone', '{{slice}}'::jsonb, '{{params}}'::jsonb, '{{note}}',
|
|
NULLIF('{{label}}', ''), NULLIF('{{bucket}}', ''))
|
|
RETURNING id
|
|
)
|
|
,ins AS (
|
|
INSERT INTO {{fc_table}} (${cloneInsertCols})
|
|
SELECT
|
|
${select},
|
|
'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
|
|
FROM (
|
|
SELECT
|
|
${groupCols([...dims, ...dateCols])}${effectiveValue ? `,\n sum(${q(effectiveValue)}) AS ${q(effectiveValue)}` : ''}${effectiveUnits ? `,\n sum(${q(effectiveUnits)}) AS ${q(effectiveUnits)}` : ''}
|
|
FROM {{fc_table}}
|
|
WHERE {{where_clause}}
|
|
{{exclude_clause}}
|
|
GROUP BY
|
|
${groupCols([...dims, ...dateCols])}
|
|
) s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''}
|
|
RETURNING *
|
|
)
|
|
${opTail('ins')}`.trim();
|
|
}
|
|
|
|
function buildUndo() {
|
|
// undo is executed as two separate queries in the route handler
|
|
// (delete from fc_table first, then delete from pf.log) to avoid
|
|
// FK constraint ordering issues within a single CTE statement.
|
|
// This entry is a placeholder — the undo route uses it as a template reference.
|
|
return `
|
|
-- step 1 (run first):
|
|
DELETE FROM {{fc_table}} WHERE pf_logid = {{logid}};
|
|
-- step 2 (run after step 1):
|
|
DELETE FROM pf.log WHERE id = {{logid}};`.trim();
|
|
}
|
|
}
|
|
|
|
// substitute {{token}} placeholders in a SQL string
|
|
function applyTokens(sql, tokens) {
|
|
let result = sql;
|
|
for (const [key, value] of Object.entries(tokens)) {
|
|
result = result.replace(new RegExp(`\\{\\{${key}\\}\\}`, 'g'), value ?? '');
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// build a SQL WHERE clause string from a slice object
|
|
// only dimension columns are included; unrecognised keys are silently skipped
|
|
// pf_segment and pf_bucket are not columns on the forecast table -- they are
|
|
// stored on the row's pf.log entry -- so a slice naming one cannot be compared
|
|
// directly. It resolves to a set of log ids instead, which is
|
|
// exact: the name lives on the log row, and every forecast row carries the
|
|
// pf_logid that points at it.
|
|
//
|
|
// Without this they were dropped from the slice, and clicking a single bucket's
|
|
// cell scaled every bucket at that dimension intersection while the panel showed
|
|
// only the one clicked.
|
|
const COMPUTED_SLICE_COLS = { pf_segment: SEGMENT_EXPR, pf_bucket: BUCKET_EXPR };
|
|
|
|
function computedSlicePredicate(col, val, versionId) {
|
|
if (versionId == null) {
|
|
const err = new Error(`Cannot filter on ${col} without a version`);
|
|
err.status = 500;
|
|
throw err;
|
|
}
|
|
const vals = (Array.isArray(val) ? val : [val]).map(v => `'${esc(v)}'`).join(', ');
|
|
return `pf_logid IN (
|
|
SELECT l.id
|
|
FROM pf.log l
|
|
WHERE l.version_id = ${parseInt(versionId)}
|
|
AND ${COMPUTED_SLICE_COLS[col]} IN (${vals})
|
|
)`;
|
|
}
|
|
|
|
function buildWhere(slice, dimCols, versionId) {
|
|
if (!slice || Object.keys(slice).length === 0) return 'TRUE';
|
|
|
|
const allowed = new Set(dimCols);
|
|
const parts = [];
|
|
|
|
for (const [col, val] of Object.entries(slice)) {
|
|
if (COMPUTED_SLICE_COLS[col]) {
|
|
parts.push(computedSlicePredicate(col, val, versionId));
|
|
continue;
|
|
}
|
|
// A pf_ key this does not understand is refused rather than skipped.
|
|
// Skipping is how a selection silently widened: the operation ran against
|
|
// everything the dropped key would have excluded. pf_iter is the one
|
|
// exception -- the client strips it deliberately, since two cells that
|
|
// differ only by iter band are the same slice.
|
|
if (col.startsWith('pf_') && col !== 'pf_iter') {
|
|
const err = new Error(`Slice names ${col}, which cannot be filtered on`);
|
|
err.status = 400;
|
|
throw err;
|
|
}
|
|
if (!allowed.has(col)) continue;
|
|
// A blank dimension value is a value. Both blanks exist in this data and
|
|
// they are different rows -- on fc_osm_skinny_34, partgroup is null on
|
|
// 51,964 and '' on 111,696 -- so a null has to become IS NULL. It used
|
|
// to fall through esc() to `= ''`, which selected the *other* blank:
|
|
// the slice named one node and the operation wrote the other.
|
|
if (val === null) {
|
|
parts.push(`"${col}" IS NULL`);
|
|
} else if (Array.isArray(val)) {
|
|
const nulls = val.some(v => v === null);
|
|
const escaped = val.filter(v => v !== null).map(v => esc(v));
|
|
const inList = escaped.length ? `"${col}" IN ('${escaped.join("', '")}')` : null;
|
|
// an IN list cannot carry a null -- `IN (NULL)` is never true -- so
|
|
// the null arm is ORed on rather than folded in
|
|
parts.push(
|
|
nulls && inList ? `("${col}" IS NULL OR ${inList})`
|
|
: nulls ? `"${col}" IS NULL`
|
|
: inList ? inList
|
|
: 'FALSE'
|
|
);
|
|
} else {
|
|
parts.push(`"${col}" = '${esc(val)}'`);
|
|
}
|
|
}
|
|
|
|
return parts.length ? parts.join('\nAND ') : 'TRUE';
|
|
}
|
|
|
|
// The pivot's own filter, carried alongside the slices so an operation writes
|
|
// exactly the rows the ledger counted.
|
|
//
|
|
// A slice is {col: value} and can only ever mean equality, so a view filtered to
|
|
// sseas_e <= 2027 could not be expressed as one. Refusing it was safe but
|
|
// useless -- a bounded season is an ordinary way to scope a forecast -- so the
|
|
// operators travel as [col, op, value] triples instead.
|
|
//
|
|
// Perspective's operator names, not SQL's, since that is where these come from.
|
|
// Anything outside this list is refused rather than ignored: a scope silently
|
|
// dropped is a write that is wider than the panel that authorised it.
|
|
//
|
|
// The substring operators are the ones a text dimension is actually filtered
|
|
// with -- `partd contains 'Keysto'` picks a product family out of a description
|
|
// nobody would enumerate by hand -- and they are **case-insensitive** in the
|
|
// engine: `t_tscalar::contains` lowercases both sides before
|
|
// `std::string::find` (scalar.cpp), as do `begins_with` and `ends_with`. So the
|
|
// SQL is ILIKE, not LIKE. LIKE would be *narrower* than the grid that
|
|
// authorised the write, which is the failure this whole mechanism exists to
|
|
// prevent, and narrower is no safer than wider: it writes a set the ledger did
|
|
// not count either way.
|
|
//
|
|
// Their term is matched literally there, so LIKE's own metacharacters are
|
|
// escaped out of it by `likeTerm` -- a filter on `50%` means those three
|
|
// characters, not "50 followed by anything".
|
|
//
|
|
// Not identical in one corner: the engine folds case byte by byte, so ASCII
|
|
// only, where ILIKE follows the database collation. They differ only on
|
|
// non-ASCII letters.
|
|
//
|
|
// `matches` / `not matches` are deliberately still refused. They are RE2 in the
|
|
// engine and Postgres regexes are not RE2 -- `\b` is a word boundary in one and
|
|
// a backspace in the other -- so a pattern valid in both could quietly match a
|
|
// different set of rows, which is worse than answering 400.
|
|
const likeTerm = (val, pre, post) =>
|
|
`'${esc(pre + String(val).replace(/[\\%_]/g, m => `\\${m}`) + post)}'`;
|
|
|
|
const SCOPE_OPS = {
|
|
'==': (c, v) => `${c} = ${v[0]}`,
|
|
'!=': (c, v) => `${c} != ${v[0]}`,
|
|
'>': (c, v) => `${c} > ${v[0]}`,
|
|
'>=': (c, v) => `${c} >= ${v[0]}`,
|
|
'<': (c, v) => `${c} < ${v[0]}`,
|
|
'<=': (c, v) => `${c} <= ${v[0]}`,
|
|
'in': (c, v) => `${c} IN (${v.join(', ')})`,
|
|
'not in': (c, v) => `${c} NOT IN (${v.join(', ')})`,
|
|
'is null': (c) => `${c} IS NULL`,
|
|
'is not null': (c) => `${c} IS NOT NULL`,
|
|
'contains': (c, v, raw) => `${c} ILIKE ${likeTerm(raw[0], '%', '%')}`,
|
|
'not contains': (c, v, raw) => `${c} NOT ILIKE ${likeTerm(raw[0], '%', '%')}`,
|
|
'begins with': (c, v, raw) => `${c} ILIKE ${likeTerm(raw[0], '', '%')}`,
|
|
'not begins with': (c, v, raw) => `${c} NOT ILIKE ${likeTerm(raw[0], '', '%')}`,
|
|
'ends with': (c, v, raw) => `${c} ILIKE ${likeTerm(raw[0], '%', '')}`,
|
|
'not ends with': (c, v, raw) => `${c} NOT ILIKE ${likeTerm(raw[0], '%', '')}`,
|
|
};
|
|
|
|
// the operators that need a term, so an empty one is refused rather than
|
|
// becoming ILIKE '%%' -- which is every non-null row, i.e. no scope at all
|
|
const SCOPE_OPS_NEED_TERM = new Set(Object.keys(SCOPE_OPS).filter(op => !op.startsWith('is ')));
|
|
|
|
function buildScopeClause(scope, dimCols, versionId) {
|
|
if (!Array.isArray(scope) || scope.length === 0) return '';
|
|
const allowed = new Set(dimCols);
|
|
const parts = scope.map((entry) => {
|
|
if (!Array.isArray(entry) || entry.length < 2) {
|
|
const err = new Error(`Malformed scope entry ${JSON.stringify(entry)}`);
|
|
err.status = 400; throw err;
|
|
}
|
|
const [col, op, ...rest] = entry;
|
|
const vals = (Array.isArray(rest[0]) ? rest[0] : rest).filter(v => v !== undefined);
|
|
|
|
if (COMPUTED_SLICE_COLS[col]) {
|
|
if (op !== '==' && op !== 'in') {
|
|
const err = new Error(`${col} can only be scoped with == or in, not ${op}`);
|
|
err.status = 400; throw err;
|
|
}
|
|
return computedSlicePredicate(col, vals, versionId);
|
|
}
|
|
if (!allowed.has(col)) {
|
|
const err = new Error(`Column "${col}" is not available for filtering`);
|
|
err.status = 400; throw err;
|
|
}
|
|
const fn = SCOPE_OPS[op];
|
|
if (!fn) {
|
|
const err = new Error(`Unsupported filter operator "${op}" on ${col}`);
|
|
err.status = 400; throw err;
|
|
}
|
|
if (SCOPE_OPS_NEED_TERM.has(op) && (vals.length === 0 || vals[0] === '' || vals[0] === null)) {
|
|
const err = new Error(`Filter "${col} ${op}" has no value`);
|
|
err.status = 400; throw err;
|
|
}
|
|
return fn(`"${col}"`, vals.map(v => `'${esc(String(v))}'`), vals);
|
|
});
|
|
return parts.join('\nAND ');
|
|
}
|
|
|
|
// The territory predicate: what this account may see and change.
|
|
//
|
|
// Server-side by construction. Today's `scope` is sent by the browser, which is
|
|
// right for a filter the user chose and would be fatal for a permission -- so
|
|
// this one is built from the session and ANDed on last, where nothing in the
|
|
// request can remove it.
|
|
//
|
|
// FALSE, not TRUE, for an account with no territory. The whole point is that a
|
|
// missing grant means no rows: an empty list that fell through to TRUE would
|
|
// hand the entire book to the first account somebody forgot to configure.
|
|
function buildTerritoryClause(territory, territoryCol, alias = '') {
|
|
if (!territory || territory.admin) return '';
|
|
if (!territoryCol) return 'FALSE';
|
|
const vals = (territory.values || []).filter(v => v != null && v !== '');
|
|
if (vals.length === 0) return 'FALSE';
|
|
const pfx = alias ? `${alias}.` : '';
|
|
return `${pfx}"${territoryCol}" IN (${vals.map(v => `'${esc(String(v))}'`).join(', ')})`;
|
|
}
|
|
|
|
// build a WHERE clause spanning several slices — an OR of AND-groups.
|
|
// A union of slices cannot be flattened into one IN list per column: slices
|
|
// {Region:East, State:NY} and {Region:West, State:CA} would become
|
|
// Region IN (East,West) AND State IN (NY,CA), which also matches East/CA.
|
|
function buildWhereAny(slices, dimCols, versionId) {
|
|
const list = (slices || []).filter(s => s && Object.keys(s).length > 0);
|
|
if (list.length === 0) return 'TRUE';
|
|
if (list.length === 1) return buildWhere(list[0], dimCols, versionId);
|
|
|
|
const groups = list
|
|
.map(s => buildWhere(s, dimCols, versionId))
|
|
.filter(w => w !== 'TRUE');
|
|
|
|
// any slice that reduced to TRUE selects everything, so the union does too
|
|
if (groups.length !== list.length) return 'TRUE';
|
|
|
|
// outer parens matter: the caller appends `AND pf_iter NOT IN (...)`,
|
|
// and AND binds tighter than OR
|
|
return `(${groups.map(g => `(${g.replace(/\n/g, ' ')})`).join('\n OR ')})`;
|
|
}
|
|
|
|
// the bare predicate for "this row participates in operations", for use in a
|
|
// FILTER clause where the excluded rows still need to be counted separately
|
|
function buildExcludePredicate(excludeIters) {
|
|
if (!excludeIters || excludeIters.length === 0) return 'TRUE';
|
|
const list = excludeIters.map(i => `'${esc(i)}'`).join(', ');
|
|
return `pf_iter NOT IN (${list})`;
|
|
}
|
|
|
|
// build AND iter NOT IN (...) from a version's exclude_iters array
|
|
function buildExcludeClause(excludeIters) {
|
|
if (!excludeIters || excludeIters.length === 0) return '';
|
|
const list = excludeIters.map(i => `'${esc(i)}'`).join(', ');
|
|
return `AND pf_iter NOT IN (${list})`;
|
|
}
|
|
|
|
// build the dimension columns portion of a SELECT for recode/clone
|
|
// replaces named dimensions with literal values, passes others through unchanged
|
|
// derivedExprs: cname -> a SQL expression to use when the caller has not set the
|
|
// column outright. Clone passes the dim_period expressions through here, so a
|
|
// cloned row's period dimensions come from the calendar against its shifted date
|
|
// rather than from the row it was copied from.
|
|
function buildSetClause(dimCols, setObj, opts = {}) {
|
|
const { derivedExprs, alias } = opts;
|
|
const pfx = alias ? `${alias}.` : '';
|
|
return dimCols.map(col => {
|
|
if (setObj && setObj[col] !== undefined) {
|
|
return `'${esc(setObj[col])}' AS "${col}"`;
|
|
}
|
|
if (derivedExprs && derivedExprs[col]) {
|
|
return `${derivedExprs[col]} AS "${col}"`;
|
|
}
|
|
return `${pfx}"${col}"`;
|
|
}).join(', ');
|
|
}
|
|
|
|
// build a SQL WHERE clause from an array of filter objects { col, op, values }
|
|
// only allows columns with role 'date' or 'filter'
|
|
function buildFilterClause(filters, colMeta) {
|
|
if (!filters || filters.length === 0) {
|
|
const err = new Error('At least one filter is required');
|
|
err.status = 400; throw err;
|
|
}
|
|
const allowed = new Set(
|
|
colMeta.filter(c => c.role !== 'ignore').map(c => c.cname)
|
|
);
|
|
const parts = filters.map(({ col, op, values = [] }) => {
|
|
if (!allowed.has(col)) {
|
|
const err = new Error(`Column "${col}" is not available for baseline filtering`);
|
|
err.status = 400; throw err;
|
|
}
|
|
const c = `"${col}"`;
|
|
const v = values.map(x => `'${esc(String(x))}'`);
|
|
switch (op) {
|
|
case '=': return `${c} = ${v[0]}`;
|
|
case '!=': return `${c} != ${v[0]}`;
|
|
case 'IN': return `${c} IN (${v.join(', ')})`;
|
|
case 'NOT IN': return `${c} NOT IN (${v.join(', ')})`;
|
|
case 'BETWEEN': return `${c} BETWEEN ${v[0]} AND ${v[1]}`;
|
|
case 'IS NULL': return `${c} IS NULL`;
|
|
case 'IS NOT NULL': return `${c} IS NOT NULL`;
|
|
default: {
|
|
const err = new Error(`Unsupported operator "${op}"`);
|
|
err.status = 400; throw err;
|
|
}
|
|
}
|
|
});
|
|
return parts.join('\nAND ');
|
|
}
|
|
|
|
// escape a value for safe SQL string substitution
|
|
function esc(val) {
|
|
if (val === null || val === undefined) return '';
|
|
return String(val).replace(/'/g, "''");
|
|
}
|
|
|
|
module.exports = { generateSQL, grainOf, COMPUTED_SLICE_COLS, buildScopeClause, buildTerritoryClause,
|
|
SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, LABEL_GROUP_COLS,
|
|
ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET, dateGroupsOf, dimPeriodMapOf, dimPeriodJoins, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc };
|