Clone from a named segment, shifted, with period dimensions re-derived
Three changes, one feature: a period with no baseline borrows a shape from prior year or plan. The generator handled one date group. sql_generator.js used find(), so a source with order, requested and ship date groups derived the order one and copied the other two raw -- shifted dates against unshifted period labels. Every group now gets its own join, and they are LEFT rather than inner: an order that has not shipped has no ship date, and an inner join would have dropped the row from the load entirely rather than leaving its period columns empty. dateGroupsOf() is now shared, like grainOf, so the routes and the generator agree on membership and on join aliases. Clone carries every date column rather than only the primary one, since it is the operation that moves rows through time, and re-derives the period dimensions from pf.dim_period against the shifted date instead of copying them from the row being cloned. from_logid names the segment to copy from, replacing the exclude clause for that one entry rather than widening it. Rows written stay pf_iter = 'clone', so scale applies to them afterwards as a second step. set is no longer required: copying a segment forward unchanged is a reasonable thing to ask for. Needs dim_period_col set in Setup (fisc_year on oseas/rseas/sseas_e, fisc_month_abbr on omon/rmon/smon_e) and Generate SQL. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
2689e95b2c
commit
518a0ca5ba
@ -51,6 +51,42 @@ function grainOf(colMeta) {
|
||||
return { cols, key, groupCols };
|
||||
}
|
||||
|
||||
// Date columns that anchor a dim_group, each with the dimensions derived from
|
||||
// pf.dim_period for it. Shared so the generator and the routes agree on both the
|
||||
// membership and the join aliases -- the same reason grainOf is a single function.
|
||||
function dateGroupsOf(colMeta) {
|
||||
return colMeta
|
||||
.filter(c => c.role === 'date' && c.is_key && c.dim_group)
|
||||
.map((keyCol, i) => ({
|
||||
alias: `dp${i + 1}`,
|
||||
dateCol: keyCol.cname,
|
||||
group: keyCol.dim_group,
|
||||
derived: colMeta
|
||||
.filter(c => c.role === 'dimension'
|
||||
&& c.dim_group === keyCol.dim_group
|
||||
&& c.dim_period_col)
|
||||
.map(c => ({ cname: c.cname, periodCol: c.dim_period_col })),
|
||||
}))
|
||||
.filter(g => g.derived.length > 0);
|
||||
}
|
||||
|
||||
// cname -> which join it comes from and which of its columns
|
||||
function dimPeriodMapOf(dateGroups) {
|
||||
return new Map(
|
||||
dateGroups.flatMap(g => g.derived.map(d => [d.cname, { alias: g.alias, periodCol: d.periodCol }]))
|
||||
);
|
||||
}
|
||||
|
||||
// The joins themselves, against a date that has already had {{date_offset}}
|
||||
// applied. LEFT because a null date -- an order not yet shipped has no ship date
|
||||
// -- must leave the period columns empty rather than drop the row.
|
||||
function dimPeriodJoins(dateGroups, alias = 's') {
|
||||
return dateGroups.map(g =>
|
||||
`\n LEFT JOIN pf.dim_period ${g.alias}`
|
||||
+ ` ON ${g.alias}.drange @> (${alias}."${g.dateCol}" + '{{date_offset}}'::interval)::date`
|
||||
).join('');
|
||||
}
|
||||
|
||||
function generateSQL(source, colMeta) {
|
||||
const dims = colMeta
|
||||
.filter(c => c.role === 'dimension')
|
||||
@ -88,21 +124,31 @@ function generateSQL(source, colMeta) {
|
||||
// Baseline and reference copy the source row wholesale, so they carry every
|
||||
// measure and every date — not just the primary one the operations act on.
|
||||
// Dropping the others would leave those columns null for the life of the version.
|
||||
// Clone carries every date column, not just the primary one, because it is the
|
||||
// operation that moves rows through time: {{date_offset}} shifts them all
|
||||
// together, and the period dimensions are re-derived from pf.dim_period against
|
||||
// the shifted dates rather than copied from the row being cloned. Cloning last
|
||||
// year's mix forward a year otherwise produces rows dated 2027 still labelled
|
||||
// with 2026's periods.
|
||||
const cloneCols = [...dims, ...dateCols, effectiveValue, effectiveUnits].filter(Boolean);
|
||||
const cloneInsertCols = [...cloneCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
||||
|
||||
const loadCols = [...dims, ...dateCols, ...valueCols, ...unitsCols];
|
||||
const loadInsertCols = [...loadCols.map(q), 'pf_iter', 'pf_logid', 'pf_user', 'pf_created_at'].join(', ');
|
||||
const dateColSet = new Set(dateCols);
|
||||
|
||||
// dim_period JOIN support: if the date column is the is_key of a dim_group,
|
||||
// dimension siblings with dim_period_col set are derived from pf.dim_period
|
||||
// instead of being copied raw from the source on baseline/reference load.
|
||||
const dateKeyGroup = colMeta.find(c => c.role === 'date' && c.is_key && c.dim_group)?.dim_group;
|
||||
const dimPeriodMap = new Map(
|
||||
dateKeyGroup
|
||||
? colMeta
|
||||
.filter(c => c.role === 'dimension' && c.dim_group === dateKeyGroup && c.dim_period_col)
|
||||
.map(c => [c.cname, c.dim_period_col])
|
||||
: []
|
||||
);
|
||||
// dim_period JOIN support: a date column that is the is_key of a dim_group
|
||||
// anchors that group, and dimension siblings with dim_period_col set are
|
||||
// derived from pf.dim_period instead of copied raw. Derivation is against the
|
||||
// date *after* {{date_offset}}, which is the whole point -- shift a baseline
|
||||
// forward a year and its period columns follow, rather than still naming the
|
||||
// year it came from.
|
||||
//
|
||||
// Every such group, not just the first. This used to be a find(), so a source
|
||||
// with order, requested and ship date groups derived the order one and copied
|
||||
// the other two raw -- shifted dates against unshifted period labels.
|
||||
const dateGroups = dateGroupsOf(colMeta);
|
||||
const dimPeriodMap = dimPeriodMapOf(dateGroups);
|
||||
const hasDimPeriod = dimPeriodMap.size > 0;
|
||||
|
||||
// display grain — when set, initial load and operations both return rows
|
||||
@ -199,15 +245,17 @@ GROUP BY
|
||||
// The offset shifts every date column, so order date and ship date stay in step.
|
||||
return loadCols.map(c => {
|
||||
if (dateColSet.has(c)) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`;
|
||||
if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`;
|
||||
if (dimPeriodMap.has(c)) {
|
||||
const { alias, periodCol } = dimPeriodMap.get(c);
|
||||
return `${alias}.${q(periodCol)} AS ${q(c)}`;
|
||||
}
|
||||
return `${pfx}${q(c)}`;
|
||||
}).join(',\n ');
|
||||
}
|
||||
|
||||
function buildFromClause() {
|
||||
if (!hasDimPeriod) return srcTable;
|
||||
return `${srcTable} s\n JOIN pf.dim_period dp`
|
||||
+ ` ON dp.drange @> (s.${q(dateCol)} + '{{date_offset}}'::interval)::date`;
|
||||
return srcTable + ' s' + dimPeriodJoins(dateGroups);
|
||||
}
|
||||
|
||||
function buildBaseline() {
|
||||
@ -327,6 +375,16 @@ ${opTail('allrows')}` : 'SELECT * FROM neg UNION ALL SELECT * FROM ins'}`.trim()
|
||||
}
|
||||
|
||||
function buildClone() {
|
||||
const select = [
|
||||
// dims: whatever {{set_clause}} resolves them to. The route builds it,
|
||||
// and substitutes the dim_period expression for any derived dimension
|
||||
// the caller has not overridden outright.
|
||||
'{{set_clause}}',
|
||||
...dateCols.map(c => `(s.${q(c)} + '{{date_offset}}'::interval)::date`),
|
||||
effectiveValue ? `round(s.${q(effectiveValue)} * {{scale_factor}}, 2)` : null,
|
||||
effectiveUnits ? `round(s.${q(effectiveUnits)} * {{scale_factor}}, 5)` : null,
|
||||
].filter(Boolean).join(',\n ');
|
||||
|
||||
return `
|
||||
WITH
|
||||
ilog AS (
|
||||
@ -335,13 +393,11 @@ ilog AS (
|
||||
RETURNING id
|
||||
)
|
||||
,ins AS (
|
||||
INSERT INTO {{fc_table}} (${insertCols})
|
||||
INSERT INTO {{fc_table}} (${cloneInsertCols})
|
||||
SELECT
|
||||
{{set_clause}},
|
||||
${q(dateCol)},
|
||||
${effectiveValue ? `round(${q(effectiveValue)} * {{scale_factor}}, 2)` : '0'}${effectiveUnits ? `,\n round(${q(effectiveUnits)} * {{scale_factor}}, 5)` : ''},
|
||||
${select},
|
||||
'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
|
||||
FROM {{fc_table}}
|
||||
FROM {{fc_table}} s${hasDimPeriod ? dimPeriodJoins(dateGroups) : ''}
|
||||
WHERE {{where_clause}}
|
||||
{{exclude_clause}}
|
||||
RETURNING *
|
||||
@ -430,12 +486,21 @@ function buildExcludeClause(excludeIters) {
|
||||
|
||||
// build the dimension columns portion of a SELECT for recode/clone
|
||||
// replaces named dimensions with literal values, passes others through unchanged
|
||||
function buildSetClause(dimCols, setObj) {
|
||||
// derivedExprs: cname -> a SQL expression to use when the caller has not set the
|
||||
// column outright. Clone passes the dim_period expressions through here, so a
|
||||
// cloned row's period dimensions come from the calendar against its shifted date
|
||||
// rather than from the row it was copied from.
|
||||
function buildSetClause(dimCols, setObj, opts = {}) {
|
||||
const { derivedExprs, alias } = opts;
|
||||
const pfx = alias ? `${alias}.` : '';
|
||||
return dimCols.map(col => {
|
||||
if (setObj && setObj[col] !== undefined) {
|
||||
return `'${esc(setObj[col])}' AS "${col}"`;
|
||||
}
|
||||
return `"${col}"`;
|
||||
if (derivedExprs && derivedExprs[col]) {
|
||||
return `${derivedExprs[col]} AS "${col}"`;
|
||||
}
|
||||
return `${pfx}"${col}"`;
|
||||
}).join(', ');
|
||||
}
|
||||
|
||||
@ -479,4 +544,4 @@ function esc(val) {
|
||||
return String(val).replace(/'/g, "''");
|
||||
}
|
||||
|
||||
module.exports = { generateSQL, grainOf, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc };
|
||||
module.exports = { generateSQL, grainOf, dateGroupsOf, dimPeriodMapOf, dimPeriodJoins, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc };
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
const express = require('express');
|
||||
const { tableFromArrays, tableToIPC } = require('apache-arrow');
|
||||
const { applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, esc } = require('../lib/sql_generator');
|
||||
const { applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc } = require('../lib/sql_generator');
|
||||
const { sessionUser } = require('../lib/auth');
|
||||
const { fcTable } = require('../lib/utils');
|
||||
|
||||
@ -721,11 +721,10 @@ module.exports = function(pool) {
|
||||
// clone one or more slices as new business under new dimension values
|
||||
// does not offset the original slice
|
||||
router.post('/versions/:id/clone', async (req, res) => {
|
||||
const { note, set, scale, apply_mode } = req.body;
|
||||
const { note, set, scale, apply_mode, from_logid, date_offset } = req.body;
|
||||
const pf_user = sessionUser(req);
|
||||
const slices = normalizeSlices(req.body);
|
||||
if (slices.length === 0) return res.status(400).json({ error: 'slice is required' });
|
||||
if (!set || Object.keys(set).length === 0) return res.status(400).json({ error: 'set is required' });
|
||||
|
||||
try {
|
||||
const ctx = await getContext(parseInt(req.params.id), 'clone');
|
||||
@ -733,8 +732,38 @@ module.exports = function(pool) {
|
||||
assertSelective(slices, ctx);
|
||||
|
||||
const scaleFactor = (scale != null) ? parseFloat(scale) : 1.0;
|
||||
const excludeClause = buildExcludeClause(ctx.version.exclude_iters);
|
||||
const setClause = buildSetClause(ctx.dimCols, set);
|
||||
const dateOffset = (date_offset || '0 days').trim() || '0 days';
|
||||
|
||||
// Cloning from a named segment is how a period with no baseline gets a
|
||||
// shape: pick the ledger line -- prior year, plan -- and copy its mix
|
||||
// forward. That means reading rows exclude_iters normally keeps
|
||||
// operations away from, so the entry is named explicitly and replaces
|
||||
// the exclusion rather than widening it. The rows written are ordinary
|
||||
// clone rows either way, so they are adjustable afterwards.
|
||||
let excludeClause;
|
||||
if (from_logid != null) {
|
||||
const srcLog = await pool.query(
|
||||
`SELECT id, operation, coalesce(nullif(tag, ''), note) AS label
|
||||
FROM pf.log WHERE id = $1 AND version_id = $2`,
|
||||
[parseInt(from_logid), ctx.version.id]
|
||||
);
|
||||
if (!srcLog.rows.length) {
|
||||
return res.status(400).json({ error: `No log entry ${from_logid} on this version` });
|
||||
}
|
||||
excludeClause = `AND pf_logid = ${parseInt(from_logid)}`;
|
||||
} else {
|
||||
excludeClause = buildExcludeClause(ctx.version.exclude_iters);
|
||||
}
|
||||
|
||||
// Period dimensions come from the calendar against the shifted date, not
|
||||
// from the row being copied -- otherwise a mix moved forward a year
|
||||
// keeps last year's period labels. An explicit set wins over both.
|
||||
const dateGroups = dateGroupsOf(ctx.colMeta);
|
||||
const derivedExprs = Object.fromEntries(
|
||||
[...dimPeriodMapOf(dateGroups)].map(([cname, { alias, periodCol }]) =>
|
||||
[cname, `${alias}."${periodCol}"`])
|
||||
);
|
||||
const setClause = buildSetClause(ctx.dimCols, set, { derivedExprs, alias: 's' });
|
||||
const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate');
|
||||
|
||||
const client = await pool.connect();
|
||||
@ -749,12 +778,17 @@ module.exports = function(pool) {
|
||||
version_id: ctx.version.id,
|
||||
pf_user: esc(pf_user || ''),
|
||||
note: esc(note || ''),
|
||||
params: esc(JSON.stringify({ slices: unit.slices, set, scale: scaleFactor, apply_mode: unit.mode })),
|
||||
params: esc(JSON.stringify({
|
||||
slices: unit.slices, set, scale: scaleFactor, apply_mode: unit.mode,
|
||||
date_offset: dateOffset,
|
||||
...(from_logid != null ? { from_logid: parseInt(from_logid) } : {}),
|
||||
})),
|
||||
slice: esc(JSON.stringify(loggedSlice)),
|
||||
where_clause: unit.where,
|
||||
exclude_clause: excludeClause,
|
||||
set_clause: setClause,
|
||||
scale_factor: scaleFactor
|
||||
scale_factor: scaleFactor,
|
||||
date_offset: esc(dateOffset)
|
||||
});
|
||||
const result = await runSQL(sql, client);
|
||||
await tagLog(client, result.rows, req.body.tag);
|
||||
|
||||
@ -715,6 +715,7 @@ export default function OperationPanel({
|
||||
recodeNote, setRecodeNote,
|
||||
cloneSet, setCloneSet,
|
||||
cloneScale, setCloneScale,
|
||||
cloneFrom, setCloneFrom, cloneOffset, setCloneOffset, cloneSources,
|
||||
cloneNote, setCloneNote,
|
||||
dimCols, lookupDerivedCols, dimMembers, versionId,
|
||||
buildPayload, submitOp,
|
||||
@ -785,13 +786,53 @@ export default function OperationPanel({
|
||||
slices={slices} lookupDerivedCols={lookupDerivedCols} dimMembers={dimMembers} versionId={versionId}
|
||||
extra={
|
||||
<div className="flex flex-col gap-2">
|
||||
{/* Where the mix comes from. Left on the adjustable rows,
|
||||
clone behaves as it always did. Naming a segment reads
|
||||
from that entry instead -- including the reference ones
|
||||
operations are normally kept away from, which is the
|
||||
point: a period with no baseline borrows a shape from
|
||||
prior year or plan. */}
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-gray-500 whitespace-nowrap">copy from</span>
|
||||
<select value={cloneFrom} onChange={e => setCloneFrom(e.target.value)}
|
||||
className={`${INPUT} w-auto`}>
|
||||
<option value="">current selection</option>
|
||||
{(cloneSources || []).map(s => (
|
||||
<option key={s.id} value={s.id}>
|
||||
{s.label || `${s.operation} #${s.id}`}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{/* Only meaningful when borrowing across time, so it
|
||||
appears with the segment rather than always. */}
|
||||
{cloneFrom && (
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-gray-500 whitespace-nowrap">shift dates by</span>
|
||||
<input value={cloneOffset} list="pf-clone-offsets"
|
||||
onChange={e => setCloneOffset(e.target.value)}
|
||||
placeholder="0 days" className={`${INPUT} w-28`} />
|
||||
<datalist id="pf-clone-offsets">
|
||||
<option value="12 months" />
|
||||
<option value="24 months" />
|
||||
<option value="0 days" />
|
||||
</datalist>
|
||||
<span className="text-gray-400 text-[11px]">
|
||||
every date column, with period dimensions re-derived
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-gray-500">scale cloned rows by</span>
|
||||
<input type="number" step="any" value={cloneScale}
|
||||
onChange={e => setCloneScale(e.target.value)} className={INPUT} />
|
||||
</div>
|
||||
{!cloneFrom && (
|
||||
<MovingTotal currentTotals={currentTotals} verb="Copying"
|
||||
factor={parseFloat(cloneScale) || 1} />
|
||||
)}
|
||||
</div>
|
||||
} />
|
||||
)}
|
||||
|
||||
@ -68,6 +68,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
|
||||
const [expandedLog, setExpandedLog] = useState(null)
|
||||
// master data per dim_group, fetched once per source: { part: { key_col, siblings, members } }
|
||||
const [dimMembers, setDimMembers] = useState({})
|
||||
// clone can borrow its mix from a named segment instead of the current selection
|
||||
const [cloneFrom, setCloneFrom] = useState('')
|
||||
const [cloneOffset, setCloneOffset] = useState('12 months')
|
||||
// what a target/percentage is measured against: the rows this operation can
|
||||
// write, or everything the pivot shows for the slice (excluded rows included)
|
||||
const [targetBasis, setTargetBasis] = useState('selected')
|
||||
@ -1123,6 +1126,17 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
|
||||
})
|
||||
}
|
||||
|
||||
// Segments worth copying a mix from: the loads. Adjustments are already in the
|
||||
// forecast, so cloning one would double it rather than seed anything.
|
||||
const cloneSources = Object.entries(logMeta)
|
||||
.filter(([, m]) => ['baseline', 'reference'].includes(m.operation))
|
||||
.map(([id, m]) => ({
|
||||
id: Number(id),
|
||||
operation: m.operation,
|
||||
label: (m.tag || m.note || '').trim(),
|
||||
}))
|
||||
.sort((a, b) => a.id - b.id)
|
||||
|
||||
function buildEffectiveSlice(raw) {
|
||||
const dimCols = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname))
|
||||
const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname))
|
||||
@ -1206,6 +1220,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
|
||||
} else if (op === 'clone') {
|
||||
const set = Object.fromEntries(Object.entries(cloneSet).filter(([, v]) => v.trim()))
|
||||
body = { ...body, note: cloneNote || undefined, set, scale: parseFloat(cloneScale) || 1 }
|
||||
if (cloneFrom) {
|
||||
body.from_logid = Number(cloneFrom)
|
||||
body.date_offset = cloneOffset.trim() || '0 days'
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
@ -1312,6 +1330,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
|
||||
recodeNote, setRecodeNote,
|
||||
cloneSet, setCloneSet,
|
||||
cloneScale, setCloneScale,
|
||||
cloneFrom, setCloneFrom, cloneOffset, setCloneOffset, cloneSources,
|
||||
cloneNote, setCloneNote,
|
||||
dimCols, lookupDerivedCols, dimMembers, versionId,
|
||||
buildPayload, submitOp,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user