pt/pf_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge ledger and slice fidelity work

Browse Source 
Slices now mean what they say -- pf_segment and pf_bucket resolve to log ids
rather than being dropped, the measure name no longer leaks in from a
collapsed column, and the pivot's own filter scopes both the ledger and the
write. The ledger names its lines from label like everything else does, shows
what cannot move first and per segment, and the column groups are ruled off
in the grid.
This commit is contained in:
Paul Trowbridge 2026-09-18 09:53:37 -04:00
commit afab81d770
5 changed files with 428 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
lib/sql_generator.js
View File

@ -512,13 +512,53 @@ function applyTokens(sql, tokens) {
// build a SQL WHERE clause string from a slice object
// only dimension columns are included; unrecognised keys are silently skipped
function buildWhere(slice, dimCols) {
// pf_segment and pf_bucket are not columns on the forecast table -- they are
// computed at read time from the row's pf.log entry -- so a slice naming one
// cannot be compared directly. It resolves to a set of log ids instead, which is
// exact: the name lives on the log row, and every forecast row carries the
// pf_logid that points at it.
//
// Without this they were dropped from the slice, and clicking a single bucket's
// cell scaled every bucket at that dimension intersection while the panel showed
// only the one clicked.
const COMPUTED_SLICE_COLS = { pf_segment: SEGMENT_EXPR, pf_bucket: BUCKET_EXPR };
function computedSlicePredicate(col, val, versionId) {
if (versionId == null) {
const err = new Error(`Cannot filter on ${col} without a version`);
err.status = 500;
throw err;
}
const vals = (Array.isArray(val) ? val : [val]).map(v => `'${esc(v)}'`).join(', ');
return `pf_logid IN (
SELECT l.id
FROM pf.log l${VERSION_JOIN}
WHERE l.version_id = ${parseInt(versionId)}
AND ${COMPUTED_SLICE_COLS[col]} IN (${vals})
)`;
}
function buildWhere(slice, dimCols, versionId) {
if (!slice || Object.keys(slice).length === 0) return 'TRUE';
const allowed = new Set(dimCols);
const parts = [];
for (const [col, val] of Object.entries(slice)) {
if (COMPUTED_SLICE_COLS[col]) {
parts.push(computedSlicePredicate(col, val, versionId));
continue;
}
// A pf_ key this does not understand is refused rather than skipped.
// Skipping is how a selection silently widened: the operation ran against
// everything the dropped key would have excluded. pf_iter is the one
// exception -- the client strips it deliberately, since two cells that
// differ only by iter band are the same slice.
if (col.startsWith('pf_') && col !== 'pf_iter') {
const err = new Error(`Slice names ${col}, which cannot be filtered on`);
err.status = 400;
throw err;
}
if (!allowed.has(col)) continue;
if (Array.isArray(val)) {
const escaped = val.map(v => esc(v));
@ -531,17 +571,73 @@ function buildWhere(slice, dimCols) {
return parts.length ? parts.join('\nAND ') : 'TRUE';
}
// The pivot's own filter, carried alongside the slices so an operation writes
// exactly the rows the ledger counted.
//
// A slice is {col: value} and can only ever mean equality, so a view filtered to
// sseas_e <= 2027 could not be expressed as one. Refusing it was safe but
// useless -- a bounded season is an ordinary way to scope a forecast -- so the
// operators travel as [col, op, value] triples instead.
//
// Perspective's operator names, not SQL's, since that is where these come from.
// Anything outside this list is refused rather than ignored: a scope silently
// dropped is a write that is wider than the panel that authorised it.
const SCOPE_OPS = {
'==': (c, v) => `${c} = ${v[0]}`,
'!=': (c, v) => `${c} != ${v[0]}`,
'>': (c, v) => `${c} > ${v[0]}`,
'>=': (c, v) => `${c} >= ${v[0]}`,
'<': (c, v) => `${c} < ${v[0]}`,
'<=': (c, v) => `${c} <= ${v[0]}`,
'in': (c, v) => `${c} IN (${v.join(', ')})`,
'not in': (c, v) => `${c} NOT IN (${v.join(', ')})`,
'is null': (c) => `${c} IS NULL`,
'is not null': (c) => `${c} IS NOT NULL`,
};
function buildScopeClause(scope, dimCols, versionId) {
if (!Array.isArray(scope) || scope.length === 0) return '';
const allowed = new Set(dimCols);
const parts = scope.map((entry) => {
if (!Array.isArray(entry) || entry.length < 2) {
const err = new Error(`Malformed scope entry ${JSON.stringify(entry)}`);
err.status = 400; throw err;
}
const [col, op, ...rest] = entry;
const vals = (Array.isArray(rest[0]) ? rest[0] : rest).filter(v => v !== undefined);
if (COMPUTED_SLICE_COLS[col]) {
if (op !== '==' && op !== 'in') {
const err = new Error(`${col} can only be scoped with == or in, not ${op}`);
err.status = 400; throw err;
}
return computedSlicePredicate(col, vals, versionId);
}
if (!allowed.has(col)) {
const err = new Error(`Column "${col}" is not available for filtering`);
err.status = 400; throw err;
}
const fn = SCOPE_OPS[op];
if (!fn) {
const err = new Error(`Unsupported filter operator "${op}" on ${col}`);
err.status = 400; throw err;
}
return fn(`"${col}"`, vals.map(v => `'${esc(String(v))}'`));
});
return parts.join('\nAND ');
}
// build a WHERE clause spanning several slices — an OR of AND-groups.
// A union of slices cannot be flattened into one IN list per column: slices
// {Region:East, State:NY} and {Region:West, State:CA} would become
// Region IN (East,West) AND State IN (NY,CA), which also matches East/CA.
function buildWhereAny(slices, dimCols) {
function buildWhereAny(slices, dimCols, versionId) {
const list = (slices || []).filter(s => s && Object.keys(s).length > 0);
if (list.length === 0) return 'TRUE';
if (list.length === 1) return buildWhere(list[0], dimCols);
if (list.length === 1) return buildWhere(list[0], dimCols, versionId);
const groups = list
.map(s => buildWhere(s, dimCols))
.map(s => buildWhere(s, dimCols, versionId))
.filter(w => w !== 'TRUE');
// any slice that reduced to TRUE selects everything, so the union does too
@ -627,6 +723,6 @@ function esc(val) {
return String(val).replace(/'/g, "''");
}
module.exports = { generateSQL, grainOf,
module.exports = { generateSQL, grainOf, COMPUTED_SLICE_COLS, buildScopeClause,
SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, LABEL_GROUP_COLS, VERSION_JOIN,
ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET, UNLABELED_LOAD, dateGroupsOf, dimPeriodMapOf, dimPeriodJoins, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc };

24
routes/operations.js
View File

@ -1,6 +1,6 @@
const express = require('express');
const { tableFromArrays, tableToIPC } = require('apache-arrow');
const { applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc,
const { applyTokens, buildWhere, buildWhereAny, COMPUTED_SLICE_COLS, buildScopeClause, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc,
SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, VERSION_JOIN,
ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET } = require('../lib/sql_generator');
const { sessionUser } = require('../lib/auth');
@ -44,10 +44,16 @@ module.exports = function(pool) {
// Only scale has a target to prorate towards; recode and clone rewrite rows
// rather than distribute an amount, so for them this only decides whether the
// work lands as one log entry or several.
function sliceUnits(slices, ctx, applyMode) {
// The scope is ANDed onto every unit rather than folded into the slices: it
// applies to all of them equally, and under apply_mode 'each' folding it in
// would repeat the same predicate in every statement for no gain.
function sliceUnits(slices, ctx, applyMode, scope) {
const vid = ctx.version.id;
const scl = buildScopeClause(scope, ctx.filterCols, vid);
const and = (w) => (scl ? (w === 'TRUE' ? scl : `${w}\nAND ${scl}`) : w);
return applyMode === 'each'
? slices.map(sl => ({ slices: [sl], where: buildWhere(sl, ctx.filterCols) }))
: [{ slices, where: buildWhereAny(slices, ctx.filterCols) }];
? slices.map(sl => ({ slices: [sl], where: and(buildWhere(sl, ctx.filterCols, vid)) }))
: [{ slices, where: and(buildWhereAny(slices, ctx.filterCols, vid)) }];
}
// The offset is interpolated into the statement as an interval literal, so a
@ -73,7 +79,7 @@ module.exports = function(pool) {
// otherwise reduce to TRUE and apply the operation to the whole version.
// Refuse rather than let a malformed selection rewrite every row.
function assertSelective(slices, ctx) {
const allowed = new Set(ctx.filterCols);
const allowed = new Set([...ctx.filterCols, ...Object.keys(COMPUTED_SLICE_COLS)]);
slices.forEach((sl, i) => {
const hits = Object.keys(sl).filter(k => allowed.has(k));
if (hits.length === 0) {
@ -90,7 +96,7 @@ module.exports = function(pool) {
// echo back what the caller asked for, for the audit log
function pickIntent(body) {
const keys = ['mode', 'target_basis', 'value_incr', 'units_incr', 'value_pct', 'units_pct', 'pct',
'target_value', 'target_units', 'target_price'];
'target_value', 'target_units', 'target_price', 'scope'];
const out = {};
for (const k of keys) if (body[k] !== undefined && body[k] !== null && body[k] !== '') out[k] = body[k];
return out;
@ -623,7 +629,7 @@ module.exports = function(pool) {
// sum() OVER () distribute the increment across the whole pool.
// 'each' runs the same statement once per slice, so every slice
// reaches the target on its own and gets its own log entry.
const units = sliceUnits(slices, ctx, applyMode);
const units = sliceUnits(slices, ctx, applyMode, req.body.scope);
const client = await pool.connect();
let committed = false;
@ -705,7 +711,7 @@ module.exports = function(pool) {
const excludeClause = buildExcludeClause(ctx.version.exclude_iters);
const setClause = buildSetClause(ctx.dimCols, set);
const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate');
const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate', req.body.scope);
const client = await pool.connect();
let committed = false;
@ -794,7 +800,7 @@ module.exports = function(pool) {
[cname, `${alias}."${periodCol}"`])
);
const setClause = buildSetClause(ctx.dimCols, set, { derivedExprs, alias: 's' });
const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate');
const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate', req.body.scope);
const client = await pool.connect();
let committed = false;

4
ui/src/components/BridgeView.jsx
View File

@ -106,7 +106,9 @@ export function buildSteps(rows, {
if (isLoad) { loads.value += v; loads.units += u; loads.rows += 1; continue }
const meta = logMeta[r.pf_logid] || {}
const tag = (meta.tag || '').trim()
// label first, the same precedence pf_segment uses, so the bridge and the
// pivot call a step by the same name
const tag = (meta.label || meta.tag || '').trim()
const label = tag || (meta.note || '').trim() ||
`${(meta.operation || r.pf_iter || 'adj')}${r.pf_logid != null ? ` #${r.pf_logid}` : ''}`
const key = tag ? `tag:${tag}` : `log:${r.pf_logid}`

102
ui/src/components/OperationPanel.jsx
View File

@ -110,11 +110,22 @@ function Submit({ onClick, children, disabled }) {
}
// ── 1. Selection ────────────────────────────────────────────────────────────
function SelectionList({ slices, currentTotals, onRemove, onClear }) {
function SelectionList({ slices, viewScope = [], currentTotals, onRemove, onClear }) {
const multi = slices.length > 1
const perSlice = currentTotals?.perSlice || []
const valueCol = currentTotals?.valueCol
// The pivot's own filter. Shown because it scopes every figure below and
// every row the operation writes, while appearing in none of the slices --
// perspective-click reports only the cell's own dimensions, so without this
// the panel prints a selection wider than the one it is acting on.
const scopeLine = viewScope
.map(([col, op, ...rest]) => {
const vals = (Array.isArray(rest[0]) ? rest[0] : rest).filter(v => v !== undefined)
return `${col} ${op}${vals.length ? ' ' + vals.join(', ') : ''}`
})
.join(' · ')
if (!slices.length) {
return (
<p className="text-gray-600 italic leading-relaxed">
@ -126,6 +137,12 @@ function SelectionList({ slices, currentTotals, onRemove, onClear }) {
return (
<div className="min-w-0">
{scopeLine && (
<div className="mb-1 flex items-baseline gap-1.5 text-[11px]">
<span className="text-gray-400 uppercase tracking-wide shrink-0">within</span>
<span className="font-mono text-gray-600 truncate" title={scopeLine}>{scopeLine}</span>
</div>
)}
<div className="overflow-auto max-h-36 -mx-1 px-1">
<table className="w-full">
<tbody>
@ -270,9 +287,16 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
const loose = []
const byTag = new Map()
for (const e of entries) {
if (e.key === 'baseline') { baseline.push({ ...e, label: 'Baseline', kind: 'baseline' }); continue }
const meta = logMeta[e.logid] || {}
const tag = (meta.tag || '').trim()
// Named like every other line: the label the pivot shows, then the older
// fallbacks. "Baseline" was hardcoded, so a segment called 03 - New Orders
// everywhere else read as "Baseline" here alone.
if (e.key === 'baseline') {
const name = (meta.label || meta.tag || meta.note || '').trim()
baseline.push({ ...e, label: name || 'Baseline', kind: 'baseline' })
continue
}
const tag = (meta.label || meta.tag || '').trim()
if (tag) {
const g = byTag.get(tag) ||
{ key: `tag:${tag}`, label: tag, kind: 'tag', value: 0, units: 0, count: 0, first: e.logid }
@ -285,7 +309,8 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
const op = meta.operation || e.iter || 'adjustment'
loose.push({
...e, kind: 'entry', count: 1,
label: (meta.note || '').trim() || `${op.charAt(0).toUpperCase()}${op.slice(1)} #${e.logid}`,
label: (meta.label || meta.note || '').trim()
|| `${op.charAt(0).toUpperCase()}${op.slice(1)} #${e.logid}`,
})
}
}
@ -299,7 +324,22 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
const hasExcl = excl.rows > 0 && (excl.value !== 0 || excl.units !== 0)
const onTotal = hasExcl && targetBasis !== 'adjustable' // 'selected total' is the default
const grand = { value: total.value + excl.value, units: total.units + excl.units }
const exclName = (currentTotals?.excludedIters || []).join(' / ') || 'excluded'
// Named by the segments themselves where we have them -- "02 - Prior Year"
// reads as a thing a forecaster recognises, where "reference" names only the
// iter band that happens to exclude it.
const exclName = (currentTotals?.excluded?.names || []).join(' · ')
|| (currentTotals?.excludedIters || []).join(' / ')
|| 'excluded'
// Everything in the selection is immovable. Worth saying outright: the panel
// otherwise prints a row of zeros and leaves the reason to be worked out.
const nothingToAdjust = hasExcl && !total.value && !total.units
// One line per immovable segment. Falls back to the combined figure for a
// selection whose rows carry no segment name.
const exclLines = (currentTotals?.excluded?.bySegment?.length
? currentTotals.excluded.bySegment
: (hasExcl ? [{ name: exclName, ...excl }] : []))
// the basis decides which line the editable rows are measured from
const basisOf = (key) => {
@ -413,6 +453,29 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
</tr>
</thead>
<tbody>
{/* What cannot move comes first: it is the constraint the rest is
worked out against. Then the walk, which sums to Adjustable, and
the two together make the selected total. */}
{exclLines.map(seg => (
<tr key={seg.name} className="text-amber-700">
<td className="pr-3 whitespace-nowrap max-w-[16rem] truncate" title={seg.name}>
{seg.name}
<span className="ml-1.5 px-1 py-0.5 rounded bg-amber-50 text-amber-700 text-[10px] uppercase tracking-wide">
final
</span>
</td>
{measures.map(m => (
<td key={m.key} className={`${numCell} text-amber-700`}>
{m.key === 'price' ? fmtNum(priceOf(seg), m.dp) : fmtNum(seg[m.key], m.dp)}
</td>
))}
</tr>
))}
{exclLines.length > 0 && (
<tr>{rule}{measures.map(m => <td key={m.key} className="p-0 px-2"><div className="border-t border-gray-300 my-1" /></td>)}</tr>
)}
{/* the walk from baseline to current, by initiative */}
{lines.map(e => (
<tr key={e.key} className="text-gray-600">
@ -440,20 +503,6 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
))}
</tr>
{/* Rows the pivot shows but operations cannot write. Listed so the
panel's figures reconcile with what the grid displays. */}
{hasExcl && (
<tr className="text-gray-600">
<td className="pr-3 whitespace-nowrap">
{exclName} <span className="text-gray-500">· fixed</span>
</td>
{measures.map(m => (
<td key={m.key} className={numCell}>
{m.key === 'price' ? fmtNum(priceOf(excl), m.dp) : fmtNum(excl[m.key], m.dp)}
</td>
))}
</tr>
)}
{hasExcl && (
<tr className={onTotal ? 'font-semibold text-gray-700' : 'text-gray-600'}>
@ -468,6 +517,19 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
</tr>
)}
{/* Zeros in the Adjustable row are a true answer to the wrong question:
they say how much can move, not why none of it can. Spell it out
where the eye already is, rather than leaving the edit rows to fail
silently below. */}
{nothingToAdjust && (
<tr>
<td colSpan={measures.length + 1} className="pt-2 text-amber-700 leading-snug">
Nothing in this selection can be adjusted — all of it is {exclName},
loaded as {(currentTotals?.excludedIters || []).join(' / ') || 'reference'}.
</td>
</tr>
)}
<tr>{rule}{measures.map(m => <td key={m.key} className="p-0 px-2"><div className="border-t border-gray-300 my-1" /></td>)}</tr>
{/* the edit — three equivalent ways to say the same thing */}
@ -738,6 +800,7 @@ function RequestPreview({ payload }) {
export default function OperationPanel({
dock,
slices, setSlices, distinctSlices,
viewScope = [],
applyMode, setApplyMode,
currentTotals,
activeOp, setActiveOp,
@ -780,6 +843,7 @@ export default function OperationPanel({
)}
<SelectionList
slices={slices}
viewScope={viewScope}
currentTotals={currentTotals}
onRemove={(i) => setSlices(prev => prev.filter((_, x) => x !== i))}
onClear={() => setSlices([])}

242
ui/src/views/Forecast.jsx
View File

@ -12,6 +12,11 @@ import '@perspective-dev/viewer/inline'
import '@perspective-dev/viewer-datagrid'
import '@perspective-dev/viewer/themes'
// Slice keys that are not col_meta columns: computed from pf.log when the rows are
// served, so they are real columns in the loaded table but have to be resolved back
// to log ids server-side. Mirrors COMPUTED_SLICE_COLS in lib/sql_generator.js.
const COMPUTED_SLICE_COLS = new Set(['pf_segment', 'pf_bucket'])
const LAYOUT_KEY = (vid) => `pf_layout_v${vid}` // last-used layout (auto restore)
const LAYOUTS_KEY = (vid) => `pf_layouts_v${vid}` // named layout list
@ -51,7 +56,25 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
const { dark } = useTheme()
const [loading, setLoading] = useState(false)
const [largeDataset, setLargeDataset] = useState(false)
// The pivot's own filter, refreshed whenever the ledger recomputes. A ref
// rather than state: it is read at dispatch and at totals time, never
// rendered from directly, and making it state would re-run the totals effect
// that sets it.
const viewFilterRef = useRef([])
const [loadProgress, setLoadProgress] = useState(null) // { received, total }
// Rows the load is waiting on, so the wait can say what it is waiting for --
// on this data the row count is the wait (see CLAUDE.md, "Load time is
// dominated by row count").
//
// Filled twice. X-Row-Count is exact but arrives with the response headers,
// and in grain mode the server aggregates before sending any: the number
// turned up just as the wait ended. So the forecast table's own count goes in
// first, from the same table-info the status bar reads, and the exact figure
// replaces it when the headers land.
const [loadRows, setLoadRows] = useState(null)
// the same filter, for display: the panel has to show the scope it is acting
// inside or the slice it prints is not the slice that gets written
const [viewScope, setViewScope] = useState([])
const [msg, setMsg] = useState(null)
// layouts
@ -318,13 +341,23 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
for (let c = c0; c < c1; c++) {
const key = userKeys[c]
if (!key) continue
// A column name is dimension values joined by | with the measure last, and
// only as many values as the axis currently shows -- collapse the column
// hierarchy and the deeper levels are simply absent. Mapping split_by
// positionally over every segment therefore read the measure as a value
// for the first collapsed dimension: a bucket subtotal came back as
// smon_e = 'sales_usd', which matches no row, so the operation silently
// had nothing to act on. Drop the measure, then map over what is left.
const segs = key.split('|').slice(0, -1)
const colFilters = splitBy
.slice(0, segs.length)
.map((col, ix) => {
const v = key.split('|')[ix]
const v = segs[ix]
return (v && !META_COL_RE.test(v)) ? [col, '==', v] : null
})
.filter(Boolean)
const slice = sliceFromFilters([...base, ...rowFilters, ...colFilters])
const slice = sliceFromFilters([...base, ...rowFilters, ...colFilters],
(cfg.columns || []).filter(Boolean))
if (!Object.keys(slice).length) continue
const y = win.start_row + i
out.push({ slice, area: { x0: c, x1: c, y0: y, y1: y } })
@ -404,14 +437,54 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
const dateNames = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname))
const ITER_ORDER = ['baseline', 'scale', 'recode', 'clone']
// A slice carries every value as a string -- it is built from filters the
// grid reports and from a payload that has to survive JSON. Perspective
// matches on type, so a string '2027' against an integer column does not
// filter to nothing, it is dropped: the ledger then totalled rows the pivot
// was hiding, which is how a season filter on sseas_e went unnoticed while
// the numbers disagreed by exactly the out-of-season rows.
// The pivot's own filter is not part of a clicked slice -- perspective-click
// reports only the cell's own dimensions -- so the ledger has to read it off
// the viewer and apply it alongside. Without this the ledger totals rows the
// grid is hiding, and the operation writes them: a grid scoped to
// sseas_e = 2027 gave a cell of 921,225.71 against a ledger of 956,485.13.
//
// Taken from viewer.save(), so the values are already in the table's own
// types and the operators are whatever the user set -- ranges and in-lists
// included, which a slice cannot express.
const viewFilter = await (async () => {
try {
const cfg = await viewerRef.current?.save()
return (cfg?.filter || []).filter(f => Array.isArray(f) && f.length >= 2)
} catch { return [] }
})()
viewFilterRef.current = viewFilter
setViewScope(viewFilter)
const schema = await tableRef.current.schema()
const typed = (col, val) => {
switch (schema[col]) {
case 'integer': case 'float': return Number(val)
case 'boolean': return val === true || val === 'true'
case 'date': case 'datetime': return Number(val)
default: return String(val)
}
}
async function totalsFor(sliceObj) {
// pf_segment and pf_bucket are computed server-side but are ordinary
// columns in the loaded table, so here they filter directly. They have to
// be applied, or the ledger totals a wider selection than the operation
// will write.
const filters = [
...viewFilter,
...Object.entries(sliceObj)
.filter(([col]) => dimNames.has(col))
.map(([col, val]) => [col, '==', val]),
...Object.entries(sliceObj)
.filter(([col]) => dateNames.has(col))
.map(([col, val]) => [col, '==', Number(val)]),
.filter(([col]) => COMPUTED_SLICE_COLS.has(col) || dimNames.has(col) || dateNames.has(col)
|| schema[col] !== undefined)
// a cell inside the filtered view cannot contradict it, so a repeated
// column is the same predicate twice and harmless
.filter(([col]) => !viewFilter.some(f => f[0] === col))
.map(([col, val]) => [col, '==', typed(col, val)]),
]
const view = await tableRef.current.view({ filter: filters })
const rows = await view.to_json()
@ -424,7 +497,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
// rows the pivot shows but operations cannot write (usually 'reference').
// Kept separate rather than filtered away: the grid total includes them,
// so the panel has to account for them or the two disagree.
const excluded = { value: 0, units: 0, rows: 0 }
// Per segment, not one lump: YTD Sales and Open Orders are different
// things, and a single "final" line hides which part of the number is
// which.
const excluded = { value: 0, units: 0, rows: 0, names: new Set(), bySegment: new Map() }
for (const r of rows) {
const k = r.pf_iter || '?'
const val = valueCol ? (parseFloat(r[valueCol]) || 0) : 0
@ -434,6 +510,16 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
excluded.value += val
excluded.units += uni
excluded.rows += 1
// Name them by what they are, not by the iter band that happens to
// exclude them: "02 - Prior Year" means something to a forecaster,
// "reference" is the mechanism.
const name = String(r.pf_segment || 'excluded')
excluded.names.add(name)
const seg = excluded.bySegment.get(name) || { name, value: 0, units: 0, rows: 0 }
seg.value += val
seg.units += uni
seg.rows += 1
excluded.bySegment.set(name, seg)
continue
}
@ -505,10 +591,25 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
value: acc.value + (ps.excluded?.value || 0),
units: acc.units + (ps.excluded?.units || 0),
rows: acc.rows + (ps.excluded?.rows || 0),
}), { value: 0, units: 0, rows: 0 })
names: new Set([...acc.names, ...(ps.excluded?.names || [])]),
bySegment: (() => {
const m = acc.bySegment
for (const seg of (ps.excluded?.bySegment?.values?.() || [])) {
const t = m.get(seg.name) || { name: seg.name, value: 0, units: 0, rows: 0 }
t.value += seg.value; t.units += seg.units; t.rows += seg.rows
m.set(seg.name, t)
}
return m
})(),
}), { value: 0, units: 0, rows: 0, names: new Set(), bySegment: new Map() })
setCurrentTotals({
byIter, byEntry, total, excluded, valueCol, unitsCol, perSlice,
byIter, byEntry, total, valueCol, unitsCol, perSlice,
excluded: {
...excluded,
names: [...excluded.names].sort(),
bySegment: [...excluded.bySegment.values()].sort((a, b) => a.name.localeCompare(b.name)),
},
excludedIters: [...excludeIters],
})
} catch {
@ -524,8 +625,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
const entries = await fetch(`/api/versions/${vid}/log`).then(r => r.json())
const map = {}
for (const e of entries) map[e.id] = {
label: e.label || null,
tag: e.tag || null, note: e.note || null, operation: e.operation,
bucket: e.bucket || null, seq: e.seq ?? null,
bucket: e.bucket || null,
}
setLogMeta(map)
} catch { setLogMeta({}) }
@ -582,6 +684,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
if (!r.ok) { const { error } = await r.json(); throw new Error(error || 'Failed to load data') }
const rowCount = parseInt(r.headers.get('X-Row-Count') || '0')
const total = parseInt(r.headers.get('Content-Length') || '0') || null
if (rowCount) setLoadRows(rowCount)
const reader = r.body.getReader()
const chunks = []
let received = 0
@ -618,6 +721,13 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
setLoading(true)
setLargeDataset(false)
setLoadProgress(null)
setLoadRows(null)
// deliberately not awaited: it is a count over the whole forecast table and
// the load must not wait on it
fetch(`/api/versions/${vid}/table-info`)
.then(r => r.ok ? r.json() : null)
.then(info => { if (info?.rows && initIdRef.current === myId) setLoadRows(n => n ?? info.rows) })
.catch(() => {})
setSlices([])
setExpandDepth(null)
adoptSplit([], 0)
@ -752,7 +862,8 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
if (!detail.row) return
const config = await viewer.save()
if (!(config.group_by || []).length) return
const s = sliceFromFilters((detail.config || {}).filter || [])
const s = sliceFromFilters((detail.config || {}).filter || [],
(config.columns || []).filter(Boolean))
if (!Object.keys(s).length) return
// the CustomEvent carries no modifier flags, so read them off the
// mousedown that produced it (captured on window below)
@ -778,6 +889,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
viewer.addEventListener('perspective-select', viewer._pspSelect)
gridRef.current = await viewer.getPlugin()
applyGroupRules()
setLargeDataset(false)
} catch (err) {
@ -841,6 +953,77 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
}
}
// Rule off the column groups, and mark each group's subtotal.
//
// Scanning across "prior · plan · forecast, each with twelve months and a
// total" is twelve columns of identical-looking numbers with nothing to say
// where one domain ends and the next begins. The annual figures and the
// monthly ones read as one run.
//
// Done through regular_table's style listener rather than CSS: which column
// starts a group, and which one is a group's subtotal, are facts about the
// data that only the cell metadata knows. The listener runs on every draw, so
// it survives scrolling and virtualisation -- a stylesheet cannot, since the
// DOM cells are recycled across columns as you scroll.
// Blank for this purpose means "no value at this level", which Perspective
// writes as a zero-width space rather than an empty string.
const notBlank = (v) =>
v != null && String(v).replace(/[\s\u200b-\u200d\ufeff]/g, '') !== ''
function applyGroupRules() {
const grid = gridRef.current
const table = grid?.regular_table
if (!table || table._pfGroupRules) return
table._pfGroupRules = true
// The grid lives in a shadow root, so a stylesheet on the page cannot reach
// these cells. Inject into whichever root actually contains the table.
// currentColor rather than a fixed grey, so the rule follows the theme
// instead of vanishing against Pro Dark.
const root = table.getRootNode() || document
if (!root.querySelector('#pf-group-rules')) {
const style = document.createElement('style')
style.id = 'pf-group-rules'
style.textContent = `
td.pf-group-start { border-left: 2px solid currentColor; opacity: 1; }
td.pf-subtotal { font-weight: 600; background: color-mix(in srgb, currentColor 7%, transparent); }
`
;(root.head || root).appendChild(style)
}
table.addStyleListener(() => {
const body = table.querySelectorAll('tbody td')
// The deepest column path is a leaf; anything shorter is an aggregate of
// the levels below it, which is what makes a subtotal a subtotal.
//
// The empty levels are not empty strings. Perspective pads a subtotal's
// path with zero-width spaces -- ['04 - Forecast', '\u200b', 'sales_usd']
// -- so every path is the same length and a naive `!== ''` test finds no
// subtotals at all.
let depth = 0
const metas = []
for (const td of body) {
let meta
try { meta = table.getMeta(td) } catch { meta = null }
metas.push([td, meta])
const path = meta?.column_header
if (Array.isArray(path)) depth = Math.max(depth, path.filter(notBlank).length)
}
let prevGroup = null
for (const [td, meta] of metas) {
td.classList.remove('pf-group-start', 'pf-subtotal')
const path = meta?.column_header
if (!Array.isArray(path) || !path.length) { prevGroup = null; continue }
const named = path.filter(notBlank)
const group = named[0]
if (group !== prevGroup) { td.classList.add('pf-group-start'); prevGroup = group }
if (named.length < depth) td.classList.add('pf-subtotal')
}
})
table.draw()
}
// Size every column to its contents.
//
// Values fit on their own: draw() calls regular_table.resetAutoSize(), which
@ -1214,6 +1397,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
async function submitOp(op) {
if (!slices.length) { flash('Select a slice first', 'error'); return }
// The pivot's filter scopes what the ledger counted, so it has to scope what
// gets written too -- otherwise the panel shows one number and the operation
// changes a larger set. It travels as [col, op, value] rather than folded
// into the slices, because a slice is {col: value} and can only mean
// equality: a view filtered to sseas_e <= 2027 has no slice form at all.
const body = buildPayload(op)
if (!body) return
if (body.slices.some(sl => !Object.keys(sl).length)) {
@ -1321,7 +1509,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
.map(([id, m]) => ({
id: Number(id),
operation: m.operation,
label: (m.tag || m.note || '').trim(),
label: (m.label || m.tag || m.note || '').trim(),
}))
.sort((a, b) => a.id - b.id)
@ -1330,6 +1518,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname))
const out = {}
for (const [k, v] of Object.entries(raw)) {
// Not col_meta columns, but real ones here and resolvable server-side to the
// set of pf.log ids that carry the name. Dropping them is what let a click on
// one bucket's cell scale every bucket at that intersection.
if (COMPUTED_SLICE_COLS.has(k)) { out[k] = v; continue }
if (dimCols.has(k)) { out[k] = v; continue }
if (dateCols.has(k)) {
const ms = Number(v)
@ -1339,7 +1531,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
return out
}
// The scope is read from the ref rather than passed in: the request preview in
// the panel calls this too, and when it was a parameter the preview defaulted
// it away -- showing a payload with no scope for a write that had one.
function buildPayload(op) {
const viewFilter = viewFilterRef.current || []
if (!slices.length) return null
// Two clicked cells can differ only by a column the operation cannot filter on
// (pf_iter, say, which is not in col_meta and so is dropped here). Those become
@ -1358,6 +1554,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
let body = {
tag: opTag.trim() || undefined,
slices: effectiveSlices,
...(viewFilter.length ? { scope: viewFilter } : {}),
...(effectiveSlices.length > 1 ? { apply_mode: applyMode } : {}),
}
if (op === 'scale') {
@ -1507,6 +1704,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
distinctSlices,
dock,
slices, setSlices,
viewScope,
applyMode, setApplyMode,
currentTotals,
activeOp, setActiveOp,
@ -1778,7 +1976,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
<div className="relative flex-1 min-w-0 min-h-0">
{loading && (
<div className="absolute inset-0 flex flex-col items-center justify-center bg-gray-50 z-10 gap-2">
<span className="text-sm text-gray-400">Loading…</span>
<span className="text-sm text-gray-400">
{loadRows ? `Loading ${loadRows.toLocaleString()} rows…` : 'Loading…'}
</span>
{loadProgress && (
<>
<span className="text-xs text-gray-400 font-mono">
@ -1926,12 +2126,22 @@ function LogCell({ entry, field, placeholder, editing, setEditing, onSave, listI
const META_COL_RE = /^__(?:ROW_PATH(?:_\d+)?|ID|GROUPING_ID)__$/
// Perspective encodes a clicked/selected row position as [col, '==', value] triples
function sliceFromFilters(filters) {
// `measures` is the view's `columns` list. Clicking a cell whose column axis is
// collapsed makes Perspective emit the measure name as the value of the first
// hidden split_by dimension -- a bucket subtotal arrives as
// ["smon_e", "==", "sales_usd"] -- because the engine maps split_by positionally
// over a column name that no longer has that many segments. Left in, the slice
// asks for a month equal to a measure, matches nothing, and the operation
// silently has no rows to act on.
function sliceFromFilters(filters, measures = []) {
const measureSet = new Set(measures)
const s = {}
for (const f of filters) {
if (!Array.isArray(f)) continue
const [col, op, val] = f
if (op === '==' && val != null) s[col] = String(val)
if (op !== '==' || val == null) continue
if (measureSet.has(String(val))) continue
s[col] = String(val)
}
return s
}