From 50a0bb42aa8b19e283b2d0f55386669d12390b9b Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 00:17:25 -0400 Subject: [PATCH 01/12] Make a slice mean what it says, and say what cannot move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The phantom: pf_segment and pf_bucket are computed from pf.log when the rows are served, so buildWhere had no column to compare and dropped them. Clicking one bucket's cell and scaling therefore wrote every bucket at that dimension intersection, while the panel showed only the bucket clicked. On the example slice that is 350,524.74 displayed against 503,446.08 written. They resolve exactly, without a new column: the name lives on the log row and every forecast row carries the pf_logid that points at it, so the predicate is pf_logid IN (SELECT id FROM pf.log WHERE = ...). Verified against version 29 -- the clause returns 350,524.74 over 12 rows. Any other pf_ key is now refused rather than skipped, since skipping is the mechanism by which a selection silently widens. pf_iter stays exempt: the client drops it deliberately, two cells differing only by iter band being the same slice. Client side they are ordinary columns in the loaded table, so both the dispatch path and the panel's own totals filter on them directly -- the latter matters as much, or the ledger reconciles against a wider selection than the operation writes. The ledger: excluded rows read "02 - Prior Year · FINAL" in amber rather than "reference · fixed" -- named by the segment a forecaster recognises instead of the iter band that happens to exclude it, and coloured because immovable is a property worth seeing before reading a number. When the whole selection is immovable it now says so in a sentence, where before it printed a row of zeros and left the reason to be inferred from the edit rows failing below. Co-Authored-By: Claude Opus 5 (1M context) --- lib/sql_generator.js | 50 +++++++++++++++++++++++++--- routes/operations.js | 9 ++--- ui/src/components/OperationPanel.jsx | 35 ++++++++++++++++--- ui/src/views/Forecast.jsx | 28 ++++++++++++++-- 4 files changed, 105 insertions(+), 17 deletions(-) diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 02a0921..445f699 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -512,13 +512,53 @@ function applyTokens(sql, tokens) { // build a SQL WHERE clause string from a slice object // only dimension columns are included; unrecognised keys are silently skipped -function buildWhere(slice, dimCols) { +// pf_segment and pf_bucket are not columns on the forecast table -- they are +// computed at read time from the row's pf.log entry -- so a slice naming one +// cannot be compared directly. It resolves to a set of log ids instead, which is +// exact: the name lives on the log row, and every forecast row carries the +// pf_logid that points at it. +// +// Without this they were dropped from the slice, and clicking a single bucket's +// cell scaled every bucket at that dimension intersection while the panel showed +// only the one clicked. +const COMPUTED_SLICE_COLS = { pf_segment: SEGMENT_EXPR, pf_bucket: BUCKET_EXPR }; + +function computedSlicePredicate(col, val, versionId) { + if (versionId == null) { + const err = new Error(`Cannot filter on ${col} without a version`); + err.status = 500; + throw err; + } + const vals = (Array.isArray(val) ? val : [val]).map(v => `'${esc(v)}'`).join(', '); + return `pf_logid IN ( + SELECT l.id + FROM pf.log l${VERSION_JOIN} + WHERE l.version_id = ${parseInt(versionId)} + AND ${COMPUTED_SLICE_COLS[col]} IN (${vals}) +)`; +} + +function buildWhere(slice, dimCols, versionId) { if (!slice || Object.keys(slice).length === 0) return 'TRUE'; const allowed = new Set(dimCols); const parts = []; for (const [col, val] of Object.entries(slice)) { + if (COMPUTED_SLICE_COLS[col]) { + parts.push(computedSlicePredicate(col, val, versionId)); + continue; + } + // A pf_ key this does not understand is refused rather than skipped. + // Skipping is how a selection silently widened: the operation ran against + // everything the dropped key would have excluded. pf_iter is the one + // exception -- the client strips it deliberately, since two cells that + // differ only by iter band are the same slice. + if (col.startsWith('pf_') && col !== 'pf_iter') { + const err = new Error(`Slice names ${col}, which cannot be filtered on`); + err.status = 400; + throw err; + } if (!allowed.has(col)) continue; if (Array.isArray(val)) { const escaped = val.map(v => esc(v)); @@ -535,13 +575,13 @@ function buildWhere(slice, dimCols) { // A union of slices cannot be flattened into one IN list per column: slices // {Region:East, State:NY} and {Region:West, State:CA} would become // Region IN (East,West) AND State IN (NY,CA), which also matches East/CA. -function buildWhereAny(slices, dimCols) { +function buildWhereAny(slices, dimCols, versionId) { const list = (slices || []).filter(s => s && Object.keys(s).length > 0); if (list.length === 0) return 'TRUE'; - if (list.length === 1) return buildWhere(list[0], dimCols); + if (list.length === 1) return buildWhere(list[0], dimCols, versionId); const groups = list - .map(s => buildWhere(s, dimCols)) + .map(s => buildWhere(s, dimCols, versionId)) .filter(w => w !== 'TRUE'); // any slice that reduced to TRUE selects everything, so the union does too @@ -627,6 +667,6 @@ function esc(val) { return String(val).replace(/'/g, "''"); } -module.exports = { generateSQL, grainOf, +module.exports = { generateSQL, grainOf, COMPUTED_SLICE_COLS, SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, LABEL_GROUP_COLS, VERSION_JOIN, ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET, UNLABELED_LOAD, dateGroupsOf, dimPeriodMapOf, dimPeriodJoins, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc }; diff --git a/routes/operations.js b/routes/operations.js index adaf42b..28cac9b 100644 --- a/routes/operations.js +++ b/routes/operations.js @@ -1,6 +1,6 @@ const express = require('express'); const { tableFromArrays, tableToIPC } = require('apache-arrow'); -const { applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc, +const { applyTokens, buildWhere, buildWhereAny, COMPUTED_SLICE_COLS, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc, SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, VERSION_JOIN, ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET } = require('../lib/sql_generator'); const { sessionUser } = require('../lib/auth'); @@ -45,9 +45,10 @@ module.exports = function(pool) { // rather than distribute an amount, so for them this only decides whether the // work lands as one log entry or several. function sliceUnits(slices, ctx, applyMode) { + const vid = ctx.version.id; return applyMode === 'each' - ? slices.map(sl => ({ slices: [sl], where: buildWhere(sl, ctx.filterCols) })) - : [{ slices, where: buildWhereAny(slices, ctx.filterCols) }]; + ? slices.map(sl => ({ slices: [sl], where: buildWhere(sl, ctx.filterCols, vid) })) + : [{ slices, where: buildWhereAny(slices, ctx.filterCols, vid) }]; } // The offset is interpolated into the statement as an interval literal, so a @@ -73,7 +74,7 @@ module.exports = function(pool) { // otherwise reduce to TRUE and apply the operation to the whole version. // Refuse rather than let a malformed selection rewrite every row. function assertSelective(slices, ctx) { - const allowed = new Set(ctx.filterCols); + const allowed = new Set([...ctx.filterCols, ...Object.keys(COMPUTED_SLICE_COLS)]); slices.forEach((sl, i) => { const hits = Object.keys(sl).filter(k => allowed.has(k)); if (hits.length === 0) { diff --git a/ui/src/components/OperationPanel.jsx b/ui/src/components/OperationPanel.jsx index 566ff1a..3803b6c 100644 --- a/ui/src/components/OperationPanel.jsx +++ b/ui/src/components/OperationPanel.jsx @@ -299,7 +299,16 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se const hasExcl = excl.rows > 0 && (excl.value !== 0 || excl.units !== 0) const onTotal = hasExcl && targetBasis !== 'adjustable' // 'selected total' is the default const grand = { value: total.value + excl.value, units: total.units + excl.units } - const exclName = (currentTotals?.excludedIters || []).join(' / ') || 'excluded' + // Named by the segments themselves where we have them -- "02 - Prior Year" + // reads as a thing a forecaster recognises, where "reference" names only the + // iter band that happens to exclude it. + const exclName = (currentTotals?.excluded?.names || []).join(' · ') + || (currentTotals?.excludedIters || []).join(' / ') + || 'excluded' + + // Everything in the selection is immovable. Worth saying outright: the panel + // otherwise prints a row of zeros and leaves the reason to be worked out. + const nothingToAdjust = hasExcl && !total.value && !total.units // the basis decides which line the editable rows are measured from const basisOf = (key) => { @@ -443,12 +452,15 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se {/* Rows the pivot shows but operations cannot write. Listed so the panel's figures reconcile with what the grid displays. */} {hasExcl && ( - - - {exclName} · fixed + + + {exclName} + + final + {measures.map(m => ( - + {m.key === 'price' ? fmtNum(priceOf(excl), m.dp) : fmtNum(excl[m.key], m.dp)} ))} @@ -468,6 +480,19 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se )} + {/* Zeros in the Adjustable row are a true answer to the wrong question: + they say how much can move, not why none of it can. Spell it out + where the eye already is, rather than leaving the edit rows to fail + silently below. */} + {nothingToAdjust && ( + + + Nothing in this selection can be adjusted — all of it is {exclName}, + loaded as {(currentTotals?.excludedIters || []).join(' / ') || 'reference'}. + + + )} + {rule}{measures.map(m =>
)} {/* the edit — three equivalent ways to say the same thing */} diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index c014557..dc8f382 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -12,6 +12,11 @@ import '@perspective-dev/viewer/inline' import '@perspective-dev/viewer-datagrid' import '@perspective-dev/viewer/themes' +// Slice keys that are not col_meta columns: computed from pf.log when the rows are +// served, so they are real columns in the loaded table but have to be resolved back +// to log ids server-side. Mirrors COMPUTED_SLICE_COLS in lib/sql_generator.js. +const COMPUTED_SLICE_COLS = new Set(['pf_segment', 'pf_bucket']) + const LAYOUT_KEY = (vid) => `pf_layout_v${vid}` // last-used layout (auto restore) const LAYOUTS_KEY = (vid) => `pf_layouts_v${vid}` // named layout list @@ -406,6 +411,13 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio async function totalsFor(sliceObj) { const filters = [ + // pf_segment and pf_bucket are computed server-side but are ordinary + // string columns in the loaded table, so here they filter directly. They + // have to be applied, or the ledger totals a wider selection than the + // operation will write. + ...Object.entries(sliceObj) + .filter(([col]) => COMPUTED_SLICE_COLS.has(col)) + .map(([col, val]) => [col, '==', String(val)]), ...Object.entries(sliceObj) .filter(([col]) => dimNames.has(col)) .map(([col, val]) => [col, '==', val]), @@ -424,7 +436,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // rows the pivot shows but operations cannot write (usually 'reference'). // Kept separate rather than filtered away: the grid total includes them, // so the panel has to account for them or the two disagree. - const excluded = { value: 0, units: 0, rows: 0 } + const excluded = { value: 0, units: 0, rows: 0, names: new Set() } for (const r of rows) { const k = r.pf_iter || '?' const val = valueCol ? (parseFloat(r[valueCol]) || 0) : 0 @@ -434,6 +446,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio excluded.value += val excluded.units += uni excluded.rows += 1 + // Name them by what they are, not by the iter band that happens to + // exclude them: "02 - Prior Year" means something to a forecaster, + // "reference" is the mechanism. + if (r.pf_segment) excluded.names.add(String(r.pf_segment)) continue } @@ -505,10 +521,12 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio value: acc.value + (ps.excluded?.value || 0), units: acc.units + (ps.excluded?.units || 0), rows: acc.rows + (ps.excluded?.rows || 0), - }), { value: 0, units: 0, rows: 0 }) + names: new Set([...acc.names, ...(ps.excluded?.names || [])]), + }), { value: 0, units: 0, rows: 0, names: new Set() }) setCurrentTotals({ - byIter, byEntry, total, excluded, valueCol, unitsCol, perSlice, + byIter, byEntry, total, valueCol, unitsCol, perSlice, + excluded: { ...excluded, names: [...excluded.names].sort() }, excludedIters: [...excludeIters], }) } catch { @@ -1330,6 +1348,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname)) const out = {} for (const [k, v] of Object.entries(raw)) { + // Not col_meta columns, but real ones here and resolvable server-side to the + // set of pf.log ids that carry the name. Dropping them is what let a click on + // one bucket's cell scale every bucket at that intersection. + if (COMPUTED_SLICE_COLS.has(k)) { out[k] = v; continue } if (dimCols.has(k)) { out[k] = v; continue } if (dateCols.has(k)) { const ms = Number(v) From f94aa4ec99c3ce2de0f345859b10daad85b9ef2a Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:16:09 -0400 Subject: [PATCH 02/12] Drop the measure name from a collapsed column's slice Clicking a bucket subtotal with the month level collapsed produced {"customer": "...", "pf_bucket": "04 - Forecast", "smon_e": "sales_usd"} -- a month equal to a measure, matching nothing, so the ledger came back empty and an operation would have had no rows to act on. Perspective maps split_by positionally over the column name, and a collapsed axis has fewer segments than there are split_by levels, so the measure lands on the first hidden dimension. Both slice paths now drop any == filter whose value is one of the view's measures; the region path additionally takes the measure off the end of the column name before mapping, which is the same error made in our own code. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index dc8f382..5982bfd 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -323,13 +323,23 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio for (let c = c0; c < c1; c++) { const key = userKeys[c] if (!key) continue + // A column name is dimension values joined by | with the measure last, and + // only as many values as the axis currently shows -- collapse the column + // hierarchy and the deeper levels are simply absent. Mapping split_by + // positionally over every segment therefore read the measure as a value + // for the first collapsed dimension: a bucket subtotal came back as + // smon_e = 'sales_usd', which matches no row, so the operation silently + // had nothing to act on. Drop the measure, then map over what is left. + const segs = key.split('|').slice(0, -1) const colFilters = splitBy + .slice(0, segs.length) .map((col, ix) => { - const v = key.split('|')[ix] + const v = segs[ix] return (v && !META_COL_RE.test(v)) ? [col, '==', v] : null }) .filter(Boolean) - const slice = sliceFromFilters([...base, ...rowFilters, ...colFilters]) + const slice = sliceFromFilters([...base, ...rowFilters, ...colFilters], + (cfg.columns || []).filter(Boolean)) if (!Object.keys(slice).length) continue const y = win.start_row + i out.push({ slice, area: { x0: c, x1: c, y0: y, y1: y } }) @@ -770,7 +780,8 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio if (!detail.row) return const config = await viewer.save() if (!(config.group_by || []).length) return - const s = sliceFromFilters((detail.config || {}).filter || []) + const s = sliceFromFilters((detail.config || {}).filter || [], + (config.columns || []).filter(Boolean)) if (!Object.keys(s).length) return // the CustomEvent carries no modifier flags, so read them off the // mousedown that produced it (captured on window below) @@ -1948,12 +1959,22 @@ function LogCell({ entry, field, placeholder, editing, setEditing, onSave, listI const META_COL_RE = /^__(?:ROW_PATH(?:_\d+)?|ID|GROUPING_ID)__$/ // Perspective encodes a clicked/selected row position as [col, '==', value] triples -function sliceFromFilters(filters) { +// `measures` is the view's `columns` list. Clicking a cell whose column axis is +// collapsed makes Perspective emit the measure name as the value of the first +// hidden split_by dimension -- a bucket subtotal arrives as +// ["smon_e", "==", "sales_usd"] -- because the engine maps split_by positionally +// over a column name that no longer has that many segments. Left in, the slice +// asks for a month equal to a measure, matches nothing, and the operation +// silently has no rows to act on. +function sliceFromFilters(filters, measures = []) { + const measureSet = new Set(measures) const s = {} for (const f of filters) { if (!Array.isArray(f)) continue const [col, op, val] = f - if (op === '==' && val != null) s[col] = String(val) + if (op !== '==' || val == null) continue + if (measureSet.has(String(val))) continue + s[col] = String(val) } return s } From c6ef35028369ac16742ad27f0523030ca7ce6ee0 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:33:34 -0400 Subject: [PATCH 03/12] Type a slice's values before filtering the ledger with them A slice carries every value as a string -- built from filters the grid reports, and shaped to survive JSON on the way to the API. Perspective matches on type, and a string '2027' against an integer column is not a filter that matches nothing, it is a filter that is dropped. So the pivot's own season filter never reached the ledger: with the grid scoped to sseas_e = 2027 the ledger totalled 956,485.13 against a cell reading 921,225.71, the difference being eleven rows of a baseline segment whose shipments fall in the next season. Only dates were being coerced, and only because someone had hit this before with them. Values are now typed against the loaded table's schema rather than against col_meta's role, which is the thing that actually decides the match. The server side was already right -- Postgres casts the literal -- and returns 921,225.71 for the same slice. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 5982bfd..3f4fb3e 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -419,22 +419,31 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const dateNames = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname)) const ITER_ORDER = ['baseline', 'scale', 'recode', 'clone'] + // A slice carries every value as a string -- it is built from filters the + // grid reports and from a payload that has to survive JSON. Perspective + // matches on type, so a string '2027' against an integer column does not + // filter to nothing, it is dropped: the ledger then totalled rows the pivot + // was hiding, which is how a season filter on sseas_e went unnoticed while + // the numbers disagreed by exactly the out-of-season rows. + const schema = await tableRef.current.schema() + const typed = (col, val) => { + switch (schema[col]) { + case 'integer': case 'float': return Number(val) + case 'boolean': return val === true || val === 'true' + case 'date': case 'datetime': return Number(val) + default: return String(val) + } + } + async function totalsFor(sliceObj) { - const filters = [ - // pf_segment and pf_bucket are computed server-side but are ordinary - // string columns in the loaded table, so here they filter directly. They - // have to be applied, or the ledger totals a wider selection than the - // operation will write. - ...Object.entries(sliceObj) - .filter(([col]) => COMPUTED_SLICE_COLS.has(col)) - .map(([col, val]) => [col, '==', String(val)]), - ...Object.entries(sliceObj) - .filter(([col]) => dimNames.has(col)) - .map(([col, val]) => [col, '==', val]), - ...Object.entries(sliceObj) - .filter(([col]) => dateNames.has(col)) - .map(([col, val]) => [col, '==', Number(val)]), - ] + // pf_segment and pf_bucket are computed server-side but are ordinary + // columns in the loaded table, so here they filter directly. They have to + // be applied, or the ledger totals a wider selection than the operation + // will write. + const filters = Object.entries(sliceObj) + .filter(([col]) => COMPUTED_SLICE_COLS.has(col) || dimNames.has(col) || dateNames.has(col) + || schema[col] !== undefined) + .map(([col, val]) => [col, '==', typed(col, val)]) const view = await tableRef.current.view({ filter: filters }) const rows = await view.to_json() await view.delete() From 6b63e9a5f3efbfc5bd1bf9e0bf6cc90e7df84c66 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:38:04 -0400 Subject: [PATCH 04/12] Say how many rows the load is waiting on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Row-Count arrives with the headers, long before the body has been read, so the overlay can name the wait instead of saying "Loading…" over a grey screen for fifteen seconds. On this data the row count *is* the wait -- the bytes are quick and the rows are not -- so it is the number worth showing beside the transfer bar, which only ever measured the fast part. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 3f4fb3e..c5fc360 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -57,6 +57,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const [loading, setLoading] = useState(false) const [largeDataset, setLargeDataset] = useState(false) const [loadProgress, setLoadProgress] = useState(null) // { received, total } + // Rows the server says it is sending, from X-Row-Count. Known as soon as the + // headers land, well before the body has been read, so the wait can say what + // it is waiting for -- on this data the row count is the wait (see CLAUDE.md, + // "Load time is dominated by row count"). + const [loadRows, setLoadRows] = useState(null) const [msg, setMsg] = useState(null) // layouts @@ -619,6 +624,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio if (!r.ok) { const { error } = await r.json(); throw new Error(error || 'Failed to load data') } const rowCount = parseInt(r.headers.get('X-Row-Count') || '0') const total = parseInt(r.headers.get('Content-Length') || '0') || null + setLoadRows(rowCount || null) const reader = r.body.getReader() const chunks = [] let received = 0 @@ -655,6 +661,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio setLoading(true) setLargeDataset(false) setLoadProgress(null) + setLoadRows(null) setSlices([]) setExpandDepth(null) adoptSplit([], 0) @@ -1820,7 +1827,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
{loading && (
- Loading… + + {loadRows ? `Loading ${loadRows.toLocaleString()} rows…` : 'Loading…'} + {loadProgress && ( <> From 9753846d340ff90e8cf83180579101efc4017351 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:39:44 -0400 Subject: [PATCH 05/12] Make the pivot's filter scope the ledger and the write perspective-click reports only the clicked cell's own dimensions -- the view-level filter is not in it -- so a slice never carried the season the grid was scoped to. The ledger therefore counted rows the grid was hiding (921,225.71 on screen against 956,485.13 in the panel) and an operation would have written them. Both now read the filter off the viewer. The ledger applies it to its own view, where the values are already in the table's types and any operator works. The operation merges the equalities into each slice, cell values winning on a shared column since a cell cannot contradict the filter it was drawn inside, and refuses outright on any other operator: a range or an in-list cannot travel in a slice, and dropping it silently is the widening this is meant to stop. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 56 ++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index c5fc360..243b339 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -430,6 +430,22 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // filter to nothing, it is dropped: the ledger then totalled rows the pivot // was hiding, which is how a season filter on sseas_e went unnoticed while // the numbers disagreed by exactly the out-of-season rows. + // The pivot's own filter is not part of a clicked slice -- perspective-click + // reports only the cell's own dimensions -- so the ledger has to read it off + // the viewer and apply it alongside. Without this the ledger totals rows the + // grid is hiding, and the operation writes them: a grid scoped to + // sseas_e = 2027 gave a cell of 921,225.71 against a ledger of 956,485.13. + // + // Taken from viewer.save(), so the values are already in the table's own + // types and the operators are whatever the user set -- ranges and in-lists + // included, which a slice cannot express. + const viewFilter = await (async () => { + try { + const cfg = await viewerRef.current?.save() + return (cfg?.filter || []).filter(f => Array.isArray(f) && f.length >= 2) + } catch { return [] } + })() + const schema = await tableRef.current.schema() const typed = (col, val) => { switch (schema[col]) { @@ -445,10 +461,16 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // columns in the loaded table, so here they filter directly. They have to // be applied, or the ledger totals a wider selection than the operation // will write. - const filters = Object.entries(sliceObj) - .filter(([col]) => COMPUTED_SLICE_COLS.has(col) || dimNames.has(col) || dateNames.has(col) - || schema[col] !== undefined) - .map(([col, val]) => [col, '==', typed(col, val)]) + const filters = [ + ...viewFilter, + ...Object.entries(sliceObj) + .filter(([col]) => COMPUTED_SLICE_COLS.has(col) || dimNames.has(col) || dateNames.has(col) + || schema[col] !== undefined) + // a cell inside the filtered view cannot contradict it, so a repeated + // column is the same predicate twice and harmless + .filter(([col]) => !viewFilter.some(f => f[0] === col)) + .map(([col, val]) => [col, '==', typed(col, val)]), + ] const view = await tableRef.current.view({ filter: filters }) const rows = await view.to_json() await view.delete() @@ -1259,7 +1281,24 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio async function submitOp(op) { if (!slices.length) { flash('Select a slice first', 'error'); return } - const body = buildPayload(op) + // The pivot's filter scopes what the ledger counted, so it has to scope what + // gets written too -- otherwise the panel shows one number and the operation + // changes a larger set. Only equalities can travel in a slice; anything else + // is refused rather than dropped, because dropping it is exactly the silent + // widening this is here to prevent. + let viewFilter = [] + try { + const cfg = await viewerRef.current?.save() + viewFilter = (cfg?.filter || []).filter(f => Array.isArray(f) && f.length >= 2) + } catch { viewFilter = [] } + const unsendable = viewFilter.filter(f => f[1] !== '==') + if (unsendable.length) { + flash(`The pivot filter ${unsendable.map(f => f.join(' ')).join(', ')} cannot be ` + + `applied to an operation. Narrow the selection instead, or use "==".`, 'error') + return + } + + const body = buildPayload(op, viewFilter) if (!body) return if (body.slices.some(sl => !Object.keys(sl).length)) { flash('No dimension or date columns in slice — check col_meta', 'error'); return @@ -1388,8 +1427,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio return out } - function buildPayload(op) { + function buildPayload(op, viewFilter = []) { if (!slices.length) return null + const scope = Object.fromEntries(viewFilter.map(([col, , val]) => [col, val])) // Two clicked cells can differ only by a column the operation cannot filter on // (pf_iter, say, which is not in col_meta and so is dropped here). Those become // the same effective slice, and sending it twice would apply the change twice @@ -1397,7 +1437,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const seen = new Set() const effectiveSlices = [] for (const sl of slices) { - const eff = buildEffectiveSlice(sl) + // the view's scope first, so a cell's own value wins if they name the + // same column -- it cannot contradict the filter it was drawn inside + const eff = buildEffectiveSlice({ ...scope, ...sl }) const key = JSON.stringify(Object.keys(eff).sort().map(k => [k, eff[k]])) if (seen.has(key)) continue seen.add(key) From 1a0a9db8d00488c46b349ed3d2975d74ebd61752 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:41:24 -0400 Subject: [PATCH 06/12] Show the row count while the load is still waiting on it X-Row-Count is exact but travels with the response headers, and in grain mode the server aggregates the whole table before sending any -- so the number appeared just as the fifteen-second wait ended, which is no use to anyone watching it. The forecast table's own count goes up first instead, from the same table-info the status bar already reads, and the exact figure replaces it when the headers arrive. The count query is deliberately not awaited: it scans the whole table, and the load must not wait on a progress message. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 243b339..88c97fc 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -57,10 +57,15 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const [loading, setLoading] = useState(false) const [largeDataset, setLargeDataset] = useState(false) const [loadProgress, setLoadProgress] = useState(null) // { received, total } - // Rows the server says it is sending, from X-Row-Count. Known as soon as the - // headers land, well before the body has been read, so the wait can say what - // it is waiting for -- on this data the row count is the wait (see CLAUDE.md, - // "Load time is dominated by row count"). + // Rows the load is waiting on, so the wait can say what it is waiting for -- + // on this data the row count is the wait (see CLAUDE.md, "Load time is + // dominated by row count"). + // + // Filled twice. X-Row-Count is exact but arrives with the response headers, + // and in grain mode the server aggregates before sending any: the number + // turned up just as the wait ended. So the forecast table's own count goes in + // first, from the same table-info the status bar reads, and the exact figure + // replaces it when the headers land. const [loadRows, setLoadRows] = useState(null) const [msg, setMsg] = useState(null) @@ -646,7 +651,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio if (!r.ok) { const { error } = await r.json(); throw new Error(error || 'Failed to load data') } const rowCount = parseInt(r.headers.get('X-Row-Count') || '0') const total = parseInt(r.headers.get('Content-Length') || '0') || null - setLoadRows(rowCount || null) + if (rowCount) setLoadRows(rowCount) const reader = r.body.getReader() const chunks = [] let received = 0 @@ -684,6 +689,12 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio setLargeDataset(false) setLoadProgress(null) setLoadRows(null) + // deliberately not awaited: it is a count over the whole forecast table and + // the load must not wait on it + fetch(`/api/versions/${vid}/table-info`) + .then(r => r.ok ? r.json() : null) + .then(info => { if (info?.rows && initIdRef.current === myId) setLoadRows(n => n ?? info.rows) }) + .catch(() => {}) setSlices([]) setExpandDepth(null) adoptSplit([], 0) From d983e2b1df7e6810c6bd158897779691944353e3 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 08:46:23 -0400 Subject: [PATCH 07/12] Carry the pivot filter as a scope, operators and all Refusing anything but == was safe and useless: a view bounded to sseas_e <= 2027 is an ordinary way to scope a forecast, and it has no slice form at all, a slice being {col: value}. The filter now travels beside the slices as [col, op, value] triples and is ANDed onto every unit -- not folded into the slices, since it applies to all of them equally and under apply_mode 'each' would just repeat itself in every statement. Operators are Perspective's, since that is where they come from, and the list is a whitelist: anything outside it is refused rather than ignored, because a scope silently dropped is a write wider than the panel that authorised it. The scope goes into the log's params too, so the audit trail records what bounded the write and not only what was clicked. The panel prints it above the selection as "within sseas_e <= 2027". It scopes every figure below it and every row the operation writes while appearing in none of the slices, so without it the panel showed a selection wider than the one it was acting on -- which is exactly what made the ledger's 956,485.13 look plausible against a cell of 921,225.71. Co-Authored-By: Claude Opus 5 (1M context) --- lib/sql_generator.js | 58 +++++++++++++++++++++++++++- routes/operations.js | 23 ++++++----- ui/src/components/OperationPanel.jsx | 21 +++++++++- ui/src/views/Forecast.jsx | 37 ++++++++---------- 4 files changed, 108 insertions(+), 31 deletions(-) diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 445f699..3d655b9 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -571,6 +571,62 @@ function buildWhere(slice, dimCols, versionId) { return parts.length ? parts.join('\nAND ') : 'TRUE'; } +// The pivot's own filter, carried alongside the slices so an operation writes +// exactly the rows the ledger counted. +// +// A slice is {col: value} and can only ever mean equality, so a view filtered to +// sseas_e <= 2027 could not be expressed as one. Refusing it was safe but +// useless -- a bounded season is an ordinary way to scope a forecast -- so the +// operators travel as [col, op, value] triples instead. +// +// Perspective's operator names, not SQL's, since that is where these come from. +// Anything outside this list is refused rather than ignored: a scope silently +// dropped is a write that is wider than the panel that authorised it. +const SCOPE_OPS = { + '==': (c, v) => `${c} = ${v[0]}`, + '!=': (c, v) => `${c} != ${v[0]}`, + '>': (c, v) => `${c} > ${v[0]}`, + '>=': (c, v) => `${c} >= ${v[0]}`, + '<': (c, v) => `${c} < ${v[0]}`, + '<=': (c, v) => `${c} <= ${v[0]}`, + 'in': (c, v) => `${c} IN (${v.join(', ')})`, + 'not in': (c, v) => `${c} NOT IN (${v.join(', ')})`, + 'is null': (c) => `${c} IS NULL`, + 'is not null': (c) => `${c} IS NOT NULL`, +}; + +function buildScopeClause(scope, dimCols, versionId) { + if (!Array.isArray(scope) || scope.length === 0) return ''; + const allowed = new Set(dimCols); + const parts = scope.map((entry) => { + if (!Array.isArray(entry) || entry.length < 2) { + const err = new Error(`Malformed scope entry ${JSON.stringify(entry)}`); + err.status = 400; throw err; + } + const [col, op, ...rest] = entry; + const vals = (Array.isArray(rest[0]) ? rest[0] : rest).filter(v => v !== undefined); + + if (COMPUTED_SLICE_COLS[col]) { + if (op !== '==' && op !== 'in') { + const err = new Error(`${col} can only be scoped with == or in, not ${op}`); + err.status = 400; throw err; + } + return computedSlicePredicate(col, vals, versionId); + } + if (!allowed.has(col)) { + const err = new Error(`Column "${col}" is not available for filtering`); + err.status = 400; throw err; + } + const fn = SCOPE_OPS[op]; + if (!fn) { + const err = new Error(`Unsupported filter operator "${op}" on ${col}`); + err.status = 400; throw err; + } + return fn(`"${col}"`, vals.map(v => `'${esc(String(v))}'`)); + }); + return parts.join('\nAND '); +} + // build a WHERE clause spanning several slices — an OR of AND-groups. // A union of slices cannot be flattened into one IN list per column: slices // {Region:East, State:NY} and {Region:West, State:CA} would become @@ -667,6 +723,6 @@ function esc(val) { return String(val).replace(/'/g, "''"); } -module.exports = { generateSQL, grainOf, COMPUTED_SLICE_COLS, +module.exports = { generateSQL, grainOf, COMPUTED_SLICE_COLS, buildScopeClause, SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, LABEL_GROUP_COLS, VERSION_JOIN, ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET, UNLABELED_LOAD, dateGroupsOf, dimPeriodMapOf, dimPeriodJoins, applyTokens, buildWhere, buildWhereAny, buildExcludeClause, buildExcludePredicate, buildSetClause, buildFilterClause, esc }; diff --git a/routes/operations.js b/routes/operations.js index 28cac9b..d4a7b48 100644 --- a/routes/operations.js +++ b/routes/operations.js @@ -1,6 +1,6 @@ const express = require('express'); const { tableFromArrays, tableToIPC } = require('apache-arrow'); -const { applyTokens, buildWhere, buildWhereAny, COMPUTED_SLICE_COLS, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc, +const { applyTokens, buildWhere, buildWhereAny, COMPUTED_SLICE_COLS, buildScopeClause, buildExcludeClause, buildExcludePredicate, buildSetClause, dateGroupsOf, dimPeriodMapOf, esc, SEGMENT_EXPR, BUCKET_EXPR, NOTE_EXPR, VERSION_JOIN, ADJUSTMENT_SEGMENT, ADJUSTMENT_BUCKET } = require('../lib/sql_generator'); const { sessionUser } = require('../lib/auth'); @@ -44,11 +44,16 @@ module.exports = function(pool) { // Only scale has a target to prorate towards; recode and clone rewrite rows // rather than distribute an amount, so for them this only decides whether the // work lands as one log entry or several. - function sliceUnits(slices, ctx, applyMode) { - const vid = ctx.version.id; + // The scope is ANDed onto every unit rather than folded into the slices: it + // applies to all of them equally, and under apply_mode 'each' folding it in + // would repeat the same predicate in every statement for no gain. + function sliceUnits(slices, ctx, applyMode, scope) { + const vid = ctx.version.id; + const scl = buildScopeClause(scope, ctx.filterCols, vid); + const and = (w) => (scl ? (w === 'TRUE' ? scl : `${w}\nAND ${scl}`) : w); return applyMode === 'each' - ? slices.map(sl => ({ slices: [sl], where: buildWhere(sl, ctx.filterCols, vid) })) - : [{ slices, where: buildWhereAny(slices, ctx.filterCols, vid) }]; + ? slices.map(sl => ({ slices: [sl], where: and(buildWhere(sl, ctx.filterCols, vid)) })) + : [{ slices, where: and(buildWhereAny(slices, ctx.filterCols, vid)) }]; } // The offset is interpolated into the statement as an interval literal, so a @@ -91,7 +96,7 @@ module.exports = function(pool) { // echo back what the caller asked for, for the audit log function pickIntent(body) { const keys = ['mode', 'target_basis', 'value_incr', 'units_incr', 'value_pct', 'units_pct', 'pct', - 'target_value', 'target_units', 'target_price']; + 'target_value', 'target_units', 'target_price', 'scope']; const out = {}; for (const k of keys) if (body[k] !== undefined && body[k] !== null && body[k] !== '') out[k] = body[k]; return out; @@ -624,7 +629,7 @@ module.exports = function(pool) { // sum() OVER () distribute the increment across the whole pool. // 'each' runs the same statement once per slice, so every slice // reaches the target on its own and gets its own log entry. - const units = sliceUnits(slices, ctx, applyMode); + const units = sliceUnits(slices, ctx, applyMode, req.body.scope); const client = await pool.connect(); let committed = false; @@ -706,7 +711,7 @@ module.exports = function(pool) { const excludeClause = buildExcludeClause(ctx.version.exclude_iters); const setClause = buildSetClause(ctx.dimCols, set); - const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate'); + const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate', req.body.scope); const client = await pool.connect(); let committed = false; @@ -795,7 +800,7 @@ module.exports = function(pool) { [cname, `${alias}."${periodCol}"`]) ); const setClause = buildSetClause(ctx.dimCols, set, { derivedExprs, alias: 's' }); - const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate'); + const units = sliceUnits(slices, ctx, apply_mode === 'each' ? 'each' : 'prorate', req.body.scope); const client = await pool.connect(); let committed = false; diff --git a/ui/src/components/OperationPanel.jsx b/ui/src/components/OperationPanel.jsx index 3803b6c..df16b0b 100644 --- a/ui/src/components/OperationPanel.jsx +++ b/ui/src/components/OperationPanel.jsx @@ -110,11 +110,22 @@ function Submit({ onClick, children, disabled }) { } // ── 1. Selection ──────────────────────────────────────────────────────────── -function SelectionList({ slices, currentTotals, onRemove, onClear }) { +function SelectionList({ slices, viewScope = [], currentTotals, onRemove, onClear }) { const multi = slices.length > 1 const perSlice = currentTotals?.perSlice || [] const valueCol = currentTotals?.valueCol + // The pivot's own filter. Shown because it scopes every figure below and + // every row the operation writes, while appearing in none of the slices -- + // perspective-click reports only the cell's own dimensions, so without this + // the panel prints a selection wider than the one it is acting on. + const scopeLine = viewScope + .map(([col, op, ...rest]) => { + const vals = (Array.isArray(rest[0]) ? rest[0] : rest).filter(v => v !== undefined) + return `${col} ${op}${vals.length ? ' ' + vals.join(', ') : ''}` + }) + .join(' · ') + if (!slices.length) { return (

@@ -126,6 +137,12 @@ function SelectionList({ slices, currentTotals, onRemove, onClear }) { return (

+ {scopeLine && ( +
+ within + {scopeLine} +
+ )}
@@ -763,6 +780,7 @@ function RequestPreview({ payload }) { export default function OperationPanel({ dock, slices, setSlices, distinctSlices, + viewScope = [], applyMode, setApplyMode, currentTotals, activeOp, setActiveOp, @@ -805,6 +823,7 @@ export default function OperationPanel({ )} setSlices(prev => prev.filter((_, x) => x !== i))} onClear={() => setSlices([])} diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 88c97fc..f25f293 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -56,6 +56,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const { dark } = useTheme() const [loading, setLoading] = useState(false) const [largeDataset, setLargeDataset] = useState(false) + // The pivot's own filter, refreshed whenever the ledger recomputes. A ref + // rather than state: it is read at dispatch and at totals time, never + // rendered from directly, and making it state would re-run the totals effect + // that sets it. + const viewFilterRef = useRef([]) const [loadProgress, setLoadProgress] = useState(null) // { received, total } // Rows the load is waiting on, so the wait can say what it is waiting for -- // on this data the row count is the wait (see CLAUDE.md, "Load time is @@ -67,6 +72,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // first, from the same table-info the status bar reads, and the exact figure // replaces it when the headers land. const [loadRows, setLoadRows] = useState(null) + // the same filter, for display: the panel has to show the scope it is acting + // inside or the slice it prints is not the slice that gets written + const [viewScope, setViewScope] = useState([]) const [msg, setMsg] = useState(null) // layouts @@ -450,6 +458,8 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio return (cfg?.filter || []).filter(f => Array.isArray(f) && f.length >= 2) } catch { return [] } })() + viewFilterRef.current = viewFilter + setViewScope(viewFilter) const schema = await tableRef.current.schema() const typed = (col, val) => { @@ -1294,22 +1304,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // The pivot's filter scopes what the ledger counted, so it has to scope what // gets written too -- otherwise the panel shows one number and the operation - // changes a larger set. Only equalities can travel in a slice; anything else - // is refused rather than dropped, because dropping it is exactly the silent - // widening this is here to prevent. - let viewFilter = [] - try { - const cfg = await viewerRef.current?.save() - viewFilter = (cfg?.filter || []).filter(f => Array.isArray(f) && f.length >= 2) - } catch { viewFilter = [] } - const unsendable = viewFilter.filter(f => f[1] !== '==') - if (unsendable.length) { - flash(`The pivot filter ${unsendable.map(f => f.join(' ')).join(', ')} cannot be ` - + `applied to an operation. Narrow the selection instead, or use "==".`, 'error') - return - } - - const body = buildPayload(op, viewFilter) + // changes a larger set. It travels as [col, op, value] rather than folded + // into the slices, because a slice is {col: value} and can only mean + // equality: a view filtered to sseas_e <= 2027 has no slice form at all. + const body = buildPayload(op, viewFilterRef.current) if (!body) return if (body.slices.some(sl => !Object.keys(sl).length)) { flash('No dimension or date columns in slice — check col_meta', 'error'); return @@ -1440,7 +1438,6 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio function buildPayload(op, viewFilter = []) { if (!slices.length) return null - const scope = Object.fromEntries(viewFilter.map(([col, , val]) => [col, val])) // Two clicked cells can differ only by a column the operation cannot filter on // (pf_iter, say, which is not in col_meta and so is dropped here). Those become // the same effective slice, and sending it twice would apply the change twice @@ -1448,9 +1445,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const seen = new Set() const effectiveSlices = [] for (const sl of slices) { - // the view's scope first, so a cell's own value wins if they name the - // same column -- it cannot contradict the filter it was drawn inside - const eff = buildEffectiveSlice({ ...scope, ...sl }) + const eff = buildEffectiveSlice(sl) const key = JSON.stringify(Object.keys(eff).sort().map(k => [k, eff[k]])) if (seen.has(key)) continue seen.add(key) @@ -1460,6 +1455,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio let body = { tag: opTag.trim() || undefined, slices: effectiveSlices, + ...(viewFilter.length ? { scope: viewFilter } : {}), ...(effectiveSlices.length > 1 ? { apply_mode: applyMode } : {}), } if (op === 'scale') { @@ -1609,6 +1605,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio distinctSlices, dock, slices, setSlices, + viewScope, applyMode, setApplyMode, currentTotals, activeOp, setActiveOp, From cdb40e7368d1327bb9b03a263b4cd759dba5ffa5 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 09:10:09 -0400 Subject: [PATCH 08/12] Make the request preview show the request The panel's preview calls buildPayload itself, and the scope arrived as a parameter that the preview had no way to supply -- so it defaulted to empty and printed a payload with no scope for a write that had one. A preview that disagrees with what is sent is worse than no preview: it is the one place someone looks to check before committing a change. buildPayload reads the scope from the ref instead, so there is one payload and both callers get it. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index f25f293..34305d5 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -1307,7 +1307,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // changes a larger set. It travels as [col, op, value] rather than folded // into the slices, because a slice is {col: value} and can only mean // equality: a view filtered to sseas_e <= 2027 has no slice form at all. - const body = buildPayload(op, viewFilterRef.current) + const body = buildPayload(op) if (!body) return if (body.slices.some(sl => !Object.keys(sl).length)) { flash('No dimension or date columns in slice — check col_meta', 'error'); return @@ -1436,7 +1436,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio return out } - function buildPayload(op, viewFilter = []) { + // The scope is read from the ref rather than passed in: the request preview in + // the panel calls this too, and when it was a parameter the preview defaulted + // it away -- showing a payload with no scope for a write that had one. + function buildPayload(op) { + const viewFilter = viewFilterRef.current || [] if (!slices.length) return null // Two clicked cells can differ only by a column the operation cannot filter on // (pf_iter, say, which is not in col_meta and so is dropped here). Those become From 54d49ebc754273e14a8ab86164f6dfd386f3429d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 09:20:27 -0400 Subject: [PATCH 09/12] Rule off the column groups and mark their subtotals Prior, plan and forecast each carry twelve months and a total, so scanning across is thirty-odd columns of identical-looking numbers with nothing to say where one domain ends and the next begins -- annual figures read as just another month. Each group's first column now takes a left rule and each group's subtotal a tint and a heavier weight. Both are derived from the cell's column path: the deepest path is a leaf, so anything shorter is an aggregate of the levels below it, which is what makes a subtotal a subtotal. Through regular_table's style listener rather than CSS, because the DOM cells are recycled across columns as you scroll -- a stylesheet would paint the wrong ones the moment the grid virtualised. The styles go into the grid's own shadow root, since a page stylesheet cannot reach it, and use currentColor so the rule follows the theme instead of disappearing against Pro Dark. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 34305d5..901a60a 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -866,6 +866,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio viewer.addEventListener('perspective-select', viewer._pspSelect) gridRef.current = await viewer.getPlugin() + applyGroupRules() setLargeDataset(false) } catch (err) { @@ -929,6 +930,67 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio } } + // Rule off the column groups, and mark each group's subtotal. + // + // Scanning across "prior · plan · forecast, each with twelve months and a + // total" is twelve columns of identical-looking numbers with nothing to say + // where one domain ends and the next begins. The annual figures and the + // monthly ones read as one run. + // + // Done through regular_table's style listener rather than CSS: which column + // starts a group, and which one is a group's subtotal, are facts about the + // data that only the cell metadata knows. The listener runs on every draw, so + // it survives scrolling and virtualisation -- a stylesheet cannot, since the + // DOM cells are recycled across columns as you scroll. + function applyGroupRules() { + const grid = gridRef.current + const table = grid?.regular_table + if (!table || table._pfGroupRules) return + table._pfGroupRules = true + + // The grid lives in a shadow root, so a stylesheet on the page cannot reach + // these cells. Inject into whichever root actually contains the table. + // currentColor rather than a fixed grey, so the rule follows the theme + // instead of vanishing against Pro Dark. + const root = table.getRootNode() || document + if (!root.querySelector('#pf-group-rules')) { + const style = document.createElement('style') + style.id = 'pf-group-rules' + style.textContent = ` + td.pf-group-start { border-left: 2px solid currentColor; opacity: 1; } + td.pf-subtotal { font-weight: 600; background: color-mix(in srgb, currentColor 7%, transparent); } + ` + ;(root.head || root).appendChild(style) + } + + table.addStyleListener(() => { + const body = table.querySelectorAll('tbody td') + // The deepest column path is a leaf; anything shorter is an aggregate of + // the levels below it, which is what makes a subtotal a subtotal. + let depth = 0 + const metas = [] + for (const td of body) { + let meta + try { meta = table.getMeta(td) } catch { meta = null } + metas.push([td, meta]) + const path = meta?.column_header + if (Array.isArray(path)) depth = Math.max(depth, path.filter(v => v != null && v !== '').length) + } + + let prevGroup = null + for (const [td, meta] of metas) { + td.classList.remove('pf-group-start', 'pf-subtotal') + const path = meta?.column_header + if (!Array.isArray(path) || !path.length) { prevGroup = null; continue } + const named = path.filter(v => v != null && v !== '') + const group = named[0] + if (group !== prevGroup) { td.classList.add('pf-group-start'); prevGroup = group } + if (named.length < depth) td.classList.add('pf-subtotal') + } + }) + table.draw() + } + // Size every column to its contents. // // Values fit on their own: draw() calls regular_table.resetAutoSize(), which From f9424d9c424b3e73ac086360127ff890001a7932 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 09:26:21 -0400 Subject: [PATCH 10/12] Recognise a subtotal by its zero-width padding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perspective pads a subtotal's column path to full length rather than shortening it -- ['04 - Forecast', '​', 'sales_usd'] -- so testing for an empty string found no subtotals and nothing was tinted. The blank test now strips zero-width spaces and the other invisibles alongside whitespace. The grand total falls out of the same rule, its path being blank at every level above the measure. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/views/Forecast.jsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 901a60a..d0f2d9b 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -942,6 +942,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // data that only the cell metadata knows. The listener runs on every draw, so // it survives scrolling and virtualisation -- a stylesheet cannot, since the // DOM cells are recycled across columns as you scroll. + // Blank for this purpose means "no value at this level", which Perspective + // writes as a zero-width space rather than an empty string. + const notBlank = (v) => + v != null && String(v).replace(/[\s\u200b-\u200d\ufeff]/g, '') !== '' + function applyGroupRules() { const grid = gridRef.current const table = grid?.regular_table @@ -967,6 +972,11 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const body = table.querySelectorAll('tbody td') // The deepest column path is a leaf; anything shorter is an aggregate of // the levels below it, which is what makes a subtotal a subtotal. + // + // The empty levels are not empty strings. Perspective pads a subtotal's + // path with zero-width spaces -- ['04 - Forecast', '\u200b', 'sales_usd'] + // -- so every path is the same length and a naive `!== ''` test finds no + // subtotals at all. let depth = 0 const metas = [] for (const td of body) { @@ -974,7 +984,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio try { meta = table.getMeta(td) } catch { meta = null } metas.push([td, meta]) const path = meta?.column_header - if (Array.isArray(path)) depth = Math.max(depth, path.filter(v => v != null && v !== '').length) + if (Array.isArray(path)) depth = Math.max(depth, path.filter(notBlank).length) } let prevGroup = null @@ -982,7 +992,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio td.classList.remove('pf-group-start', 'pf-subtotal') const path = meta?.column_header if (!Array.isArray(path) || !path.length) { prevGroup = null; continue } - const named = path.filter(v => v != null && v !== '') + const named = path.filter(notBlank) const group = named[0] if (group !== prevGroup) { td.classList.add('pf-group-start'); prevGroup = group } if (named.length < depth) td.classList.add('pf-subtotal') From 2f862a6c8bf06b59af606006f4360992a6ed5b49 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 09:38:37 -0400 Subject: [PATCH 11/12] Name the ledger's lines the way everything else names them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The walk read from tag and note, and hardcoded the word "Baseline" for the baseline load -- so a segment called 03 - New Orders in the pivot, in the bridge and on the Baseline page read as "Baseline" in the one place you go to check a number before changing it. label comes first now, the same precedence pf_segment uses, in the ledger and the bridge alike. logMeta did not carry label at all, which is why neither could reach it. The immovable rows split one line per segment. Combined, "01 - YTD Sales · 02 - Open Orders" said 1.6m was untouchable without saying how much of it was billed and how much was booked -- different things a forecaster treats differently. The FINAL badge also gains the space it was missing, having rendered as "02 - Open Ordersfinal". Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/components/BridgeView.jsx | 4 +++- ui/src/components/OperationPanel.jsx | 34 +++++++++++++++++++-------- ui/src/views/Forecast.jsx | 35 +++++++++++++++++++++++----- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ui/src/components/BridgeView.jsx b/ui/src/components/BridgeView.jsx index c4781ae..f61b71d 100644 --- a/ui/src/components/BridgeView.jsx +++ b/ui/src/components/BridgeView.jsx @@ -106,7 +106,9 @@ export function buildSteps(rows, { if (isLoad) { loads.value += v; loads.units += u; loads.rows += 1; continue } const meta = logMeta[r.pf_logid] || {} - const tag = (meta.tag || '').trim() + // label first, the same precedence pf_segment uses, so the bridge and the + // pivot call a step by the same name + const tag = (meta.label || meta.tag || '').trim() const label = tag || (meta.note || '').trim() || `${(meta.operation || r.pf_iter || 'adj')}${r.pf_logid != null ? ` #${r.pf_logid}` : ''}` const key = tag ? `tag:${tag}` : `log:${r.pf_logid}` diff --git a/ui/src/components/OperationPanel.jsx b/ui/src/components/OperationPanel.jsx index df16b0b..fcdb223 100644 --- a/ui/src/components/OperationPanel.jsx +++ b/ui/src/components/OperationPanel.jsx @@ -287,9 +287,16 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se const loose = [] const byTag = new Map() for (const e of entries) { - if (e.key === 'baseline') { baseline.push({ ...e, label: 'Baseline', kind: 'baseline' }); continue } const meta = logMeta[e.logid] || {} - const tag = (meta.tag || '').trim() + // Named like every other line: the label the pivot shows, then the older + // fallbacks. "Baseline" was hardcoded, so a segment called 03 - New Orders + // everywhere else read as "Baseline" here alone. + if (e.key === 'baseline') { + const name = (meta.label || meta.tag || meta.note || '').trim() + baseline.push({ ...e, label: name || 'Baseline', kind: 'baseline' }) + continue + } + const tag = (meta.label || meta.tag || '').trim() if (tag) { const g = byTag.get(tag) || { key: `tag:${tag}`, label: tag, kind: 'tag', value: 0, units: 0, count: 0, first: e.logid } @@ -302,7 +309,8 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se const op = meta.operation || e.iter || 'adjustment' loose.push({ ...e, kind: 'entry', count: 1, - label: (meta.note || '').trim() || `${op.charAt(0).toUpperCase()}${op.slice(1)} #${e.logid}`, + label: (meta.label || meta.note || '').trim() + || `${op.charAt(0).toUpperCase()}${op.slice(1)} #${e.logid}`, }) } } @@ -327,6 +335,12 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se // otherwise prints a row of zeros and leaves the reason to be worked out. const nothingToAdjust = hasExcl && !total.value && !total.units + // One line per immovable segment. Falls back to the combined figure for a + // selection whose rows carry no segment name. + const exclLines = (currentTotals?.excluded?.bySegment?.length + ? currentTotals.excluded.bySegment + : (hasExcl ? [{ name: exclName, ...excl }] : [])) + // the basis decides which line the editable rows are measured from const basisOf = (key) => { if (!onTotal) return key === 'price' ? curPrice : total[key] @@ -468,21 +482,21 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se {/* Rows the pivot shows but operations cannot write. Listed so the panel's figures reconcile with what the grid displays. */} - {hasExcl && ( - - + {measures.map(m => ( ))} - )} + ))} {hasExcl && ( diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index d0f2d9b..3618c8f 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -497,7 +497,10 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // rows the pivot shows but operations cannot write (usually 'reference'). // Kept separate rather than filtered away: the grid total includes them, // so the panel has to account for them or the two disagree. - const excluded = { value: 0, units: 0, rows: 0, names: new Set() } + // Per segment, not one lump: YTD Sales and Open Orders are different + // things, and a single "final" line hides which part of the number is + // which. + const excluded = { value: 0, units: 0, rows: 0, names: new Set(), bySegment: new Map() } for (const r of rows) { const k = r.pf_iter || '?' const val = valueCol ? (parseFloat(r[valueCol]) || 0) : 0 @@ -510,7 +513,13 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio // Name them by what they are, not by the iter band that happens to // exclude them: "02 - Prior Year" means something to a forecaster, // "reference" is the mechanism. - if (r.pf_segment) excluded.names.add(String(r.pf_segment)) + const name = String(r.pf_segment || 'excluded') + excluded.names.add(name) + const seg = excluded.bySegment.get(name) || { name, value: 0, units: 0, rows: 0 } + seg.value += val + seg.units += uni + seg.rows += 1 + excluded.bySegment.set(name, seg) continue } @@ -583,11 +592,24 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio units: acc.units + (ps.excluded?.units || 0), rows: acc.rows + (ps.excluded?.rows || 0), names: new Set([...acc.names, ...(ps.excluded?.names || [])]), - }), { value: 0, units: 0, rows: 0, names: new Set() }) + bySegment: (() => { + const m = acc.bySegment + for (const seg of (ps.excluded?.bySegment?.values?.() || [])) { + const t = m.get(seg.name) || { name: seg.name, value: 0, units: 0, rows: 0 } + t.value += seg.value; t.units += seg.units; t.rows += seg.rows + m.set(seg.name, t) + } + return m + })(), + }), { value: 0, units: 0, rows: 0, names: new Set(), bySegment: new Map() }) setCurrentTotals({ byIter, byEntry, total, valueCol, unitsCol, perSlice, - excluded: { ...excluded, names: [...excluded.names].sort() }, + excluded: { + ...excluded, + names: [...excluded.names].sort(), + bySegment: [...excluded.bySegment.values()].sort((a, b) => a.name.localeCompare(b.name)), + }, excludedIters: [...excludeIters], }) } catch { @@ -603,8 +625,9 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio const entries = await fetch(`/api/versions/${vid}/log`).then(r => r.json()) const map = {} for (const e of entries) map[e.id] = { + label: e.label || null, tag: e.tag || null, note: e.note || null, operation: e.operation, - bucket: e.bucket || null, seq: e.seq ?? null, + bucket: e.bucket || null, } setLogMeta(map) } catch { setLogMeta({}) } @@ -1486,7 +1509,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio .map(([id, m]) => ({ id: Number(id), operation: m.operation, - label: (m.tag || m.note || '').trim(), + label: (m.label || m.tag || m.note || '').trim(), })) .sort((a, b) => a.id - b.id) From 28aa7012f1e3722ec60544a3e054a7c22f8b7733 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 18 Sep 2026 09:43:12 -0400 Subject: [PATCH 12/12] Put what cannot move at the top of the ledger The immovable rows sat between Adjustable and Selected total, which made them read as an afterthought to a figure they in fact constrain. They come first now: this much is already booked and billed, this is what is left to work with, and here is how that got to where it is. The walk stays immediately above Adjustable, because it sums to it -- the two are one statement and separating them would leave a column of numbers adding up to nothing on the page. Co-Authored-By: Claude Opus 5 (1M context) --- ui/src/components/OperationPanel.jsx | 40 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/ui/src/components/OperationPanel.jsx b/ui/src/components/OperationPanel.jsx index fcdb223..917bd5e 100644 --- a/ui/src/components/OperationPanel.jsx +++ b/ui/src/components/OperationPanel.jsx @@ -453,6 +453,29 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se + {/* What cannot move comes first: it is the constraint the rest is + worked out against. Then the walk, which sums to Adjustable, and + the two together make the selected total. */} + {exclLines.map(seg => ( + + + {measures.map(m => ( + + ))} + + ))} + + {exclLines.length > 0 && ( + {rule}{measures.map(m => )} + )} + {/* the walk from baseline to current, by initiative */} {lines.map(e => ( @@ -480,23 +503,6 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se ))} - {/* Rows the pivot shows but operations cannot write. Listed so the - panel's figures reconcile with what the grid displays. */} - {exclLines.map(seg => ( - - - {measures.map(m => ( - - ))} - - ))} {hasExcl && (
- {exclName} - + {exclLines.map(seg => ( +
+ {seg.name} + final - {m.key === 'price' ? fmtNum(priceOf(excl), m.dp) : fmtNum(excl[m.key], m.dp)} + {m.key === 'price' ? fmtNum(priceOf(seg), m.dp) : fmtNum(seg[m.key], m.dp)}
+ {seg.name} + + final + + + {m.key === 'price' ? fmtNum(priceOf(seg), m.dp) : fmtNum(seg[m.key], m.dp)} +
- {seg.name} - - final - - - {m.key === 'price' ? fmtNum(priceOf(seg), m.dp) : fmtNum(seg[m.key], m.dp)} -