From cf391286a21f865586bac1b6c672eae4c2eca1d2 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 2 May 2026 22:59:21 -0400 Subject: [PATCH 01/12] Improve theme toggle icons; document light/dark in CLAUDE.md Replace Bootstrap fill icons with Feather-style stroke SVGs (sun with rays + crescent moon) in StatusBar toggle. Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 11 +++++++++++ ui/src/components/StatusBar.jsx | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8283f04..45cc615 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,6 +97,17 @@ All three operations follow the same structure: insert a `pf.log` row in a CTE, --- +## Light / dark mode + +Theme state lives in `ui/src/theme.jsx` — a React context (`ThemeContext`) with a `ThemeProvider` that wraps the app in `main.jsx`. + +- **Storage key:** `pf_dark` in `localStorage`; falls back to `window.matchMedia('(prefers-color-scheme: dark)')` on first visit +- **Toggle:** `setDark(d => !d)` in `StatusBar.jsx`; effect writes `localStorage` and toggles the `.dark` class on `` +- **CSS:** `ui/src/index.css` defines CSS custom properties under `:root` (light) and `.dark`. All Tailwind color overrides are written as `.dark .bg-white { ... }` etc. — no Tailwind dark-mode config needed +- **Palette:** dark mode uses Perspective's "Pro Dark" colours (`--bg-primary: #242526`, panels `#2a2c2f`, gridlines `#3b3f46`, text `#c5c9d0`) +- **Perspective viewer:** `Forecast.jsx` calls `viewer.setAttribute('theme', dark ? 'Pro Dark' : 'Pro Light')` both on initial load and in a `useEffect([dark, versionId])` so the viewer stays in sync when the toggle fires +- **Consuming the theme:** `import useTheme from '../theme.jsx'` then `const { dark, setDark } = useTheme()` + ## Known issues / active work (see todo.md for detail) - Operation panel (Scale/Recode/Clone) wiring to API is a stub — needs completion diff --git a/ui/src/components/StatusBar.jsx b/ui/src/components/StatusBar.jsx index 3da49cf..55784cd 100644 --- a/ui/src/components/StatusBar.jsx +++ b/ui/src/components/StatusBar.jsx @@ -48,12 +48,20 @@ export default function StatusBar({ view, sources = [], sourceId, setSourceId, v title={dark ? 'Switch to light mode' : 'Switch to dark mode'} > {dark ? ( - - + + + + + + + + + + ) : ( - - + + )} From 682968b820c8db2681c61c682ace84d1d4899e4f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sun, 3 May 2026 22:06:17 -0400 Subject: [PATCH 02/12] Fix slice handling: payload preview, date column support, subtotal filter - Add PayloadPreview component showing the exact JSON that will be POSTed, live-updating as form fields change (value_incr shown as computed delta) - buildEffectiveSlice strips expression/system columns and converts Perspective ms-timestamps to ISO date strings for date-role columns - fetchCurrentTotals now includes date columns in Perspective view filter (passing ms number as Perspective expects) so subtotals respect the clicked date - Server buildWhere now receives filterCols (dimensions + date cols) so date values reach the SQL WHERE clause correctly Co-Authored-By: Claude Sonnet 4.6 --- routes/operations.js | 9 ++++-- ui/src/views/Forecast.jsx | 66 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/routes/operations.js b/routes/operations.js index ca4cb6e..ca44ab9 100644 --- a/routes/operations.js +++ b/routes/operations.js @@ -31,6 +31,7 @@ module.exports = function(pool) { ); const colMeta = colResult.rows; const dimCols = colMeta.filter(c => c.role === 'dimension').map(c => c.cname); + const dateCols = colMeta.filter(c => c.role === 'date').map(c => c.cname); const valueCol = colMeta.find(c => c.role === 'value')?.cname; const unitsCol = colMeta.find(c => c.role === 'units')?.cname; @@ -48,6 +49,8 @@ module.exports = function(pool) { table: fcTable(version.tname, version.id), colMeta, dimCols, + dateCols, + filterCols: [...dimCols, ...dateCols], valueCol, unitsCol, sql: sqlResult.rows[0].sql @@ -303,7 +306,7 @@ module.exports = function(pool) { const ctx = await getContext(parseInt(req.params.id), 'scale'); if (!guardOpen(ctx.version, res)) return; - const whereClause = buildWhere(slice, ctx.dimCols); + const whereClause = buildWhere(slice, ctx.filterCols); const excludeClause = buildExcludeClause(ctx.version.exclude_iters); let absValueIncr = value_incr || 0; @@ -361,7 +364,7 @@ module.exports = function(pool) { const ctx = await getContext(parseInt(req.params.id), 'recode'); if (!guardOpen(ctx.version, res)) return; - const whereClause = buildWhere(slice, ctx.dimCols); + const whereClause = buildWhere(slice, ctx.filterCols); const excludeClause = buildExcludeClause(ctx.version.exclude_iters); const setClause = buildSetClause(ctx.dimCols, set); @@ -398,7 +401,7 @@ module.exports = function(pool) { if (!guardOpen(ctx.version, res)) return; const scaleFactor = (scale != null) ? parseFloat(scale) : 1.0; - const whereClause = buildWhere(slice, ctx.dimCols); + const whereClause = buildWhere(slice, ctx.filterCols); const excludeClause = buildExcludeClause(ctx.version.exclude_iters); const setClause = buildSetClause(ctx.dimCols, set); diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 3fffda6..93192ca 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -113,11 +113,15 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou const unitsCol = colMetaRef.current.find(c => c.role === 'units')?.cname if (!valueCol && !unitsCol) return try { - const dimNames = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname)) + const dimNames = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname)) + const dateNames = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname)) const filters = [ ...Object.entries(sliceObj) .filter(([col]) => dimNames.has(col)) .map(([col, val]) => [col, '==', val]), + ...Object.entries(sliceObj) + .filter(([col]) => dateNames.has(col)) + .map(([col, val]) => [col, '==', Number(val)]), ['pf_iter', '!=', 'reference'], ] const view = await tableRef.current.view({ filter: filters }) @@ -385,7 +389,9 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou async function submitOp(op) { if (!Object.keys(slice).length) { flash('Select a slice first', 'error'); return } - let body = { pf_user: 'admin', slice } + const effectiveSlice = buildEffectiveSlice(slice) + if (!Object.keys(effectiveSlice).length) { flash('No dimension or date columns in slice — check col_meta', 'error'); return } + let body = { pf_user: 'admin', slice: effectiveSlice } if (op === 'scale') { let vi = null, ui = null @@ -430,6 +436,50 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou } catch (err) { flash(err.message, 'error') } } + function buildEffectiveSlice(raw) { + const dimCols = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname)) + const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname)) + const out = {} + for (const [k, v] of Object.entries(raw)) { + if (dimCols.has(k)) { out[k] = v; continue } + if (dateCols.has(k)) { + const ms = Number(v) + out[k] = isFinite(ms) ? new Date(ms).toISOString().slice(0, 10) : v + } + } + return out + } + + function buildPayload(op) { + if (!Object.keys(slice).length) return null + const effectiveSlice = buildEffectiveSlice(slice) + let body = { pf_user: 'admin', slice: effectiveSlice } + if (op === 'scale') { + let vi = null, ui = null + if (scaleMode === 'target') { + const curValue = currentTotals?.total?.value + const curUnits = currentTotals?.total?.units + if (scalePrice !== '' && curUnits != null && curValue != null) + vi = (parseFloat(scalePrice) * curUnits) - curValue + if (scaleValue !== '' && curValue != null) + vi = parseFloat(scaleValue) - curValue + if (scaleUnits !== '' && curUnits != null) + ui = parseFloat(scaleUnits) - curUnits + } else { + if (scaleValue !== '') vi = parseFloat(scaleValue) + if (scaleUnits !== '') ui = parseFloat(scaleUnits) + } + body = { ...body, note: scaleNote || undefined, value_incr: vi, units_incr: ui, pct: scaleMode === 'delta' && scalePct } + } else if (op === 'recode') { + const set = Object.fromEntries(Object.entries(recodeSet).filter(([, v]) => v.trim())) + body = { ...body, note: recodeNote || undefined, set } + } else if (op === 'clone') { + const set = Object.fromEntries(Object.entries(cloneSet).filter(([, v]) => v.trim())) + body = { ...body, note: cloneNote || undefined, set, scale: parseFloat(cloneScale) || 1 } + } + return body + } + function flash(text, type = 'ok') { setMsg({ text, type }) setTimeout(() => setMsg(null), 3000) @@ -785,6 +835,7 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou )} + submitOp('scale')}>Apply Scale } @@ -797,6 +848,7 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou ))} + submitOp('recode')}>Apply Recode } @@ -810,6 +862,7 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou ))} + submitOp('clone')}>Apply Clone } @@ -868,3 +921,12 @@ function Submit({ onClick, children }) { ) } + +function PayloadPreview({ payload }) { + if (!payload) return null + return ( + 
+      {JSON.stringify(payload, null, 2)}
+
+ ) +} From c4ba90ae4cd658de0c43ddbe21ddeffceb745c93 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 22 May 2026 23:29:35 -0400 Subject: [PATCH 03/12] note --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 45cc615..9e224dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ Open work: `todo.md` ## Tech stack -- **Backend:** Node.js / Express (`server.js`), runs on port 3010 +- **Backend:** Node.js / Express (`server.js`) - **Database:** PostgreSQL — isolated `pf` schema - **Frontend:** React + Vite + Tailwind CSS in `ui/`; built output lands in `public/app/` - **Pivot:** [Perspective](https://perspective.finos.org/) 4.4.0 loaded from CDN at runtime From 56733df5d488dc878a73f23633436c4cffe14bc6 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 23 May 2026 01:46:16 -0400 Subject: [PATCH 04/12] Add dim_group to col_meta and pf.dim_period calendar table - col_meta: add dim_group field to group related columns (dimension hierarchies, date-adjacent columns); is_key now enabled for date role to mark group parent - sources.js: upsert includes dim_group - Setup.jsx: group column in col_meta editor, key checkbox enabled for date role - gen_dim_period.sql: create and populate pf.dim_period with calendar and fiscal period cuts (monthly grain, 2018-2035) Co-Authored-By: Claude Sonnet 4.6 --- routes/sources.js | 22 +++++---- setup_sql/01_schema.sql | 5 +- setup_sql/gen_dim_period.sql | 94 ++++++++++++++++++++++++++++++++++++ ui/src/views/Setup.jsx | 13 ++++- 4 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 setup_sql/gen_dim_period.sql diff --git a/routes/sources.js b/routes/sources.js index cc27fa7..4347774 100644 --- a/routes/sources.js +++ b/routes/sources.js @@ -89,20 +89,22 @@ module.exports = function(pool) { await client.query('BEGIN'); for (const col of cols) { await client.query(` - INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, opos) - VALUES ($1, $2, $3, $4, $5, $6) + INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, opos) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (source_id, cname) DO UPDATE SET - label = EXCLUDED.label, - role = EXCLUDED.role, - is_key = EXCLUDED.is_key, - opos = EXCLUDED.opos + label = EXCLUDED.label, + role = EXCLUDED.role, + is_key = EXCLUDED.is_key, + dim_group = EXCLUDED.dim_group, + opos = EXCLUDED.opos `, [ sourceId, col.cname, - col.label || null, - col.role || 'ignore', - col.is_key || false, - col.opos || null + col.label || null, + col.role || 'ignore', + col.is_key || false, + col.dim_group || null, + col.opos || null ]); } await client.query('COMMIT'); diff --git a/setup_sql/01_schema.sql b/setup_sql/01_schema.sql index ea9ed8c..dfd677d 100644 --- a/setup_sql/01_schema.sql +++ b/setup_sql/01_schema.sql @@ -15,8 +15,11 @@ CREATE TABLE IF NOT EXISTS pf.source ( UNIQUE (schema, tname) ); --- backfill column for existing installs +-- backfill columns for existing installs ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb; +ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text; + +-- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate CREATE TABLE IF NOT EXISTS pf.col_meta ( id serial PRIMARY KEY, diff --git a/setup_sql/gen_dim_period.sql b/setup_sql/gen_dim_period.sql new file mode 100644 index 0000000..fd8009f --- /dev/null +++ b/setup_sql/gen_dim_period.sql @@ -0,0 +1,94 @@ +-- pf.dim_period — create and populate +-- Adjust fiscal_start_month: 1=Jan (calendar year), 4=Apr, 6=Jun, 7=Jul, 10=Oct, etc. +-- Safe to re-run: ON CONFLICT DO NOTHING, so existing rows are never overwritten. + +CREATE TABLE IF NOT EXISTS pf.dim_period ( + sdat date NOT NULL PRIMARY KEY, + edat date NOT NULL, + drange daterange NOT NULL, + ndays integer NOT NULL, + -- calendar + cal_year integer NOT NULL, + cal_quarter integer NOT NULL, + cal_month integer NOT NULL, + cal_month_abbr text NOT NULL, -- 01 - Jan + cal_month_name text NOT NULL, -- 01 - January + cal_label text NOT NULL, -- 2025-01 Jan + -- fiscal + fisc_year integer NOT NULL, + fisc_quarter integer NOT NULL, + fisc_quarter_label text NOT NULL, -- FY2025 Q3 + fisc_month integer NOT NULL, + fisc_month_abbr text NOT NULL, -- 07 - Jan + fisc_month_name text NOT NULL, -- 07 - January + fisc_label text NOT NULL, -- FY2025 P07 + -- sort key + period_key text NOT NULL -- 2025.07 (ltree-compatible) +); + +CREATE INDEX IF NOT EXISTS dim_period_drange_idx ON pf.dim_period USING gist (drange); +CREATE INDEX IF NOT EXISTS dim_period_fisc_idx ON pf.dim_period (fisc_year, fisc_month); +CREATE INDEX IF NOT EXISTS dim_period_cal_idx ON pf.dim_period (cal_year, cal_month); + +WITH +cfg AS ( + SELECT 6 AS fiscal_start_month -- change to match your fiscal year start month +) +,periods AS ( + SELECT + gs.d::date AS sdat, + (gs.d + '1 month'::interval)::date AS edat, + extract(year FROM gs.d)::int AS cal_year, + extract(month FROM gs.d)::int AS cal_month, + extract(quarter FROM gs.d)::int AS cal_quarter, + ((extract(month FROM gs.d)::int - cfg.fiscal_start_month + 12) % 12) + 1 + AS fisc_month, + extract(year FROM gs.d)::int + + CASE + WHEN cfg.fiscal_start_month > 1 + AND extract(month FROM gs.d)::int >= cfg.fiscal_start_month + THEN 1 ELSE 0 + END AS fisc_year + FROM + generate_series('2018-01-01'::date, '2035-12-01'::date, '1 month') gs(d) + CROSS JOIN cfg +) +INSERT INTO pf.dim_period ( + sdat, edat, drange, ndays, + cal_year, cal_quarter, cal_month, + cal_month_abbr, cal_month_name, cal_label, + fisc_year, fisc_quarter, fisc_quarter_label, + fisc_month, fisc_month_abbr, fisc_month_name, + fisc_label, period_key +) +SELECT + sdat, + edat, + daterange(sdat, edat) AS drange, + edat - sdat AS ndays, + cal_year, + cal_quarter, + cal_month, + to_char(cal_month, 'FM00') || ' - ' || to_char(sdat, 'Mon') AS cal_month_abbr, + to_char(cal_month, 'FM00') || ' - ' || to_char(sdat, 'Month') AS cal_month_name, + to_char(sdat, 'YYYY-MM') || ' ' || to_char(sdat, 'Mon') AS cal_label, + fisc_year, + ceil(fisc_month / 3.0)::int AS fisc_quarter, + 'FY' || fisc_year || ' Q' || ceil(fisc_month / 3.0)::int AS fisc_quarter_label, + fisc_month, + to_char(fisc_month, 'FM00') || ' - ' || to_char(sdat, 'Mon') AS fisc_month_abbr, + to_char(fisc_month, 'FM00') || ' - ' || to_char(sdat, 'Month') AS fisc_month_name, + 'FY' || fisc_year || ' P' || to_char(fisc_month, 'FM00') AS fisc_label, + to_char(fisc_year, 'FM0000') || '.' || to_char(fisc_month, 'FM00') AS period_key +FROM periods +ON CONFLICT (sdat) DO NOTHING; + +-- preview first 24 months +SELECT + period_key, sdat, edat, ndays, + cal_year, cal_quarter, cal_month, cal_month_abbr, + fisc_year, fisc_quarter, fisc_month, fisc_month_abbr, + fisc_quarter_label, fisc_label, cal_label +FROM pf.dim_period +ORDER BY sdat +LIMIT 24; diff --git a/ui/src/views/Setup.jsx b/ui/src/views/Setup.jsx index d267417..edac978 100644 --- a/ui/src/views/Setup.jsx +++ b/ui/src/views/Setup.jsx @@ -281,6 +281,7 @@ export default function Setup({ refreshSources }) { column role key + group label @@ -302,10 +303,20 @@ export default function Setup({ refreshSources }) { type="checkbox" checked={!!col.is_key} onChange={e => updateCol(i, 'is_key', e.target.checked)} - disabled={col.role !== 'dimension'} + disabled={col.role !== 'dimension' && col.role !== 'date'} className="cursor-pointer disabled:opacity-20" /> + + updateCol(i, 'dim_group', e.target.value || null)} + placeholder="—" + disabled={col.role !== 'dimension' && col.role !== 'date'} + className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default" + /> + Date: Sat, 23 May 2026 02:01:19 -0400 Subject: [PATCH 05/12] Add PATCH note route and row_count to change log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PATCH /api/log/:logid — saves note updates to pf.log (was missing, frontend call was silently failing) - GET /api/versions/:id/log — joins fc_table to return row_count per entry so the change log modal shows rows affected instead of '—' Co-Authored-By: Claude Sonnet 4.6 --- routes/log.js | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/routes/log.js b/routes/log.js index eb1c42c..85f84f8 100644 --- a/routes/log.js +++ b/routes/log.js @@ -4,11 +4,31 @@ const { fcTable } = require('../lib/utils'); module.exports = function(pool) { const router = express.Router(); - // list all log entries for a version, newest first + // list all log entries for a version, newest first, with row counts from fc_table router.get('/versions/:id/log', async (req, res) => { try { + const verResult = await pool.query(` + SELECT v.*, s.tname + FROM pf.version v + JOIN pf.source s ON s.id = v.source_id + WHERE v.id = $1 + `, [req.params.id]); + if (verResult.rows.length === 0) return res.status(404).json({ error: 'Version not found' }); + + const { tname, id: version_id } = verResult.rows[0]; + const table = fcTable(tname, version_id); + const result = await pool.query( - `SELECT * FROM pf.log WHERE version_id = $1 ORDER BY stamp DESC`, + `SELECT l.*, + counts.row_count + FROM pf.log l + LEFT JOIN ( + SELECT pf_logid, count(*)::int AS row_count + FROM ${table} + GROUP BY pf_logid + ) counts ON counts.pf_logid = l.id + WHERE l.version_id = $1 + ORDER BY l.stamp DESC`, [req.params.id] ); res.json(result.rows); @@ -18,6 +38,22 @@ module.exports = function(pool) { } }); + // update note on a log entry + router.patch('/log/:logid', async (req, res) => { + const { note } = req.body; + try { + const result = await pool.query( + `UPDATE pf.log SET note = $1 WHERE id = $2 RETURNING *`, + [note ?? null, parseInt(req.params.logid)] + ); + if (result.rows.length === 0) return res.status(404).json({ error: 'Log entry not found' }); + res.json(result.rows[0]); + } catch (err) { + console.error(err); + res.status(500).json({ error: err.message }); + } + }); + // undo an operation — deletes all forecast rows with this logid, then the log entry // two separate queries in a transaction to avoid FK ordering issues router.delete('/log/:logid', async (req, res) => { From cf9bdea9a858f909a9f24bf7367ac9342a12cd74 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 23 May 2026 09:55:31 -0400 Subject: [PATCH 06/12] Add recode/clone dim_group sibling auto-lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - GET /api/sources/:id/lookup?col=X&value=Y — given a key column value, queries the source table for sibling column values in the same dim_group; returns null if no match or ambiguous - Recode and Clone panels: key columns (is_key + dim_group) trigger lookup on blur and auto-fill sibling inputs that the user hasn't already typed into - Row labels now use col_meta label field when set, falling back to cname Co-Authored-By: Claude Sonnet 4.6 --- routes/sources.js | 35 +++++++++++++++++++++++++++++++++++ ui/src/views/Forecast.jsx | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/routes/sources.js b/routes/sources.js index 4347774..5b2352e 100644 --- a/routes/sources.js +++ b/routes/sources.js @@ -224,6 +224,41 @@ module.exports = function(pool) { } }); + // given a key column value, look up sibling dim_group column values from source + // returns { sibling_col: value, ... } if exactly one match, null if none or ambiguous + router.get('/sources/:id/lookup', async (req, res) => { + const { col, value } = req.query; + if (!col || value == null || value === '') return res.json(null); + try { + const [srcResult, metaResult] = await Promise.all([ + pool.query(`SELECT schema, tname FROM pf.source WHERE id = $1`, [req.params.id]), + pool.query(`SELECT * FROM pf.col_meta WHERE source_id = $1 ORDER BY opos`, [req.params.id]) + ]); + if (srcResult.rows.length === 0) return res.status(404).json({ error: 'Source not found' }); + + const keyCol = metaResult.rows.find(c => c.cname === col && c.is_key && c.dim_group); + if (!keyCol) return res.json(null); + + const siblings = metaResult.rows.filter(c => + c.dim_group === keyCol.dim_group && c.cname !== col + ); + if (!siblings.length) return res.json(null); + + const { schema, tname } = srcResult.rows[0]; + const sibCols = siblings.map(c => `"${c.cname}"`).join(', '); + const result = await pool.query( + `SELECT DISTINCT ${sibCols} FROM "${schema}"."${tname}" WHERE "${col}" = $1 LIMIT 2`, + [value] + ); + + if (result.rows.length !== 1) return res.json(null); + res.json(result.rows[0]); + } catch (err) { + console.error(err); + res.status(500).json({ error: err.message }); + } + }); + // set or clear the default Perspective layout for a source. // Body: a Perspective view config (group_by, split_by, columns, plugin_config, …). // Pass null or {} to clear. diff --git a/ui/src/views/Forecast.jsx b/ui/src/views/Forecast.jsx index 93192ca..94e85fd 100644 --- a/ui/src/views/Forecast.jsx +++ b/ui/src/views/Forecast.jsx @@ -436,6 +436,21 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou } catch (err) { flash(err.message, 'error') } } + async function lookupDerivedCols(col, value, setter) { + if (!sourceId || !value.trim()) return + const res = await fetch(`/api/sources/${sourceId}/lookup?col=${encodeURIComponent(col)}&value=${encodeURIComponent(value)}`) + if (!res.ok) return + const derived = await res.json() + if (!derived) return + setter(prev => { + const next = { ...prev } + for (const [k, v] of Object.entries(derived)) { + if (!prev[k] || prev[k] === '') next[k] = String(v ?? '') + } + return next + }) + } + function buildEffectiveSlice(raw) { const dimCols = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname)) const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname)) @@ -842,9 +857,15 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou {activeOp === 'recode' && <>

New values for dimensions to replace. Leave blank to keep.

{dimCols.map(c => ( - - setRecodeSet(s => ({ ...s, [c.cname]: e.target.value }))} - placeholder={slice[c.cname] || '—'} className={`${inp} font-mono`} /> + + setRecodeSet(s => ({ ...s, [c.cname]: e.target.value }))} + onBlur={c.is_key && c.dim_group + ? e => lookupDerivedCols(c.cname, e.target.value, setRecodeSet) + : undefined} + placeholder={slice[c.cname] || '—'} + className={`${inp} font-mono`} /> ))} setRecodeNote(e.target.value)} placeholder="optional" className={inp} /> @@ -855,9 +876,15 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou {activeOp === 'clone' && <>

Override dimensions on cloned rows. Leave blank to keep.

{dimCols.map(c => ( - - setCloneSet(s => ({ ...s, [c.cname]: e.target.value }))} - placeholder={slice[c.cname] || '—'} className={`${inp} font-mono`} /> + + setCloneSet(s => ({ ...s, [c.cname]: e.target.value }))} + onBlur={c.is_key && c.dim_group + ? e => lookupDerivedCols(c.cname, e.target.value, setCloneSet) + : undefined} + placeholder={slice[c.cname] || '—'} + className={`${inp} font-mono`} /> ))} setCloneScale(e.target.value)} className={inp} /> From 16c296d52904bc9b9335d3cf9acd370536774b86 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 23 May 2026 10:06:05 -0400 Subject: [PATCH 07/12] SQL generator: derive date-adjacent columns from pf.dim_period at baseline load - col_meta gets dim_period_col field: maps a dimension column to its pf.dim_period counterpart (e.g. year -> cal_year, month -> cal_month) - When the date column is is_key of a dim_group and any sibling dimension has dim_period_col set, baseline and reference SQL JOIN pf.dim_period on the shifted date instead of copying raw source values - No dim_period config = identical SQL to before (fully backwards compatible) - Setup UI: period col input in col_meta editor, enabled for dimension columns with a dim_group set - Schema migration applied: dim_period_col text null on pf.col_meta Co-Authored-By: Claude Sonnet 4.6 --- lib/sql_generator.js | 48 +++++++++++++++++++++++++++++++---------- routes/sources.js | 26 +++++++++++----------- setup_sql/01_schema.sql | 1 + ui/src/views/Setup.jsx | 11 ++++++++++ 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 01c485c..7794cbc 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -25,7 +25,7 @@ function generateSQL(source, colMeta) { if (!dateCol) throw new Error('No date column defined in col_meta'); if (dims.length === 0) throw new Error('No dimension columns defined in col_meta'); - const srcTable = `${source.schema}.${source.tname}`; + const srcTable = `"${source.schema}"."${source.tname}"`; const dataCols = [...dims, dateCol, valueCol, unitsCol].filter(Boolean); const effectiveValue = dataCols.includes(valueCol) ? valueCol : null; const effectiveUnits = dataCols.includes(unitsCol) ? unitsCol : null; @@ -33,6 +33,19 @@ function generateSQL(source, colMeta) { const selectData = dataCols.map(q).join(', '); const dimsJoined = dims.map(q).join(', '); + // dim_period JOIN support: if the date column is the is_key of a dim_group, + // dimension siblings with dim_period_col set are derived from pf.dim_period + // instead of being copied raw from the source on baseline/reference load. + const dateKeyGroup = colMeta.find(c => c.role === 'date' && c.is_key && c.dim_group)?.dim_group; + const dimPeriodMap = new Map( + dateKeyGroup + ? colMeta + .filter(c => c.role === 'dimension' && c.dim_group === dateKeyGroup && c.dim_period_col) + .map(c => [c.cname, c.dim_period_col]) + : [] + ); + const hasDimPeriod = dimPeriodMap.size > 0; + return { get_data: buildGetData(), baseline: buildBaseline(), @@ -47,10 +60,22 @@ function generateSQL(source, colMeta) { return `SELECT * FROM {{fc_table}}`; } + function buildLoadSelect(pfx) { + // pfx: table alias prefix ('s.' when joining dim_period, '' otherwise) + return dataCols.map(c => { + if (c === dateCol) return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`; + if (dimPeriodMap.has(c)) return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`; + return `${pfx}${q(c)}`; + }).join(',\n '); + } + + function buildFromClause() { + if (!hasDimPeriod) return srcTable; + return `${srcTable} s\n JOIN pf.dim_period dp` + + ` ON dp.drange @> (s.${q(dateCol)} + '{{date_offset}}'::interval)::date`; + } + function buildBaseline() { - const baselineSelect = dataCols.map(c => - c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c) - ).join(', '); return ` WITH ilog AS ( @@ -60,8 +85,10 @@ ilog AS ( ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${baselineSelect}, 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() - FROM ${srcTable} + SELECT + ${buildLoadSelect(hasDimPeriod ? 's.' : '')}, + 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM ${buildFromClause()} WHERE {{filter_clause}} RETURNING * ) @@ -69,9 +96,6 @@ SELECT count(*) AS rows_affected FROM ins`.trim(); } function buildReference() { - const referenceSelect = dataCols.map(c => - c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c) - ).join(', '); return ` WITH ilog AS ( @@ -81,8 +105,10 @@ ilog AS ( ) ,ins AS ( INSERT INTO {{fc_table}} (${insertCols}) - SELECT ${referenceSelect}, 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() - FROM ${srcTable} + SELECT + ${buildLoadSelect(hasDimPeriod ? 's.' : '')}, + 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM ${buildFromClause()} WHERE {{filter_clause}} RETURNING * ) diff --git a/routes/sources.js b/routes/sources.js index 5b2352e..d1c1157 100644 --- a/routes/sources.js +++ b/routes/sources.js @@ -89,22 +89,24 @@ module.exports = function(pool) { await client.query('BEGIN'); for (const col of cols) { await client.query(` - INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, opos) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, dim_period_col, opos) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (source_id, cname) DO UPDATE SET - label = EXCLUDED.label, - role = EXCLUDED.role, - is_key = EXCLUDED.is_key, - dim_group = EXCLUDED.dim_group, - opos = EXCLUDED.opos + label = EXCLUDED.label, + role = EXCLUDED.role, + is_key = EXCLUDED.is_key, + dim_group = EXCLUDED.dim_group, + dim_period_col = EXCLUDED.dim_period_col, + opos = EXCLUDED.opos `, [ sourceId, col.cname, - col.label || null, - col.role || 'ignore', - col.is_key || false, - col.dim_group || null, - col.opos || null + col.label || null, + col.role || 'ignore', + col.is_key || false, + col.dim_group || null, + col.dim_period_col || null, + col.opos || null ]); } await client.query('COMMIT'); diff --git a/setup_sql/01_schema.sql b/setup_sql/01_schema.sql index dfd677d..b555002 100644 --- a/setup_sql/01_schema.sql +++ b/setup_sql/01_schema.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS pf.source ( -- backfill columns for existing installs ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb; ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text; +ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text; -- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate diff --git a/ui/src/views/Setup.jsx b/ui/src/views/Setup.jsx index edac978..a9999b4 100644 --- a/ui/src/views/Setup.jsx +++ b/ui/src/views/Setup.jsx @@ -282,6 +282,7 @@ export default function Setup({ refreshSources }) { role key group + period col label @@ -317,6 +318,16 @@ export default function Setup({ refreshSources }) { className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default" /> + + updateCol(i, 'dim_period_col', e.target.value || null)} + placeholder="—" + disabled={col.role !== 'dimension' || !col.dim_group} + className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default font-mono text-xs" + /> + Date: Fri, 12 Jun 2026 23:27:05 -0400 Subject: [PATCH 08/12] Update CLAUDE.md and spec: units optional, dim_group/dim_period, delete todo.md - units role is now optional; spec and CLAUDE.md reflect conditionality in SQL patterns - pf.col_meta gains dim_group and dim_period_col fields (documented in both files) - pf.dim_period calendar table added to schema docs - pf.source default_layout column added to spec DDL - Forecast table metadata columns corrected to pf_iter/pf_logid/pf_created_at throughout spec - SQL patterns updated with correct CTE structure and RETURNING * to match generated code - Project status updated to 2026-06-12; stale Arrow IPC open question removed - todo.md deleted; open items retained in CLAUDE.md known issues Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 11 ++- pf_spec.md | 252 +++++++++++++++++++++++++++++++---------------------- todo.md | 113 ------------------------ 3 files changed, 153 insertions(+), 223 deletions(-) delete mode 100644 todo.md diff --git a/CLAUDE.md b/CLAUDE.md index 9e224dc..65ae2e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,6 @@ A web app for building named forecast scenarios against any PostgreSQL table. Th Full spec: `pf_spec.md` Data transport architecture options: `pf_perspective_options.md` UX mockup: `pf_ux_mockup.md` -Open work: `todo.md` --- @@ -51,11 +50,12 @@ ui/src/ ## Database schema (`pf`) - **`pf.source`** — registered source tables -- **`pf.col_meta`** — column roles: `dimension` | `value` | `units` | `date` | `filter` | `ignore`; `is_key` marks dimensions used in slice WHERE clauses +- **`pf.col_meta`** — column roles: `dimension` | `value` | `units` | `date` | `filter` | `ignore`; `is_key` marks dimensions used in slice WHERE clauses; `dim_group` groups functionally dependent columns (e.g. date + its derived year/month dimensions); `dim_period_col` maps a dimension to a `pf.dim_period` column so date-adjacent values are derived at load time rather than copied raw - **`pf.version`** — named forecast scenarios; `exclude_iters` (default `["reference"]`) blocks those iter values from all operations -- **`pf.fc_{tname}_{version_id}`** — one forecast table per version; contains both operational rows (`iter = baseline|scale|recode|clone`) and reference rows (`iter = reference`) +- **`pf.fc_{tname}_{version_id}`** — one forecast table per version; contains both operational rows (`pf_iter = baseline|scale|recode|clone`) and reference rows (`pf_iter = reference`) - **`pf.log`** — audit log; every write gets one entry; `slice` + `params` stored as jsonb - **`pf.sql`** — generated SQL templates per source/operation; tokens substituted at request time +- **`pf.dim_period`** — calendar lookup table (2018–2035); one row per month keyed on `sdat` (month start date); provides cal/fiscal year, quarter, and month columns; populated by `setup_sql/gen_dim_period.sql` with a configurable fiscal year start month ### Key token substitution tokens `{{fc_table}}`, `{{where_clause}}`, `{{exclude_clause}}`, `{{logid}}`, `{{pf_user}}`, `{{value_incr}}`, `{{units_incr}}`, `{{pct}}`, `{{set_clause}}`, `{{scale_factor}}`, `{{date_offset}}`, `{{filter_clause}}` @@ -108,10 +108,9 @@ Theme state lives in `ui/src/theme.jsx` — a React context (`ThemeContext`) wit - **Perspective viewer:** `Forecast.jsx` calls `viewer.setAttribute('theme', dark ? 'Pro Dark' : 'Pro Light')` both on initial load and in a `useEffect([dark, versionId])` so the viewer stays in sync when the toggle fires - **Consuming the theme:** `import useTheme from '../theme.jsx'` then `const { dark, setDark } = useTheme()` -## Known issues / active work (see todo.md for detail) +## Known issues / active work -- Operation panel (Scale/Recode/Clone) wiring to API is a stub — needs completion -- Status bar is hardcoded — needs to reflect actual selected source/version +- Operation panel (Scale/Recode/Clone) SQL generation and dim_period JOIN are complete; UI wiring to API still needs completion - Load progress bar is jittery — needs throttle (~10 updates/sec) - Default pivot layout should be configurable per source (currently hardcodes first 2 dimensions) - Source/version selection doesn't persist across page reload diff --git a/pf_spec.md b/pf_spec.md index 8eeda72..b8c85cd 100644 --- a/pf_spec.md +++ b/pf_spec.md @@ -24,13 +24,14 @@ Registered source tables available for forecasting. ```sql CREATE TABLE pf.source ( - id serial PRIMARY KEY, - schema text NOT NULL, - tname text NOT NULL, - label text, -- friendly display name - status text DEFAULT 'active', -- active | archived - created_at timestamptz DEFAULT now(), - created_by text, + id serial PRIMARY KEY, + schema text NOT NULL, + tname text NOT NULL, + label text, -- friendly display name + status text DEFAULT 'active', -- active | archived + default_layout jsonb, -- Perspective view config used as per-source default + created_at timestamptz DEFAULT now(), + created_by text, UNIQUE (schema, tname) ); ``` @@ -40,25 +41,40 @@ Column configuration for each registered source table. Determines how the app tr ```sql CREATE TABLE pf.col_meta ( - id serial PRIMARY KEY, - source_id integer REFERENCES pf.source(id), - cname text NOT NULL, -- column name in source table - label text, -- friendly display name - role text NOT NULL, -- 'dimension' | 'value' | 'units' | 'date' | 'ignore' - is_key boolean DEFAULT false, -- true = part of natural key (used in WHERE slice) - opos integer, -- ordinal position (for ordering) + id serial PRIMARY KEY, + source_id integer REFERENCES pf.source(id), + cname text NOT NULL, -- column name in source table + label text, -- friendly display name + role text NOT NULL, -- 'dimension' | 'value' | 'units' | 'date' | 'filter' | 'ignore' + is_key boolean DEFAULT false, -- true = part of natural key (used in WHERE slice) + opos integer, -- ordinal position (for ordering) + dim_group text, -- groups functionally dependent columns (see below) + dim_period_col text, -- maps this dimension to a pf.dim_period column UNIQUE (source_id, cname) ); ``` **Roles:** - `dimension` — categorical field (customer, part, channel, rep, geography, etc.) — appears as pivot rows/cols, used in WHERE filters for operations -- `value` — the money/revenue field to scale -- `units` — the quantity field to scale -- `date` — the primary date field; used for baseline/reference date range and stored in the forecast table +- `value` — the money/revenue field to scale (**required** — SQL generation fails without it) +- `units` — the quantity field to scale (**optional** — if absent, units columns are omitted from the forecast table and all SQL patterns) +- `date` — the primary date field; used for baseline/reference date range and stored in the forecast table (**required**) - `filter` — columns available as filter conditions in the Baseline Workbench (e.g. order status, ship date, open flag); used in baseline WHERE clauses but **not stored** in the forecast table - `ignore` — exclude from forecast table entirely +**`dim_group`** — a free-text group name linking a `date` column to its derived dimension siblings. When the `date` column has `is_key = true` and a `dim_group` value, the SQL generator looks for `dimension` columns in the same group that also have a `dim_period_col` value. Those columns are sourced from `pf.dim_period` on baseline/reference load (via a JOIN on `drange @> date`) rather than copied raw from the source table. This allows fiscal year, quarter, and month columns to be stored in the forecast table with calendar-correct values even if those columns don't exist in the source. + +**`dim_period_col`** — names the column in `pf.dim_period` to use as the value for this dimension on load. Only meaningful when the column is in a `dim_group` whose `date` key has `is_key = true`. Example: `cal_year`, `fisc_quarter`, `fisc_label`. + +### `pf.dim_period` +Calendar lookup table. One row per month from 2018-01-01 through 2035-12-01. Keyed on `sdat` (month start date). Used to derive fiscal/calendar period columns at baseline load time when `dim_group` / `dim_period_col` are configured on col_meta. + +Populated by `setup_sql/gen_dim_period.sql` (safe to re-run; `ON CONFLICT DO NOTHING`). Fiscal year start month is configurable at the top of that script (default: June, i.e. fiscal month 1 = June). + +Key columns: `sdat`, `edat`, `drange` (GiST-indexed daterange), `cal_year`, `cal_quarter`, `cal_month`, `cal_month_abbr`, `cal_month_name`, `cal_label`, `fisc_year`, `fisc_quarter`, `fisc_quarter_label`, `fisc_month`, `fisc_month_abbr`, `fisc_month_name`, `fisc_label`, `period_key`. + +The baseline/reference SQL JOINs this table when `hasDimPeriod` is true: `JOIN pf.dim_period dp ON dp.drange @> (s.{date_col} + '{{date_offset}}'::interval)::date`. + ### `pf.version` Named forecast scenarios. One forecast table (`pf.fc_{tname}_{version_id}`) is created per version. @@ -102,31 +118,31 @@ CREATE TABLE pf.log ( ``` ### `pf.fc_{tname}_{version_id}` (dynamic, one per version) -Created when a version is created. Mirrors source table dimension/value/units/date columns plus forecast metadata. Contains both operational rows (`iter = 'baseline' | 'scale' | 'recode' | 'clone'`) and reference rows (`iter = 'reference'`). +Created when a version is created. Mirrors source table dimension/value/date columns (and units if configured) plus any `dim_period_col`-derived dimension columns, plus forecast metadata. Contains both operational rows (`pf_iter = 'baseline' | 'scale' | 'recode' | 'clone'`) and reference rows (`pf_iter = 'reference'`). ```sql -- Example: source table "sales", version id 3 → pf.fc_sales_3 CREATE TABLE pf.fc_sales_3 ( - id bigserial PRIMARY KEY, + id bigserial PRIMARY KEY, -- mirrored from source (role = dimension | value | units | date only): - customer text, - channel text, - part text, - geography text, - order_date date, - units numeric, - value numeric, + customer text, + channel text, + part text, + geography text, + order_date date, + value numeric, + units numeric, -- omitted if no 'units' role in col_meta -- forecast metadata: - iter text, -- 'baseline' | 'reference' | 'scale' | 'recode' | 'clone' - logid bigint REFERENCES pf.log(id), - pf_user text, - created_at timestamptz DEFAULT now() + pf_iter text, -- 'baseline' | 'reference' | 'scale' | 'recode' | 'clone' + pf_logid bigint REFERENCES pf.log(id), + pf_user text, + pf_created_at timestamptz DEFAULT now() ); ``` -Note: no `version_id` column on the forecast table — it's implied by the table itself. +Note: no `version_id` column on the forecast table — it's implied by the table itself. The `units` column is only present when a column with `role = 'units'` exists in col_meta. ### `pf.sql` Generated SQL stored per source and operation. Built once when col_meta is finalized, fetched at request time. @@ -148,7 +164,7 @@ CREATE TABLE pf.sql ( |-------|--------------| | `{{fc_table}}` | `pf.fc_{tname}_{version_id}` — derived at request time | | `{{where_clause}}` | built from `slice` JSON by `build_where()` in JS | -| `{{exclude_clause}}` | built from `version.exclude_iters` — e.g. `AND iter NOT IN ('reference')` | +| `{{exclude_clause}}` | built from `version.exclude_iters` — e.g. `AND pf_iter NOT IN ('reference')` | | `{{logid}}` | newly inserted `pf.log` id | | `{{pf_user}}` | from request body | | `{{date_from}}` / `{{date_to}}` | baseline/reference date range (source period) | @@ -521,7 +537,11 @@ AG Grid list of log entries — user, timestamp, operation, slice, note, rows af ## Forecast SQL Patterns -Column names baked in at generation time. Tokens substituted at request time. +Column names baked in at generation time. Tokens substituted at request time. Metadata columns are `pf_iter`, `pf_logid`, `pf_user`, `pf_created_at`. + +**Units conditionality:** `{units_col}` appears in INSERT column lists and SELECT expressions only when a `units` role is configured in col_meta. The SQL generator omits it entirely otherwise — no placeholder column, no zero-fill. + +**dim_period JOIN:** when any `dimension` column has `dim_period_col` set (and its group's `date` key has `is_key = true`), the FROM clause becomes `{schema}.{tname} s JOIN pf.dim_period dp ON dp.drange @> (s.{date_col} + '{{date_offset}}'::interval)::date`. Those dimension columns are selected as `dp.{dim_period_col} AS {col}` instead of `s.{col}`. ### Baseline Load (one segment) @@ -531,18 +551,23 @@ WITH ilog AS ( VALUES ({{version_id}}, '{{pf_user}}', 'baseline', NULL, '{{params}}'::jsonb, '{{note}}') RETURNING id ) -INSERT INTO {{fc_table}} ( - {dimension_cols}, {value_col}, {units_col}, {date_col}, - iter, logid, pf_user, created_at +,ins AS ( + INSERT INTO {{fc_table}} ( + {dimension_cols}, {date_col}, {value_col} [, {units_col}], + pf_iter, pf_logid, pf_user, pf_created_at + ) + SELECT + {dimension_cols}, + ({date_col} + '{{date_offset}}'::interval)::date, + {value_col} [, {units_col}], + 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM + {schema}.{tname} -- or with dim_period JOIN (see above) + WHERE + {{filter_clause}} + RETURNING * ) -SELECT - {dimension_cols}, {value_col}, {units_col}, - ({date_col} + '{{date_offset}}'::interval)::date, - 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now() -FROM - {schema}.{tname} -WHERE - {{filter_clause}} +SELECT count(*) AS rows_affected FROM ins ``` Baseline loads are **additive** — no DELETE before INSERT. Each segment appends independently. @@ -557,7 +582,7 @@ Token details: Two queries, run in a transaction: ```sql -DELETE FROM {{fc_table}} WHERE iter = 'baseline'; +DELETE FROM {{fc_table}} WHERE pf_iter = 'baseline'; DELETE FROM pf.log WHERE version_id = {{version_id}} AND operation = 'baseline'; ``` @@ -569,20 +594,24 @@ WITH ilog AS ( VALUES ({{version_id}}, '{{pf_user}}', 'reference', NULL, '{{params}}'::jsonb, '{{note}}') RETURNING id ) -INSERT INTO {{fc_table}} ( - {dimension_cols}, {value_col}, {units_col}, {date_col}, - iter, logid, pf_user, created_at +,ins AS ( + INSERT INTO {{fc_table}} ( + {dimension_cols}, {date_col}, {value_col} [, {units_col}], + pf_iter, pf_logid, pf_user, pf_created_at + ) + SELECT + {dimension_cols}, {date_col}, {value_col} [, {units_col}], + 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM + {schema}.{tname} -- or with dim_period JOIN (see above) + WHERE + {{filter_clause}} + RETURNING * ) -SELECT - {dimension_cols}, {value_col}, {units_col}, {date_col}, - 'reference', (SELECT id FROM ilog), '{{pf_user}}', now() -FROM - {schema}.{tname} -WHERE - {date_col} BETWEEN '{{date_from}}' AND '{{date_to}}' +SELECT count(*) AS rows_affected FROM ins ``` -No date offset — reference rows land at their original dates for prior-period comparison. +No date offset applied — reference rows land at their original dates for prior-period comparison. Same dim_period JOIN logic applies as baseline. ### Scale @@ -595,26 +624,30 @@ WITH ilog AS ( ,base AS ( SELECT {dimension_cols}, {date_col}, - {value_col}, {units_col}, - sum({value_col}) OVER () AS total_value, - sum({units_col}) OVER () AS total_units + {value_col} [, {units_col}], + sum({value_col}) OVER () AS total_value + [, sum({units_col}) OVER () AS total_units] FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} ) -INSERT INTO {{fc_table}} ( - {dimension_cols}, {date_col}, {value_col}, {units_col}, - iter, logid, pf_user, created_at +,ins AS ( + INSERT INTO {{fc_table}} ( + {dimension_cols}, {date_col}, {value_col} [, {units_col}], + pf_iter, pf_logid, pf_user, pf_created_at + ) + SELECT + {dimension_cols}, {date_col}, + round(({value_col} / NULLIF(total_value, 0)) * {{value_incr}}, 2) + [, round(({units_col} / NULLIF(total_units, 0)) * {{units_incr}}, 5)], + 'scale', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM base + RETURNING * ) -SELECT - {dimension_cols}, {date_col}, - round(({value_col} / NULLIF(total_value, 0)) * {{value_incr}}, 2), - round(({units_col} / NULLIF(total_units, 0)) * {{units_incr}}, 5), - 'scale', (SELECT id FROM ilog), '{{pf_user}}', now() -FROM base +SELECT * FROM ins ``` -`{{value_incr}}` / `{{units_incr}}` are pre-computed in JS when `pct: true` (multiply slice total by pct). +`{{value_incr}}` / `{{units_incr}}` are pre-computed in JS when `pct: true` (multiply slice total by pct). Units expressions are omitted when no units column is configured. ### Recode @@ -625,22 +658,27 @@ WITH ilog AS ( RETURNING id ) ,src AS ( - SELECT {dimension_cols}, {date_col}, {value_col}, {units_col} + SELECT {dimension_cols}, {date_col}, {value_col} [, {units_col}] FROM {{fc_table}} WHERE {{where_clause}} {{exclude_clause}} ) -,negatives AS ( - INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at) - SELECT {dimension_cols}, {date_col}, -{value_col}, -{units_col}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() +,neg AS ( + INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at) + SELECT {dimension_cols}, {date_col}, -{value_col} [, -{units_col}], 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() FROM src + RETURNING * ) -INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at) -SELECT {{set_clause}}, {date_col}, {value_col}, {units_col}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() -FROM src +,ins AS ( + INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at) + SELECT {{set_clause}}, {date_col}, {value_col} [, {units_col}], 'recode', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM src + RETURNING * +) +SELECT * FROM neg UNION ALL SELECT * FROM ins ``` -`{{set_clause}}` replaces the listed dimension columns with new values, passes others through unchanged. +`{{set_clause}}` replaces the listed dimension columns with new values, passes others through unchanged. Both the negative (zero-out) and positive (replacement) rows share the same `pf_logid` and are undone together. ### Clone @@ -650,21 +688,27 @@ WITH ilog AS ( VALUES ({{version_id}}, '{{pf_user}}', 'clone', '{{slice}}'::jsonb, '{{params}}'::jsonb, '{{note}}') RETURNING id ) -INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at) -SELECT - {{set_clause}}, {date_col}, - round({value_col} * {{scale_factor}}, 2), - round({units_col} * {{scale_factor}}, 5), - 'clone', (SELECT id FROM ilog), '{{pf_user}}', now() -FROM {{fc_table}} -WHERE {{where_clause}} -{{exclude_clause}} +,ins AS ( + INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at) + SELECT + {{set_clause}}, {date_col}, + round({value_col} * {{scale_factor}}, 2) + [, round({units_col} * {{scale_factor}}, 5)], + 'clone', (SELECT id FROM ilog), '{{pf_user}}', now() + FROM {{fc_table}} + WHERE {{where_clause}} + {{exclude_clause}} + RETURNING * +) +SELECT * FROM ins ``` ### Undo +Two queries run sequentially (not in a CTE — FK ordering): + ```sql -DELETE FROM {{fc_table}} WHERE logid = {{logid}}; +DELETE FROM {{fc_table}} WHERE pf_logid = {{logid}}; DELETE FROM pf.log WHERE id = {{logid}}; ``` @@ -698,38 +742,38 @@ DELETE FROM pf.log WHERE id = {{logid}}; ## Open Questions / Future Scope - **Baseline replay** — re-execute change log against a restated baseline (`replay: true`); v1 returns 501 -- **Arrow IPC for initial data load** — at large row counts (1M+) the `/versions/:id/data` JSON response becomes a bottleneck. Option: serve Arrow IPC binary instead of JSON; Perspective's `worker.table()` accepts Arrow buffers natively. Incremental operation rows (scale/recode/clone) can stay as JSON fed to `table.update()` since they're always small. Could be implemented with `pg` + `apache-arrow` in Node, or by adding a server-side DuckDB instance (Postgres scanner → Arrow IPC) if a caching layer is also needed. - **Approval workflow** — user submits, admin approves before changes are visible to others (deferred) - **Territory filtering** — restrict what a user can see/edit by dimension value (deferred) - **Export** — download forecast as CSV or push results to a reporting table - **Version comparison** — side-by-side view of two versions (facilitated by isolated tables via UNION) -- **Col meta / version schema drift** — if col_meta roles are changed after a version's forecast table is already created, the generated SQL and the table DDL go out of sync (e.g. a column added to SQL that doesn't exist in the table). UI should detect this: compare col_meta against the forecast table's actual columns via `information_schema`, warn the user, and offer to rebuild the version (drop + recreate table, preserving the version record and log). For now the workaround is to delete and recreate the version manually. +- **Col meta / version schema drift** — if col_meta roles are changed after a version's forecast table is already created, the generated SQL and the table DDL go out of sync. UI should detect this: compare col_meta against the forecast table's actual columns via `information_schema`, warn the user, and offer to rebuild the version (drop + recreate table, preserving the version record and log). Workaround: delete and recreate the version manually. - **Multi-connection support** — currently one DB via `.env`. Full vision: `pf.connection` table (host, port, dbname, user, password as env-var ref), `connection_id` on `pf.source`, per-connection pg pools at runtime. `pf` schema stays on a "home" connection; source data can live anywhere. Connections UI in Setup. Safe to defer while in dev — requires clean reinstall when added since it changes the source schema. --- -## Project Status — 2026-04-25 +## Project Status — 2026-06-12 ### What's working - Full backend: source registration, col_meta, SQL generation, versions, baseline segments, reference load, scale, recode, clone, undo -- React + Vite + Tailwind CSS frontend scaffolded in `ui/`, built output to `public/app/`, served by Express -- 3-step collapsible sidebar (Setup / Baseline / Forecast) — addresses prior UX concern about opaque 5-tab nav -- Setup view: DB table browser with preview modal, source registration, col_meta editor, SQL generation +- `units` column is optional — sources without a units column register and generate SQL correctly +- `dim_group` / `dim_period_col` on col_meta: baseline/reference load JOINs `pf.dim_period` to derive fiscal/calendar period columns rather than copying them raw from the source +- `pf.dim_period` calendar table (2018–2035): populated by `setup_sql/gen_dim_period.sql`, configurable fiscal year start +- React + Vite + Tailwind CSS frontend in `ui/`, built output to `public/app/`, served by Express +- Data transport: Arrow IPC binary stream (`GET /api/versions/:id/data`); server accumulates all rows into one record batch; client hands buffer directly to Perspective WASM +- 3-step collapsible sidebar (Setup / Baseline / Forecast) +- Setup view: DB table browser with preview modal, source registration, col_meta editor (`dim_group`/`dim_period_col` fields included), SQL generation - Baseline view: version management (create/close/reopen/delete), multi-segment baseline workbench, canvas timeline, filter builder -- Perspective pivot in Forecast view: loads all version rows, interactive group/split/filter/chart, layout saved per version +- Perspective pivot in Forecast view: loads all version rows, interactive group/split/filter/chart, layout saved per version to localStorage - Slice extraction from `perspective-click` event feeds operation panel directly -- Incremental row streaming: operation results (`RETURNING *`) stream into Perspective table without full reload +- Incremental row streaming: operation results (`RETURNING *`) applied to Perspective table via `pspTable.update()` — no full reload - Status bar: shows current source · version · baseline row count · status ### Known issues / next focus -- **Forecast view** — operation panel (Scale / Recode / Clone) is a stub; needs wiring to API -- **Status bar** — currently hardcoded; needs to reflect actual selected source/version from state -- **Col_meta / version schema drift** — if col_meta changes after a version's forecast table is created, the SQL and table DDL go out of sync. UI should detect this (compare col_meta against `information_schema`), warn, and offer rebuild. Workaround: delete and recreate the version. -- **No "current version" persistence** — source/version selection resets on page reload; session context not persisted -- **Perspective slice limitation** — computed date columns (Month, YearDate) extracted via split_by don't filter back to raw rows; only native dimension columns work for slice extraction - -### Branch status -- `baseline-workbench` — merged to origin, stable -- `perspective-forecast` — active development branch; React UI scaffolded, Forecast operation panel pending +- **Forecast view** — operation panel SQL generation complete; UI wiring to API still needed +- **Load progress bar** — jittery at high throughput; throttle to ~10 updates/sec +- **Default pivot layout** — per-source configurable layout not yet implemented; currently hardcodes first 2 dimensions +- **No "current version" persistence** — source/version selection resets on page reload +- **Perspective slice limitation** — computed date columns (Month, YearDate) from split_by don't map back to raw rows; only native dimension columns work for slice extraction +- **Col_meta / version schema drift** — if col_meta changes after a version's forecast table is created, SQL and DDL go out of sync. Workaround: delete and recreate the version. diff --git a/todo.md b/todo.md deleted file mode 100644 index 42d3579..0000000 --- a/todo.md +++ /dev/null @@ -1,113 +0,0 @@ -- [ ] when you enter the forecast, be able to enter in a context so you dont have to open the whole thing (should show in status bar and be a filter for SQL and spi calls) - - -- [x] should be able to edit and revise forecase segments that constitute baseline or reference. if you edit, maybe a warning that your forecast values wont mean a lot, and have an option to delete them. - - Notes: A baseline/reference segment is a `pf.log` row plus the - forecast rows it produced (joined by pf_logid). Editing has the - shape of a delete-then-replay: drop the rows by pf_logid, drop the - log entry, re-run the segment with the new params (offset, filter, - iter type), insert the new log entry. New endpoint: - `PUT /versions/:id/baseline/:logid` (and the same for reference). - UI: an Edit button on each segment in Baseline view, populating the - form with the original `params`. - - Cascade warning: if any scale/recode/clone log entries exist *after* - this segment was added, those operations were calibrated against - the old totals and will no longer reconcile cleanly. Show a banner - like "3 forecast operations applied after this segment may be - invalidated. View / Delete / Continue." Probably want a CASCADE - option that deletes downstream forecast entries too, plus a plain - "edit only" option for the user who knows what they're doing. - - Implementation order: API + cascade detection first (compare - pf.log.stamp ordering); UI second. - -- [~] be able to copy an existing forecast and it's segments to adjust some parameters without having to start from scrath. - - Notes: A version is the unit of copy. Need a `POST /versions/:id/copy` - endpoint that creates a new pf.version row with the same source/ - col_meta, creates the new fc__ table via the same DDL - path, and replays each pf.log entry's INSERT against the new table - (preserving stamp ordering). Each log entry gets re-inserted - pointing at the new version_id; the new pf_logid feeds the row - inserts. Notes/users come along. - - UI: "Copy" button next to each version in Baseline. Copy modal - asks for a new name and optional description, then runs the API - call (likely 5–30s for a 350k-row version since every segment is - re-evaluated). Show progress. - - Two design questions worth deciding up front: - - Copy as-of-now (re-fetch source data, so freshly-arrived rows - show up in baseline)? Or freeze (replay from existing forecast - rows, i.e. clone the forecast table directly)? Different - semantics, different SQL — pick one before building. - - Should the copy track its origin? A `parent_version_id` column - on pf.version makes "show me variants of FY2026 Plan" easy. - -- [x] need the list of filters to have an and/or specification - - Notes: Spec already covers this in `pf_spec.md:245` — `filters` is - an array of groups; conditions within a group are AND-ed, groups - OR-ed. Backend has `buildFilterClause` in - `lib/sql_generator.js:247` but it's not wired into the routes - (baseline currently takes raw `where_clause`). Wiring + UI is the - remaining work. - - UI: each group is a card with a header ("Group 1", "Group 2 — OR"), - rows of `column / operator / values`, a `+ Add condition` link, - and a `+ Add OR group` button at the bottom. The Baseline view - already has a single-group filter builder; extend it to wrap the - current rows in a group container and allow adding more groups. - -- [x] the filters should have the option to just write the WHERE clause SQL - - Notes: Spec covers this too (`pf_spec.md:251`, `:454`) as the - `raw_where` admin-only escape hatch. The current baseline endpoint - *already* takes `where_clause` as a raw string — so the API is - effectively in "raw only" mode today; it's the structured side - that's missing. Two things to add: - - - Once structured `filters` is wired in, gate `raw_where` behind - an admin check (`pf_user` in admin list — needs admin list - config) and reject 400 if both are sent. - - UI toggle: a "Switch to manual SQL" link in the Baseline filter - builder swaps the structured rows for a `