Update CLAUDE.md and spec: units optional, dim_group/dim_period, delete todo.md

- units role is now optional; spec and CLAUDE.md reflect conditionality in SQL patterns
- pf.col_meta gains dim_group and dim_period_col fields (documented in both files)
- pf.dim_period calendar table added to schema docs
- pf.source default_layout column added to spec DDL
- Forecast table metadata columns corrected to pf_iter/pf_logid/pf_created_at throughout spec
- SQL patterns updated with correct CTE structure and RETURNING * to match generated code
- Project status updated to 2026-06-12; stale Arrow IPC open question removed
- todo.md deleted; open items retained in CLAUDE.md known issues

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

CLAUDE.md

@@ -7,7 +7,6 @@ A web app for building named forecast scenarios against any PostgreSQL table. Th
 Full spec: `pf_spec.md`
 Data transport architecture options: `pf_perspective_options.md`
 UX mockup: `pf_ux_mockup.md`
-Open work: `todo.md`
 
 ---
 
@@ -51,11 +50,12 @@ ui/src/
 ## Database schema (`pf`)
 
 - **`pf.source`** — registered source tables
-- **`pf.col_meta`** — column roles: `dimension` | `value` | `units` | `date` | `filter` | `ignore`; `is_key` marks dimensions used in slice WHERE clauses
+- **`pf.col_meta`** — column roles: `dimension` | `value` | `units` | `date` | `filter` | `ignore`; `is_key` marks dimensions used in slice WHERE clauses; `dim_group` groups functionally dependent columns (e.g. date + its derived year/month dimensions); `dim_period_col` maps a dimension to a `pf.dim_period` column so date-adjacent values are derived at load time rather than copied raw
 - **`pf.version`** — named forecast scenarios; `exclude_iters` (default `["reference"]`) blocks those iter values from all operations
-- **`pf.fc_{tname}_{version_id}`** — one forecast table per version; contains both operational rows (`iter = baseline|scale|recode|clone`) and reference rows (`iter = reference`)
+- **`pf.fc_{tname}_{version_id}`** — one forecast table per version; contains both operational rows (`pf_iter = baseline|scale|recode|clone`) and reference rows (`pf_iter = reference`)
 - **`pf.log`** — audit log; every write gets one entry; `slice` + `params` stored as jsonb
 - **`pf.sql`** — generated SQL templates per source/operation; tokens substituted at request time
+- **`pf.dim_period`** — calendar lookup table (2018–2035); one row per month keyed on `sdat` (month start date); provides cal/fiscal year, quarter, and month columns; populated by `setup_sql/gen_dim_period.sql` with a configurable fiscal year start month
 
 ### Key token substitution tokens
 `{{fc_table}}`, `{{where_clause}}`, `{{exclude_clause}}`, `{{logid}}`, `{{pf_user}}`, `{{value_incr}}`, `{{units_incr}}`, `{{pct}}`, `{{set_clause}}`, `{{scale_factor}}`, `{{date_offset}}`, `{{filter_clause}}`
@@ -108,10 +108,9 @@ Theme state lives in `ui/src/theme.jsx` — a React context (`ThemeContext`) wit
 - **Perspective viewer:** `Forecast.jsx` calls `viewer.setAttribute('theme', dark ? 'Pro Dark' : 'Pro Light')` both on initial load and in a `useEffect([dark, versionId])` so the viewer stays in sync when the toggle fires
 - **Consuming the theme:** `import useTheme from '../theme.jsx'` then `const { dark, setDark } = useTheme()`
 
-## Known issues / active work (see todo.md for detail)
+## Known issues / active work
 
-- Operation panel (Scale/Recode/Clone) wiring to API is a stub — needs completion
-- Status bar is hardcoded — needs to reflect actual selected source/version
+- Operation panel (Scale/Recode/Clone) SQL generation and dim_period JOIN are complete; UI wiring to API still needs completion
 - Load progress bar is jittery — needs throttle (~10 updates/sec)
 - Default pivot layout should be configurable per source (currently hardcodes first 2 dimensions)
 - Source/version selection doesn't persist across page reload

lib/sql_generator.js

@@ -25,7 +25,7 @@ function generateSQL(source, colMeta) {
     if (!dateCol)  throw new Error('No date column defined in col_meta');
     if (dims.length === 0) throw new Error('No dimension columns defined in col_meta');
 
-    const srcTable        = `${source.schema}.${source.tname}`;
+    const srcTable        = `"${source.schema}"."${source.tname}"`;
     const dataCols        = [...dims, dateCol, valueCol, unitsCol].filter(Boolean);
     const effectiveValue  = dataCols.includes(valueCol) ? valueCol : null;
     const effectiveUnits  = dataCols.includes(unitsCol) ? unitsCol : null;
@@ -33,6 +33,19 @@ function generateSQL(source, colMeta) {
     const selectData      = dataCols.map(q).join(', ');
     const dimsJoined      = dims.map(q).join(', ');
 
+    // dim_period JOIN support: if the date column is the is_key of a dim_group,
+    // dimension siblings with dim_period_col set are derived from pf.dim_period
+    // instead of being copied raw from the source on baseline/reference load.
+    const dateKeyGroup = colMeta.find(c => c.role === 'date' && c.is_key && c.dim_group)?.dim_group;
+    const dimPeriodMap = new Map(
+        dateKeyGroup
+            ? colMeta
+                .filter(c => c.role === 'dimension' && c.dim_group === dateKeyGroup && c.dim_period_col)
+                .map(c => [c.cname, c.dim_period_col])
+            : []
+    );
+    const hasDimPeriod = dimPeriodMap.size > 0;
+
     return {
         get_data:  buildGetData(),
         baseline:  buildBaseline(),
@@ -47,10 +60,22 @@ function generateSQL(source, colMeta) {
         return `SELECT * FROM {{fc_table}}`;
     }
 
+    function buildLoadSelect(pfx) {
+        // pfx: table alias prefix ('s.' when joining dim_period, '' otherwise)
+        return dataCols.map(c => {
+            if (c === dateCol)          return `(${pfx}${q(c)} + '{{date_offset}}'::interval)::date`;
+            if (dimPeriodMap.has(c))    return `dp.${q(dimPeriodMap.get(c))} AS ${q(c)}`;
+            return `${pfx}${q(c)}`;
+        }).join(',\n        ');
+    }
+
+    function buildFromClause() {
+        if (!hasDimPeriod) return srcTable;
+        return `${srcTable} s\n    JOIN pf.dim_period dp`
+            + ` ON dp.drange @> (s.${q(dateCol)} + '{{date_offset}}'::interval)::date`;
+    }
+
     function buildBaseline() {
-        const baselineSelect = dataCols.map(c =>
-            c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c)
-        ).join(', ');
         return `
 WITH
 ilog AS (
@@ -60,8 +85,10 @@ ilog AS (
 )
 ,ins AS (
     INSERT INTO {{fc_table}} (${insertCols})
-    SELECT ${baselineSelect}, 'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
-    FROM ${srcTable}
+    SELECT
+        ${buildLoadSelect(hasDimPeriod ? 's.' : '')},
+        'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM ${buildFromClause()}
     WHERE {{filter_clause}}
     RETURNING *
 )
@@ -69,9 +96,6 @@ SELECT count(*) AS rows_affected FROM ins`.trim();
     }
 
     function buildReference() {
-        const referenceSelect = dataCols.map(c =>
-            c === dateCol ? `(${q(c)} + '{{date_offset}}'::interval)::date` : q(c)
-        ).join(', ');
         return `
 WITH
 ilog AS (
@@ -81,8 +105,10 @@ ilog AS (
 )
 ,ins AS (
     INSERT INTO {{fc_table}} (${insertCols})
-    SELECT ${referenceSelect}, 'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
-    FROM ${srcTable}
+    SELECT
+        ${buildLoadSelect(hasDimPeriod ? 's.' : '')},
+        'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM ${buildFromClause()}
     WHERE {{filter_clause}}
     RETURNING *
 )

pf_spec.md

@@ -24,13 +24,14 @@ Registered source tables available for forecasting.
 
 ```sql
 CREATE TABLE pf.source (
-    id          serial PRIMARY KEY,
-    schema      text NOT NULL,
-    tname       text NOT NULL,
-    label       text,                   -- friendly display name
-    status      text DEFAULT 'active',  -- active | archived
-    created_at  timestamptz DEFAULT now(),
-    created_by  text,
+    id              serial PRIMARY KEY,
+    schema          text NOT NULL,
+    tname           text NOT NULL,
+    label           text,                   -- friendly display name
+    status          text DEFAULT 'active',  -- active | archived
+    default_layout  jsonb,                  -- Perspective view config used as per-source default
+    created_at      timestamptz DEFAULT now(),
+    created_by      text,
     UNIQUE (schema, tname)
 );
 ```
@@ -40,25 +41,40 @@ Column configuration for each registered source table. Determines how the app tr
 
 ```sql
 CREATE TABLE pf.col_meta (
-    id          serial PRIMARY KEY,
-    source_id   integer REFERENCES pf.source(id),
-    cname       text NOT NULL,          -- column name in source table
-    label       text,                   -- friendly display name
-    role        text NOT NULL,          -- 'dimension' | 'value' | 'units' | 'date' | 'ignore'
-    is_key      boolean DEFAULT false,  -- true = part of natural key (used in WHERE slice)
-    opos        integer,                -- ordinal position (for ordering)
+    id              serial PRIMARY KEY,
+    source_id       integer REFERENCES pf.source(id),
+    cname           text NOT NULL,          -- column name in source table
+    label           text,                   -- friendly display name
+    role            text NOT NULL,          -- 'dimension' | 'value' | 'units' | 'date' | 'filter' | 'ignore'
+    is_key          boolean DEFAULT false,  -- true = part of natural key (used in WHERE slice)
+    opos            integer,                -- ordinal position (for ordering)
+    dim_group       text,                   -- groups functionally dependent columns (see below)
+    dim_period_col  text,                   -- maps this dimension to a pf.dim_period column
     UNIQUE (source_id, cname)
 );
 ```
 
 **Roles:**
 - `dimension` — categorical field (customer, part, channel, rep, geography, etc.) — appears as pivot rows/cols, used in WHERE filters for operations
-- `value` — the money/revenue field to scale
-- `units` — the quantity field to scale
-- `date` — the primary date field; used for baseline/reference date range and stored in the forecast table
+- `value` — the money/revenue field to scale (**required** — SQL generation fails without it)
+- `units` — the quantity field to scale (**optional** — if absent, units columns are omitted from the forecast table and all SQL patterns)
+- `date` — the primary date field; used for baseline/reference date range and stored in the forecast table (**required**)
 - `filter` — columns available as filter conditions in the Baseline Workbench (e.g. order status, ship date, open flag); used in baseline WHERE clauses but **not stored** in the forecast table
 - `ignore` — exclude from forecast table entirely
 
+**`dim_group`** — a free-text group name linking a `date` column to its derived dimension siblings. When the `date` column has `is_key = true` and a `dim_group` value, the SQL generator looks for `dimension` columns in the same group that also have a `dim_period_col` value. Those columns are sourced from `pf.dim_period` on baseline/reference load (via a JOIN on `drange @> date`) rather than copied raw from the source table. This allows fiscal year, quarter, and month columns to be stored in the forecast table with calendar-correct values even if those columns don't exist in the source.
+
+**`dim_period_col`** — names the column in `pf.dim_period` to use as the value for this dimension on load. Only meaningful when the column is in a `dim_group` whose `date` key has `is_key = true`. Example: `cal_year`, `fisc_quarter`, `fisc_label`.
+
+### `pf.dim_period`
+Calendar lookup table. One row per month from 2018-01-01 through 2035-12-01. Keyed on `sdat` (month start date). Used to derive fiscal/calendar period columns at baseline load time when `dim_group` / `dim_period_col` are configured on col_meta.
+
+Populated by `setup_sql/gen_dim_period.sql` (safe to re-run; `ON CONFLICT DO NOTHING`). Fiscal year start month is configurable at the top of that script (default: June, i.e. fiscal month 1 = June).
+
+Key columns: `sdat`, `edat`, `drange` (GiST-indexed daterange), `cal_year`, `cal_quarter`, `cal_month`, `cal_month_abbr`, `cal_month_name`, `cal_label`, `fisc_year`, `fisc_quarter`, `fisc_quarter_label`, `fisc_month`, `fisc_month_abbr`, `fisc_month_name`, `fisc_label`, `period_key`.
+
+The baseline/reference SQL JOINs this table when `hasDimPeriod` is true: `JOIN pf.dim_period dp ON dp.drange @> (s.{date_col} + '{{date_offset}}'::interval)::date`.
+
 ### `pf.version`
 Named forecast scenarios. One forecast table (`pf.fc_{tname}_{version_id}`) is created per version.
 
@@ -102,31 +118,31 @@ CREATE TABLE pf.log (
 ```
 
 ### `pf.fc_{tname}_{version_id}` (dynamic, one per version)
-Created when a version is created. Mirrors source table dimension/value/units/date columns plus forecast metadata. Contains both operational rows (`iter = 'baseline' | 'scale' | 'recode' | 'clone'`) and reference rows (`iter = 'reference'`).
+Created when a version is created. Mirrors source table dimension/value/date columns (and units if configured) plus any `dim_period_col`-derived dimension columns, plus forecast metadata. Contains both operational rows (`pf_iter = 'baseline' | 'scale' | 'recode' | 'clone'`) and reference rows (`pf_iter = 'reference'`).
 
 ```sql
 -- Example: source table "sales", version id 3 → pf.fc_sales_3
 CREATE TABLE pf.fc_sales_3 (
-    id          bigserial PRIMARY KEY,
+    id              bigserial PRIMARY KEY,
 
     -- mirrored from source (role = dimension | value | units | date only):
-    customer    text,
-    channel     text,
-    part        text,
-    geography   text,
-    order_date  date,
-    units       numeric,
-    value       numeric,
+    customer        text,
+    channel         text,
+    part            text,
+    geography       text,
+    order_date      date,
+    value           numeric,
+    units           numeric,    -- omitted if no 'units' role in col_meta
 
     -- forecast metadata:
-    iter        text,       -- 'baseline' | 'reference' | 'scale' | 'recode' | 'clone'
-    logid       bigint REFERENCES pf.log(id),
-    pf_user     text,
-    created_at  timestamptz DEFAULT now()
+    pf_iter         text,       -- 'baseline' | 'reference' | 'scale' | 'recode' | 'clone'
+    pf_logid        bigint REFERENCES pf.log(id),
+    pf_user         text,
+    pf_created_at   timestamptz DEFAULT now()
 );
 ```
 
-Note: no `version_id` column on the forecast table — it's implied by the table itself.
+Note: no `version_id` column on the forecast table — it's implied by the table itself. The `units` column is only present when a column with `role = 'units'` exists in col_meta.
 
 ### `pf.sql`
 Generated SQL stored per source and operation. Built once when col_meta is finalized, fetched at request time.
@@ -148,7 +164,7 @@ CREATE TABLE pf.sql (
 |-------|--------------|
 | `{{fc_table}}` | `pf.fc_{tname}_{version_id}` — derived at request time |
 | `{{where_clause}}` | built from `slice` JSON by `build_where()` in JS |
-| `{{exclude_clause}}` | built from `version.exclude_iters` — e.g. `AND iter NOT IN ('reference')` |
+| `{{exclude_clause}}` | built from `version.exclude_iters` — e.g. `AND pf_iter NOT IN ('reference')` |
 | `{{logid}}` | newly inserted `pf.log` id |
 | `{{pf_user}}` | from request body |
 | `{{date_from}}` / `{{date_to}}` | baseline/reference date range (source period) |
@@ -521,7 +537,11 @@ AG Grid list of log entries — user, timestamp, operation, slice, note, rows af
 
 ## Forecast SQL Patterns
 
-Column names baked in at generation time. Tokens substituted at request time.
+Column names baked in at generation time. Tokens substituted at request time. Metadata columns are `pf_iter`, `pf_logid`, `pf_user`, `pf_created_at`.
+
+**Units conditionality:** `{units_col}` appears in INSERT column lists and SELECT expressions only when a `units` role is configured in col_meta. The SQL generator omits it entirely otherwise — no placeholder column, no zero-fill.
+
+**dim_period JOIN:** when any `dimension` column has `dim_period_col` set (and its group's `date` key has `is_key = true`), the FROM clause becomes `{schema}.{tname} s JOIN pf.dim_period dp ON dp.drange @> (s.{date_col} + '{{date_offset}}'::interval)::date`. Those dimension columns are selected as `dp.{dim_period_col} AS {col}` instead of `s.{col}`.
 
 ### Baseline Load (one segment)
 
@@ -531,18 +551,23 @@ WITH ilog AS (
     VALUES ({{version_id}}, '{{pf_user}}', 'baseline', NULL, '{{params}}'::jsonb, '{{note}}')
     RETURNING id
 )
-INSERT INTO {{fc_table}} (
-    {dimension_cols}, {value_col}, {units_col}, {date_col},
-    iter, logid, pf_user, created_at
+,ins AS (
+    INSERT INTO {{fc_table}} (
+        {dimension_cols}, {date_col}, {value_col} [, {units_col}],
+        pf_iter, pf_logid, pf_user, pf_created_at
+    )
+    SELECT
+        {dimension_cols},
+        ({date_col} + '{{date_offset}}'::interval)::date,
+        {value_col} [, {units_col}],
+        'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM
+        {schema}.{tname}  -- or with dim_period JOIN (see above)
+    WHERE
+        {{filter_clause}}
+    RETURNING *
 )
-SELECT
-    {dimension_cols}, {value_col}, {units_col},
-    ({date_col} + '{{date_offset}}'::interval)::date,
-    'baseline', (SELECT id FROM ilog), '{{pf_user}}', now()
-FROM
-    {schema}.{tname}
-WHERE
-    {{filter_clause}}
+SELECT count(*) AS rows_affected FROM ins
 ```
 
 Baseline loads are **additive** — no DELETE before INSERT. Each segment appends independently.
@@ -557,7 +582,7 @@ Token details:
 
 Two queries, run in a transaction:
 ```sql
-DELETE FROM {{fc_table}} WHERE iter = 'baseline';
+DELETE FROM {{fc_table}} WHERE pf_iter = 'baseline';
 DELETE FROM pf.log WHERE version_id = {{version_id}} AND operation = 'baseline';
 ```
 
@@ -569,20 +594,24 @@ WITH ilog AS (
     VALUES ({{version_id}}, '{{pf_user}}', 'reference', NULL, '{{params}}'::jsonb, '{{note}}')
     RETURNING id
 )
-INSERT INTO {{fc_table}} (
-    {dimension_cols}, {value_col}, {units_col}, {date_col},
-    iter, logid, pf_user, created_at
+,ins AS (
+    INSERT INTO {{fc_table}} (
+        {dimension_cols}, {date_col}, {value_col} [, {units_col}],
+        pf_iter, pf_logid, pf_user, pf_created_at
+    )
+    SELECT
+        {dimension_cols}, {date_col}, {value_col} [, {units_col}],
+        'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM
+        {schema}.{tname}  -- or with dim_period JOIN (see above)
+    WHERE
+        {{filter_clause}}
+    RETURNING *
 )
-SELECT
-    {dimension_cols}, {value_col}, {units_col}, {date_col},
-    'reference', (SELECT id FROM ilog), '{{pf_user}}', now()
-FROM
-    {schema}.{tname}
-WHERE
-    {date_col} BETWEEN '{{date_from}}' AND '{{date_to}}'
+SELECT count(*) AS rows_affected FROM ins
 ```
 
-No date offset — reference rows land at their original dates for prior-period comparison.
+No date offset applied — reference rows land at their original dates for prior-period comparison. Same dim_period JOIN logic applies as baseline.
 
 ### Scale
 
@@ -595,26 +624,30 @@ WITH ilog AS (
 ,base AS (
     SELECT
         {dimension_cols}, {date_col},
-        {value_col}, {units_col},
-        sum({value_col}) OVER () AS total_value,
-        sum({units_col}) OVER () AS total_units
+        {value_col} [, {units_col}],
+        sum({value_col}) OVER () AS total_value
+        [, sum({units_col}) OVER () AS total_units]
     FROM {{fc_table}}
     WHERE {{where_clause}}
     {{exclude_clause}}
 )
-INSERT INTO {{fc_table}} (
-    {dimension_cols}, {date_col}, {value_col}, {units_col},
-    iter, logid, pf_user, created_at
+,ins AS (
+    INSERT INTO {{fc_table}} (
+        {dimension_cols}, {date_col}, {value_col} [, {units_col}],
+        pf_iter, pf_logid, pf_user, pf_created_at
+    )
+    SELECT
+        {dimension_cols}, {date_col},
+        round(({value_col} / NULLIF(total_value, 0)) * {{value_incr}}, 2)
+        [, round(({units_col} / NULLIF(total_units, 0)) * {{units_incr}}, 5)],
+        'scale', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM base
+    RETURNING *
 )
-SELECT
-    {dimension_cols}, {date_col},
-    round(({value_col} / NULLIF(total_value, 0)) * {{value_incr}}, 2),
-    round(({units_col} / NULLIF(total_units, 0)) * {{units_incr}}, 5),
-    'scale', (SELECT id FROM ilog), '{{pf_user}}', now()
-FROM base
+SELECT * FROM ins
 ```
 
-`{{value_incr}}` / `{{units_incr}}` are pre-computed in JS when `pct: true` (multiply slice total by pct).
+`{{value_incr}}` / `{{units_incr}}` are pre-computed in JS when `pct: true` (multiply slice total by pct). Units expressions are omitted when no units column is configured.
 
 ### Recode
 
@@ -625,22 +658,27 @@ WITH ilog AS (
     RETURNING id
 )
 ,src AS (
-    SELECT {dimension_cols}, {date_col}, {value_col}, {units_col}
+    SELECT {dimension_cols}, {date_col}, {value_col} [, {units_col}]
     FROM {{fc_table}}
     WHERE {{where_clause}}
     {{exclude_clause}}
 )
-,negatives AS (
-    INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at)
-    SELECT {dimension_cols}, {date_col}, -{value_col}, -{units_col}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
+,neg AS (
+    INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at)
+    SELECT {dimension_cols}, {date_col}, -{value_col} [, -{units_col}], 'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
     FROM src
+    RETURNING *
 )
-INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at)
-SELECT {{set_clause}}, {date_col}, {value_col}, {units_col}, 'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
-FROM src
+,ins AS (
+    INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at)
+    SELECT {{set_clause}}, {date_col}, {value_col} [, {units_col}], 'recode', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM src
+    RETURNING *
+)
+SELECT * FROM neg UNION ALL SELECT * FROM ins
 ```
 
-`{{set_clause}}` replaces the listed dimension columns with new values, passes others through unchanged.
+`{{set_clause}}` replaces the listed dimension columns with new values, passes others through unchanged. Both the negative (zero-out) and positive (replacement) rows share the same `pf_logid` and are undone together.
 
 ### Clone
 
@@ -650,21 +688,27 @@ WITH ilog AS (
     VALUES ({{version_id}}, '{{pf_user}}', 'clone', '{{slice}}'::jsonb, '{{params}}'::jsonb, '{{note}}')
     RETURNING id
 )
-INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col}, {units_col}, iter, logid, pf_user, created_at)
-SELECT
-    {{set_clause}}, {date_col},
-    round({value_col} * {{scale_factor}}, 2),
-    round({units_col} * {{scale_factor}}, 5),
-    'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
-FROM {{fc_table}}
-WHERE {{where_clause}}
-{{exclude_clause}}
+,ins AS (
+    INSERT INTO {{fc_table}} ({dimension_cols}, {date_col}, {value_col} [, {units_col}], pf_iter, pf_logid, pf_user, pf_created_at)
+    SELECT
+        {{set_clause}}, {date_col},
+        round({value_col} * {{scale_factor}}, 2)
+        [, round({units_col} * {{scale_factor}}, 5)],
+        'clone', (SELECT id FROM ilog), '{{pf_user}}', now()
+    FROM {{fc_table}}
+    WHERE {{where_clause}}
+    {{exclude_clause}}
+    RETURNING *
+)
+SELECT * FROM ins
 ```
 
 ### Undo
 
+Two queries run sequentially (not in a CTE — FK ordering):
+
 ```sql
-DELETE FROM {{fc_table}} WHERE logid = {{logid}};
+DELETE FROM {{fc_table}} WHERE pf_logid = {{logid}};
 DELETE FROM pf.log WHERE id = {{logid}};
 ```
 
@@ -698,38 +742,38 @@ DELETE FROM pf.log WHERE id = {{logid}};
 ## Open Questions / Future Scope
 
 - **Baseline replay** — re-execute change log against a restated baseline (`replay: true`); v1 returns 501
-- **Arrow IPC for initial data load** — at large row counts (1M+) the `/versions/:id/data` JSON response becomes a bottleneck. Option: serve Arrow IPC binary instead of JSON; Perspective's `worker.table()` accepts Arrow buffers natively. Incremental operation rows (scale/recode/clone) can stay as JSON fed to `table.update()` since they're always small. Could be implemented with `pg` + `apache-arrow` in Node, or by adding a server-side DuckDB instance (Postgres scanner → Arrow IPC) if a caching layer is also needed.
 - **Approval workflow** — user submits, admin approves before changes are visible to others (deferred)
 - **Territory filtering** — restrict what a user can see/edit by dimension value (deferred)
 - **Export** — download forecast as CSV or push results to a reporting table
 - **Version comparison** — side-by-side view of two versions (facilitated by isolated tables via UNION)
-- **Col meta / version schema drift** — if col_meta roles are changed after a version's forecast table is already created, the generated SQL and the table DDL go out of sync (e.g. a column added to SQL that doesn't exist in the table). UI should detect this: compare col_meta against the forecast table's actual columns via `information_schema`, warn the user, and offer to rebuild the version (drop + recreate table, preserving the version record and log). For now the workaround is to delete and recreate the version manually.
+- **Col meta / version schema drift** — if col_meta roles are changed after a version's forecast table is already created, the generated SQL and the table DDL go out of sync. UI should detect this: compare col_meta against the forecast table's actual columns via `information_schema`, warn the user, and offer to rebuild the version (drop + recreate table, preserving the version record and log). Workaround: delete and recreate the version manually.
 - **Multi-connection support** — currently one DB via `.env`. Full vision: `pf.connection` table (host, port, dbname, user, password as env-var ref), `connection_id` on `pf.source`, per-connection pg pools at runtime. `pf` schema stays on a "home" connection; source data can live anywhere. Connections UI in Setup. Safe to defer while in dev — requires clean reinstall when added since it changes the source schema.
 
 ---
 
-## Project Status — 2026-04-25
+## Project Status — 2026-06-12
 
 ### What's working
 - Full backend: source registration, col_meta, SQL generation, versions, baseline segments, reference load, scale, recode, clone, undo
-- React + Vite + Tailwind CSS frontend scaffolded in `ui/`, built output to `public/app/`, served by Express
-- 3-step collapsible sidebar (Setup / Baseline / Forecast) — addresses prior UX concern about opaque 5-tab nav
-- Setup view: DB table browser with preview modal, source registration, col_meta editor, SQL generation
+- `units` column is optional — sources without a units column register and generate SQL correctly
+- `dim_group` / `dim_period_col` on col_meta: baseline/reference load JOINs `pf.dim_period` to derive fiscal/calendar period columns rather than copying them raw from the source
+- `pf.dim_period` calendar table (2018–2035): populated by `setup_sql/gen_dim_period.sql`, configurable fiscal year start
+- React + Vite + Tailwind CSS frontend in `ui/`, built output to `public/app/`, served by Express
+- Data transport: Arrow IPC binary stream (`GET /api/versions/:id/data`); server accumulates all rows into one record batch; client hands buffer directly to Perspective WASM
+- 3-step collapsible sidebar (Setup / Baseline / Forecast)
+- Setup view: DB table browser with preview modal, source registration, col_meta editor (`dim_group`/`dim_period_col` fields included), SQL generation
 - Baseline view: version management (create/close/reopen/delete), multi-segment baseline workbench, canvas timeline, filter builder
-- Perspective pivot in Forecast view: loads all version rows, interactive group/split/filter/chart, layout saved per version
+- Perspective pivot in Forecast view: loads all version rows, interactive group/split/filter/chart, layout saved per version to localStorage
 - Slice extraction from `perspective-click` event feeds operation panel directly
-- Incremental row streaming: operation results (`RETURNING *`) stream into Perspective table without full reload
+- Incremental row streaming: operation results (`RETURNING *`) applied to Perspective table via `pspTable.update()` — no full reload
 - Status bar: shows current source · version · baseline row count · status
 
 ### Known issues / next focus
 
-- **Forecast view** — operation panel (Scale / Recode / Clone) is a stub; needs wiring to API
-- **Status bar** — currently hardcoded; needs to reflect actual selected source/version from state
-- **Col_meta / version schema drift** — if col_meta changes after a version's forecast table is created, the SQL and table DDL go out of sync. UI should detect this (compare col_meta against `information_schema`), warn, and offer rebuild. Workaround: delete and recreate the version.
-- **No "current version" persistence** — source/version selection resets on page reload; session context not persisted
-- **Perspective slice limitation** — computed date columns (Month, YearDate) extracted via split_by don't filter back to raw rows; only native dimension columns work for slice extraction
-
-### Branch status
-- `baseline-workbench` — merged to origin, stable
-- `perspective-forecast` — active development branch; React UI scaffolded, Forecast operation panel pending
+- **Forecast view** — operation panel SQL generation complete; UI wiring to API still needed
+- **Load progress bar** — jittery at high throughput; throttle to ~10 updates/sec
+- **Default pivot layout** — per-source configurable layout not yet implemented; currently hardcodes first 2 dimensions
+- **No "current version" persistence** — source/version selection resets on page reload
+- **Perspective slice limitation** — computed date columns (Month, YearDate) from split_by don't map back to raw rows; only native dimension columns work for slice extraction
+- **Col_meta / version schema drift** — if col_meta changes after a version's forecast table is created, SQL and DDL go out of sync. Workaround: delete and recreate the version.
 

routes/log.js

@@ -4,11 +4,31 @@ const { fcTable } = require('../lib/utils');
 module.exports = function(pool) {
     const router = express.Router();
 
-    // list all log entries for a version, newest first
+    // list all log entries for a version, newest first, with row counts from fc_table
     router.get('/versions/:id/log', async (req, res) => {
         try {
+            const verResult = await pool.query(`
+                SELECT v.*, s.tname
+                FROM pf.version v
+                JOIN pf.source s ON s.id = v.source_id
+                WHERE v.id = $1
+            `, [req.params.id]);
+            if (verResult.rows.length === 0) return res.status(404).json({ error: 'Version not found' });
+
+            const { tname, id: version_id } = verResult.rows[0];
+            const table = fcTable(tname, version_id);
+
             const result = await pool.query(
-                `SELECT * FROM pf.log WHERE version_id = $1 ORDER BY stamp DESC`,
+                `SELECT l.*,
+                        counts.row_count
+                 FROM pf.log l
+                 LEFT JOIN (
+                     SELECT pf_logid, count(*)::int AS row_count
+                     FROM ${table}
+                     GROUP BY pf_logid
+                 ) counts ON counts.pf_logid = l.id
+                 WHERE l.version_id = $1
+                 ORDER BY l.stamp DESC`,
                 [req.params.id]
             );
             res.json(result.rows);
@@ -18,6 +38,22 @@ module.exports = function(pool) {
         }
     });
 
+    // update note on a log entry
+    router.patch('/log/:logid', async (req, res) => {
+        const { note } = req.body;
+        try {
+            const result = await pool.query(
+                `UPDATE pf.log SET note = $1 WHERE id = $2 RETURNING *`,
+                [note ?? null, parseInt(req.params.logid)]
+            );
+            if (result.rows.length === 0) return res.status(404).json({ error: 'Log entry not found' });
+            res.json(result.rows[0]);
+        } catch (err) {
+            console.error(err);
+            res.status(500).json({ error: err.message });
+        }
+    });
+
     // undo an operation — deletes all forecast rows with this logid, then the log entry
     // two separate queries in a transaction to avoid FK ordering issues
     router.delete('/log/:logid', async (req, res) => {

routes/sources.js

@@ -89,22 +89,24 @@ module.exports = function(pool) {
             await client.query('BEGIN');
             for (const col of cols) {
                 await client.query(`
-                    INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, opos)
-                    VALUES ($1, $2, $3, $4, $5, $6, $7)
+                    INSERT INTO pf.col_meta (source_id, cname, label, role, is_key, dim_group, dim_period_col, opos)
+                    VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
                     ON CONFLICT (source_id, cname) DO UPDATE SET
-                        label     = EXCLUDED.label,
-                        role      = EXCLUDED.role,
-                        is_key    = EXCLUDED.is_key,
-                        dim_group = EXCLUDED.dim_group,
-                        opos      = EXCLUDED.opos
+                        label          = EXCLUDED.label,
+                        role           = EXCLUDED.role,
+                        is_key         = EXCLUDED.is_key,
+                        dim_group      = EXCLUDED.dim_group,
+                        dim_period_col = EXCLUDED.dim_period_col,
+                        opos           = EXCLUDED.opos
                 `, [
                     sourceId,
                     col.cname,
-                    col.label     || null,
-                    col.role      || 'ignore',
-                    col.is_key    || false,
-                    col.dim_group || null,
-                    col.opos      || null
+                    col.label          || null,
+                    col.role           || 'ignore',
+                    col.is_key         || false,
+                    col.dim_group      || null,
+                    col.dim_period_col || null,
+                    col.opos           || null
                 ]);
             }
             await client.query('COMMIT');
@@ -224,6 +226,41 @@ module.exports = function(pool) {
         }
     });
 
+    // given a key column value, look up sibling dim_group column values from source
+    // returns { sibling_col: value, ... } if exactly one match, null if none or ambiguous
+    router.get('/sources/:id/lookup', async (req, res) => {
+        const { col, value } = req.query;
+        if (!col || value == null || value === '') return res.json(null);
+        try {
+            const [srcResult, metaResult] = await Promise.all([
+                pool.query(`SELECT schema, tname FROM pf.source WHERE id = $1`, [req.params.id]),
+                pool.query(`SELECT * FROM pf.col_meta WHERE source_id = $1 ORDER BY opos`, [req.params.id])
+            ]);
+            if (srcResult.rows.length === 0) return res.status(404).json({ error: 'Source not found' });
+
+            const keyCol = metaResult.rows.find(c => c.cname === col && c.is_key && c.dim_group);
+            if (!keyCol) return res.json(null);
+
+            const siblings = metaResult.rows.filter(c =>
+                c.dim_group === keyCol.dim_group && c.cname !== col
+            );
+            if (!siblings.length) return res.json(null);
+
+            const { schema, tname } = srcResult.rows[0];
+            const sibCols = siblings.map(c => `"${c.cname}"`).join(', ');
+            const result = await pool.query(
+                `SELECT DISTINCT ${sibCols} FROM "${schema}"."${tname}" WHERE "${col}" = $1 LIMIT 2`,
+                [value]
+            );
+
+            if (result.rows.length !== 1) return res.json(null);
+            res.json(result.rows[0]);
+        } catch (err) {
+            console.error(err);
+            res.status(500).json({ error: err.message });
+        }
+    });
+
     // set or clear the default Perspective layout for a source.
     // Body: a Perspective view config (group_by, split_by, columns, plugin_config, …).
     // Pass null or {} to clear.

setup_sql/01_schema.sql

@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS pf.source (
 -- backfill columns for existing installs
 ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb;
 ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text;
+ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text;
 
 -- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate
 

todo.md

@@ -1,113 +0,0 @@
-- [ ] when you enter the forecast, be able to enter in a context so you dont have to open the whole thing (should show in status bar and be a filter for SQL and spi calls)
-
-
-- [x] should be able to edit and revise forecase segments that constitute baseline or reference. if you edit, maybe a warning that your forecast values wont mean a lot, and have an option to delete them.
-
-  Notes: A baseline/reference segment is a `pf.log` row plus the
-  forecast rows it produced (joined by pf_logid). Editing has the
-  shape of a delete-then-replay: drop the rows by pf_logid, drop the
-  log entry, re-run the segment with the new params (offset, filter,
-  iter type), insert the new log entry. New endpoint:
-  `PUT /versions/:id/baseline/:logid` (and the same for reference).
-  UI: an Edit button on each segment in Baseline view, populating the
-  form with the original `params`.
-
-  Cascade warning: if any scale/recode/clone log entries exist *after*
-  this segment was added, those operations were calibrated against
-  the old totals and will no longer reconcile cleanly. Show a banner
-  like "3 forecast operations applied after this segment may be
-  invalidated. View / Delete / Continue." Probably want a CASCADE
-  option that deletes downstream forecast entries too, plus a plain
-  "edit only" option for the user who knows what they're doing.
-
-  Implementation order: API + cascade detection first (compare
-  pf.log.stamp ordering); UI second.
-
-- [~] be able to copy an existing forecast and it's segments to adjust some parameters without having to start from scrath.
-
-  Notes: A version is the unit of copy. Need a `POST /versions/:id/copy`
-  endpoint that creates a new pf.version row with the same source/
-  col_meta, creates the new fc_<tname>_<id> table via the same DDL
-  path, and replays each pf.log entry's INSERT against the new table
-  (preserving stamp ordering). Each log entry gets re-inserted
-  pointing at the new version_id; the new pf_logid feeds the row
-  inserts. Notes/users come along.
-
-  UI: "Copy" button next to each version in Baseline. Copy modal
-  asks for a new name and optional description, then runs the API
-  call (likely 5–30s for a 350k-row version since every segment is
-  re-evaluated). Show progress.
-
-  Two design questions worth deciding up front:
-  - Copy as-of-now (re-fetch source data, so freshly-arrived rows
-    show up in baseline)? Or freeze (replay from existing forecast
-    rows, i.e. clone the forecast table directly)? Different
-    semantics, different SQL — pick one before building.
-  - Should the copy track its origin? A `parent_version_id` column
-    on pf.version makes "show me variants of FY2026 Plan" easy.
-
-- [x] need the list of filters to have an and/or specification
-
-  Notes: Spec already covers this in `pf_spec.md:245` — `filters` is
-  an array of groups; conditions within a group are AND-ed, groups
-  OR-ed. Backend has `buildFilterClause` in
-  `lib/sql_generator.js:247` but it's not wired into the routes
-  (baseline currently takes raw `where_clause`). Wiring + UI is the
-  remaining work.
-
-  UI: each group is a card with a header ("Group 1", "Group 2 — OR"),
-  rows of `column / operator / values`, a `+ Add condition` link,
-  and a `+ Add OR group` button at the bottom. The Baseline view
-  already has a single-group filter builder; extend it to wrap the
-  current rows in a group container and allow adding more groups.
-
-- [x] the filters should have the option to just write the WHERE clause SQL
-
-  Notes: Spec covers this too (`pf_spec.md:251`, `:454`) as the
-  `raw_where` admin-only escape hatch. The current baseline endpoint
-  *already* takes `where_clause` as a raw string — so the API is
-  effectively in "raw only" mode today; it's the structured side
-  that's missing. Two things to add:
-
-  - Once structured `filters` is wired in, gate `raw_where` behind
-    an admin check (`pf_user` in admin list — needs admin list
-    config) and reject 400 if both are sent.
-  - UI toggle: a "Switch to manual SQL" link in the Baseline filter
-    builder swaps the structured rows for a `<textarea>`; warning
-    banner: "Raw SQL is not validated. You are responsible for
-    correctness and security."
-
-- [ ] load status bar is super jittery and the numbers wildly change
-
-  Notes: `setLoadProgress` fires per chunk in the body-stream reader
-  (Forecast.jsx:155–161). On localhost or fast connections the
-  reader yields chunks in tight bursts and React re-renders the
-  overlay text on each — the visible bytes value flickers because
-  paints land out of order with respect to setState batching.
-
-  Fix: throttle to ~10 updates/sec. Either `if (now - lastUpdate >
-  100)` before `setLoadProgress`, or accumulate received bytes into
-  a ref and flush on `requestAnimationFrame`. Five-line change.
-
-- [ ] default layout for the pivot should be sales_usd group by pending_rep, split by pf_iter
-
-  Notes: Default-layout logic lives in `initViewer` (Forecast.jsx
-  ~line 240) and currently picks `group_by = first 2 dimensions`,
-  `split_by = date column`. `sales_usd` / `pending_rep` are
-  source-specific, so hardcoding them in the view would break for
-  any other source.
-
-  Two paths:
-  - **Quick**: hardcode for the current source. Cheap, but rots
-    the moment a second source comes along.
-  - **Right**: store a default-layout config on `pf.source` (e.g., a
-    JSON `default_view` column with `{ group_by, split_by, columns }`)
-    and let initViewer read it. Setup view gets a small "Default
-    pivot" editor — pick a value column, group_by columns, split_by
-    column.
-
-  Suggest the right version since the spec already implies
-  per-source customization in col_meta, and you're going to want
-  this for any future source you register. The col_meta path is
-  even tighter: extend col_meta with role flags like `default_value`,
-  `default_group`, `default_split` that initViewer reads.

ui/src/views/Forecast.jsx

@@ -436,6 +436,21 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou
     } catch (err) { flash(err.message, 'error') }
   }
 
+  async function lookupDerivedCols(col, value, setter) {
+    if (!sourceId || !value.trim()) return
+    const res = await fetch(`/api/sources/${sourceId}/lookup?col=${encodeURIComponent(col)}&value=${encodeURIComponent(value)}`)
+    if (!res.ok) return
+    const derived = await res.json()
+    if (!derived) return
+    setter(prev => {
+      const next = { ...prev }
+      for (const [k, v] of Object.entries(derived)) {
+        if (!prev[k] || prev[k] === '') next[k] = String(v ?? '')
+      }
+      return next
+    })
+  }
+
   function buildEffectiveSlice(raw) {
     const dimCols  = new Set(colMetaRef.current.filter(c => c.role === 'dimension').map(c => c.cname))
     const dateCols = new Set(colMetaRef.current.filter(c => c.role === 'date').map(c => c.cname))
@@ -842,9 +857,15 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou
                 {activeOp === 'recode' && <>
                   <p className="text-gray-400">New values for dimensions to replace. Leave blank to keep.</p>
                   {dimCols.map(c => (
-                    <Row key={c.cname} label={c.cname}>
-                      <input value={recodeSet[c.cname] || ''} onChange={e => setRecodeSet(s => ({ ...s, [c.cname]: e.target.value }))}
-                        placeholder={slice[c.cname] || '—'} className={`${inp} font-mono`} />
+                    <Row key={c.cname} label={c.label || c.cname}>
+                      <input
+                        value={recodeSet[c.cname] || ''}
+                        onChange={e => setRecodeSet(s => ({ ...s, [c.cname]: e.target.value }))}
+                        onBlur={c.is_key && c.dim_group
+                          ? e => lookupDerivedCols(c.cname, e.target.value, setRecodeSet)
+                          : undefined}
+                        placeholder={slice[c.cname] || '—'}
+                        className={`${inp} font-mono`} />
                     </Row>
                   ))}
                   <Row label="Note"><input value={recodeNote} onChange={e => setRecodeNote(e.target.value)} placeholder="optional" className={inp} /></Row>
@@ -855,9 +876,15 @@ export default function Forecast({ sources = [], sourceId, versionId, refreshSou
                 {activeOp === 'clone' && <>
                   <p className="text-gray-400">Override dimensions on cloned rows. Leave blank to keep.</p>
                   {dimCols.map(c => (
-                    <Row key={c.cname} label={c.cname}>
-                      <input value={cloneSet[c.cname] || ''} onChange={e => setCloneSet(s => ({ ...s, [c.cname]: e.target.value }))}
-                        placeholder={slice[c.cname] || '—'} className={`${inp} font-mono`} />
+                    <Row key={c.cname} label={c.label || c.cname}>
+                      <input
+                        value={cloneSet[c.cname] || ''}
+                        onChange={e => setCloneSet(s => ({ ...s, [c.cname]: e.target.value }))}
+                        onBlur={c.is_key && c.dim_group
+                          ? e => lookupDerivedCols(c.cname, e.target.value, setCloneSet)
+                          : undefined}
+                        placeholder={slice[c.cname] || '—'}
+                        className={`${inp} font-mono`} />
                     </Row>
                   ))}
                   <Row label="Scale"><input type="number" step="any" value={cloneScale} onChange={e => setCloneScale(e.target.value)} className={inp} /></Row>

ui/src/views/Setup.jsx

@@ -282,6 +282,7 @@ export default function Setup({ refreshSources }) {
                       <th className="px-3 py-1.5 font-medium">role</th>
                       <th className="px-3 py-1.5 font-medium text-center">key</th>
                       <th className="px-3 py-1.5 font-medium">group</th>
+                      <th className="px-3 py-1.5 font-medium">period col</th>
                       <th className="px-3 py-1.5 font-medium">label</th>
                     </tr>
                   </thead>
@@ -317,6 +318,16 @@ export default function Setup({ refreshSources }) {
                             className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default"
                           />
                         </td>
+                        <td className="px-3 py-1.5">
+                          <input
+                            type="text"
+                            value={col.dim_period_col || ''}
+                            onChange={e => updateCol(i, 'dim_period_col', e.target.value || null)}
+                            placeholder="—"
+                            disabled={col.role !== 'dimension' || !col.dim_group}
+                            className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default font-mono text-xs"
+                          />
+                        </td>
                         <td className="px-3 py-1.5">
                           <input
                             type="text"