Paul Trowbridge
654a368672
Ship rows pre-aggregated to the grain the pivot displays instead of raw
forecast rows. This is Path B from pf_perspective_options.md: it keeps
Perspective's native WASM engine — so expand/collapse/depth/sort/filter
all still work — and fixes load time by cutting rows, not transport.
Measured on pf.fc_osm_stack_20 at pending_rep x customer x smon:
534,902 -> 6,154 rows (~87x), pf_gkey unique across all 6,154, and both
measures reconcile exactly to the raw totals.
The grain is static: flagged once per source in Setup and baked into the
stored pf.sql templates, so load and operations agree by construction.
Sources with no flagged column keep the previous raw-row behaviour, so
this is backward compatible.
- pf.col_meta gains in_grain; grainOf() in lib/sql_generator.js is the
single definition of the grain and is reused by routes/log.js.
- New get_agg template + GET /api/versions/:id/agg, generated only when a
grain is defined. Regenerating drops templates no longer produced, so
clearing the grain falls back to /data.
- scale/recode/clone now aggregate their own new rows to grain before
returning. Because pf_logid is part of pf_gkey those keys are always
new, so table.update() appends and the view re-sums — the Excel
pivot-cache pattern, no bucket recomputation.
- Undo reports pf_gkeys (RETURNING cannot take DISTINCT, so the delete
feeds a CTE that reduces to distinct keys); the client removes those
index values and the view re-sums.
- pf_gkey is concat_ws(chr(31), COALESCE(col::text, chr(30)), ...).
The separator and NULL sentinel are load-bearing: plain concat_ws skips
NULLs, so ('a',NULL) and (NULL,'a') would collide and silently merge two
groups into one indexed row.
- Forecast.jsx reads col_meta first to pick /agg vs /data; the Arrow
streaming logic is extracted to fetchArrow() since both share it.
- Setup.jsx gains a grain checkbox and shows the resulting grain.
- 01_schema.sql: move the col_meta ALTERs after its CREATE TABLE — they
referenced the table before it existed on a fresh install.
All six generated statements verified to plan against the real forecast
table; the in_grain column has been added to the dev database.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
76 lines
3.1 KiB
SQL
76 lines
3.1 KiB
SQL
-- Pivot Forecast schema install
|
|
-- Run once against target database: psql -d <db> -f setup_sql/01_schema.sql
|
|
|
|
CREATE SCHEMA IF NOT EXISTS pf;
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.source (
|
|
id serial PRIMARY KEY,
|
|
schema text NOT NULL,
|
|
tname text NOT NULL,
|
|
label text,
|
|
status text NOT NULL DEFAULT 'active', -- active | archived
|
|
default_layout jsonb, -- Perspective view config used as the per-source default
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
created_by text,
|
|
UNIQUE (schema, tname)
|
|
);
|
|
|
|
-- backfill columns for existing installs
|
|
ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb;
|
|
|
|
-- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.col_meta (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE,
|
|
cname text NOT NULL,
|
|
label text,
|
|
role text NOT NULL DEFAULT 'ignore', -- dimension | value | units | date | ignore
|
|
is_key boolean NOT NULL DEFAULT false, -- true = usable in WHERE slice
|
|
dim_group text, -- groups functionally dependent columns
|
|
dim_period_col text, -- pf.dim_period column this dimension derives from
|
|
in_grain boolean NOT NULL DEFAULT false, -- true = column defines the display grain
|
|
opos integer,
|
|
UNIQUE (source_id, cname)
|
|
);
|
|
|
|
-- backfill columns for existing installs (must follow the CREATE above)
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text;
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text;
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS in_grain boolean NOT NULL DEFAULT false;
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.version (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE RESTRICT,
|
|
name text NOT NULL,
|
|
description text,
|
|
status text NOT NULL DEFAULT 'open', -- open | closed
|
|
exclude_iters jsonb NOT NULL DEFAULT '["reference"]'::jsonb,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
created_by text,
|
|
closed_at timestamptz,
|
|
closed_by text,
|
|
UNIQUE (source_id, name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.log (
|
|
id bigserial PRIMARY KEY,
|
|
version_id integer NOT NULL REFERENCES pf.version(id) ON DELETE CASCADE,
|
|
pf_user text NOT NULL,
|
|
stamp timestamptz NOT NULL DEFAULT now(),
|
|
operation text NOT NULL, -- baseline | reference | scale | recode | clone
|
|
slice jsonb,
|
|
params jsonb,
|
|
note text
|
|
);
|
|
|
|
-- generated operation SQL per source, stored after col_meta is configured
|
|
CREATE TABLE IF NOT EXISTS pf.sql (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE,
|
|
operation text NOT NULL, -- get_data | baseline | reference | scale | recode | clone | undo
|
|
sql text NOT NULL,
|
|
generated_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE (source_id, operation)
|
|
);
|