Paul Trowbridge
c6005bd17c
params says what was asked for. That is not enough to explain a surprising result, because the same intent produces different rows depending on state the entry does not carry, and because the translation from intent to SQL is itself a place bugs live. So both, not one. env records the state that cannot be reconstructed later: the territory in force, the version's exclude_iters, and when the template was generated -- all mutable rows elsewhere with nothing remembering what they were. sql_text records the statement as executed, territory and scope already resolved into it. The template generation is a fingerprint rather than a version. It cannot bring the old template back; it can tell you the entry did not run under the current one, which is what would otherwise make a comparison quietly wrong. Generate SQL has overwritten those templates four times today. The statement is fetched on demand through GET /log/:logid/debug and left out of the list, which is opened to scan rather than to read SQL. Under an entry's payload in the change log there is now an "executed SQL" toggle. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
206 lines
10 KiB
SQL
206 lines
10 KiB
SQL
-- Pivot Forecast schema install
|
|
-- Run once against target database: psql -d <db> -f setup_sql/01_schema.sql
|
|
|
|
CREATE SCHEMA IF NOT EXISTS pf;
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.source (
|
|
id serial PRIMARY KEY,
|
|
schema text NOT NULL,
|
|
tname text NOT NULL,
|
|
label text,
|
|
status text NOT NULL DEFAULT 'active', -- active | archived
|
|
default_layout jsonb, -- Perspective view config used as the per-source default
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
created_by text,
|
|
UNIQUE (schema, tname)
|
|
);
|
|
|
|
-- backfill columns for existing installs
|
|
ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb;
|
|
|
|
-- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.col_meta (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE,
|
|
cname text NOT NULL,
|
|
label text,
|
|
role text NOT NULL DEFAULT 'ignore', -- dimension | value | units | date | ignore
|
|
is_key boolean NOT NULL DEFAULT false, -- true = usable in WHERE slice
|
|
dim_group text, -- groups functionally dependent columns
|
|
dim_period_col text, -- pf.dim_period column this dimension derives from
|
|
in_grain boolean NOT NULL DEFAULT false, -- true = column defines the display grain
|
|
opos integer,
|
|
UNIQUE (source_id, cname)
|
|
);
|
|
|
|
-- backfill columns for existing installs (must follow the CREATE above)
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text;
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text;
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS in_grain boolean NOT NULL DEFAULT false;
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.version (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE RESTRICT,
|
|
name text NOT NULL,
|
|
description text,
|
|
status text NOT NULL DEFAULT 'open', -- open | closed
|
|
exclude_iters jsonb NOT NULL DEFAULT '["reference"]'::jsonb,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
created_by text,
|
|
closed_at timestamptz,
|
|
closed_by text,
|
|
UNIQUE (source_id, name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pf.log (
|
|
id bigserial PRIMARY KEY,
|
|
version_id integer NOT NULL REFERENCES pf.version(id) ON DELETE CASCADE,
|
|
pf_user text NOT NULL,
|
|
stamp timestamptz NOT NULL DEFAULT now(),
|
|
operation text NOT NULL, -- baseline | reference | scale | recode | clone
|
|
slice jsonb,
|
|
params jsonb,
|
|
note text,
|
|
tag text -- initiative label, e.g. 'reduce_spend'; groups
|
|
-- adjustments into a bridge from baseline to current
|
|
);
|
|
|
|
-- adding tags to an install that predates them
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS tag text;
|
|
CREATE INDEX IF NOT EXISTS log_tag_idx ON pf.log (tag) WHERE tag IS NOT NULL;
|
|
|
|
-- seed tags for loads that predate the column: a baseline/reference note is the
|
|
-- segment's name ('Open Orders', 'Prior Year'), which is exactly what tag holds.
|
|
-- Adjustment notes are free text, not labels, so they are left alone.
|
|
UPDATE pf.log
|
|
SET tag = note
|
|
WHERE TRUE
|
|
AND tag IS NULL
|
|
AND note IS NOT NULL
|
|
AND note <> ''
|
|
AND operation IN ('baseline', 'reference');
|
|
|
|
-- Display order for the pivot's segment and bucket columns.
|
|
--
|
|
-- The segment's display name in the pivot, falling back to tag then note.
|
|
--
|
|
-- Separate from both because those have jobs already -- tag groups adjustments
|
|
-- into initiatives for the bridge, note is free commentary -- and because the
|
|
-- label carries the sort order. Perspective orders column groups by the value
|
|
-- string, so a leading "01 - " is how ordering is expressed; putting that in the
|
|
-- note would put it in every note.
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS label text;
|
|
|
|
-- Vestigial, both of them. They held the ordinal when the "01 - " prefix was
|
|
-- computed for the pivot rather than typed into label and bucket: bucket_order
|
|
-- sequenced the bucket columns, log.seq the segments within them. Nothing reads
|
|
-- either now, and nothing writes them -- kept only because dropping a column is
|
|
-- not worth a migration to reclaim two that cost nothing.
|
|
ALTER TABLE pf.version ADD COLUMN IF NOT EXISTS bucket_order jsonb;
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS seq integer;
|
|
|
|
-- What a segment contributes to, independent of pf_iter.
|
|
--
|
|
-- pf_iter answers "can operations write to these rows"; bucket answers "does this
|
|
-- belong in the forecast number". Those are not the same question -- Open Orders is
|
|
-- loaded as reference so nothing adjusts it, yet it is part of the forecast -- so
|
|
-- neither can be derived from the other.
|
|
--
|
|
-- Free text with suggested values (Forecast / Prior Year / Prior Prior Year / Plan)
|
|
-- rather than an enum, so a new banner does not need a migration. Blank by default:
|
|
-- until a segment is labelled, the pivot falls back to showing its own name.
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS bucket text;
|
|
CREATE INDEX IF NOT EXISTS log_bucket_idx ON pf.log (bucket) WHERE bucket IS NOT NULL;
|
|
|
|
-- The names a row falls back to when nobody has named it, per scenario. Null
|
|
-- means "use the built-in", which is the DISPLAY DEFAULTS block in
|
|
-- lib/sql_generator.js. Read through a join at query time, not baked into
|
|
-- pf.sql: those templates are keyed on (source_id, operation) and shared by
|
|
-- every version of a source.
|
|
ALTER TABLE pf.version ADD COLUMN IF NOT EXISTS adjustment_segment text;
|
|
ALTER TABLE pf.version ADD COLUMN IF NOT EXISTS adjustment_bucket text;
|
|
ALTER TABLE pf.version ADD COLUMN IF NOT EXISTS unlabeled_load text;
|
|
|
|
-- What the entry did, stamped when it did it.
|
|
--
|
|
-- Not a cache: a log entry's forecast rows never change after it is written.
|
|
-- Only the operation that owns the logid inserts them, and the only thing that
|
|
-- removes them is undo, which deletes this row too -- so these totals are fixed
|
|
-- at write time rather than derived from something that can move underneath
|
|
-- them. The change log was joining the whole forecast table to recompute them
|
|
-- on every open, 2.5M rows to total a few thousand.
|
|
--
|
|
-- measure_cols records which columns they are denominated in, since the value
|
|
-- and units roles can be reassigned in col_meta and the numbers would otherwise
|
|
-- quietly come to mean something else.
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS row_count integer;
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS value_total double precision;
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS units_total double precision;
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS measure_cols jsonb;
|
|
|
|
-- The column an account's territory is expressed in -- the rep, the region,
|
|
-- whatever this source divides ownership by. Exactly one per source.
|
|
--
|
|
-- Flagged here rather than named in code because it is source-specific, the
|
|
-- same way the grain is: a second source should not need a code change to be
|
|
-- scoped by something other than a sales rep.
|
|
--
|
|
-- It does two jobs. The server scopes every read and write to the values on the
|
|
-- account, and recode refuses to *set* this column unless the account is an
|
|
-- admin -- moving a row between territories is reassignment, not forecasting.
|
|
ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS is_territory boolean NOT NULL DEFAULT false;
|
|
|
|
-- The debug path: what the entry was trying to do, and what that became.
|
|
--
|
|
-- Both, deliberately. params records the intent -- the slice, the scope, the
|
|
-- resolved increments -- and sql_text records the statement as executed, with
|
|
-- territory and scope already resolved into it. Keeping only one of them
|
|
-- assumes the translation between them is correct, which is exactly the
|
|
-- assumption in doubt when the rows look wrong.
|
|
--
|
|
-- env captures the state the intent was executed against that cannot be
|
|
-- reconstructed afterwards: the territory in force, the version's
|
|
-- exclude_iters, and when the template was generated. All three are mutable
|
|
-- rows elsewhere, and nothing remembers what they were.
|
|
--
|
|
-- The template generation is a fingerprint, not a version: it cannot reproduce
|
|
-- the old template, but it can tell you the entry ran under a different one,
|
|
-- which is what would otherwise make a replay quietly wrong.
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS env jsonb;
|
|
ALTER TABLE pf.log ADD COLUMN IF NOT EXISTS sql_text text;
|
|
|
|
-- Master data for a dim_group: one row per key value, with its sibling columns.
|
|
--
|
|
-- The source is transactional and often a view over all history, so deriving a
|
|
-- member list from it is both slow and wrong -- slow because it means scanning
|
|
-- millions of rows, wrong because it can only describe what was transacted and
|
|
-- has no way to say a part is discontinued or that a new one exists before it
|
|
-- has sold. This table is the app's own list, refreshed from the source but
|
|
-- curatable independently of it.
|
|
CREATE TABLE IF NOT EXISTS pf.dim_member (
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE,
|
|
dim_group text NOT NULL, -- matches pf.col_meta.dim_group
|
|
key_value text NOT NULL, -- the is_key column's value
|
|
attrs jsonb NOT NULL DEFAULT '{}'::jsonb, -- the sibling columns
|
|
is_active boolean NOT NULL DEFAULT true,
|
|
source_seen boolean NOT NULL DEFAULT true, -- present in the source at last refresh
|
|
added_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
refreshed_at timestamptz,
|
|
PRIMARY KEY (source_id, dim_group, key_value)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS dim_member_active_idx
|
|
ON pf.dim_member (source_id, dim_group) WHERE is_active;
|
|
|
|
-- generated operation SQL per source, stored after col_meta is configured
|
|
CREATE TABLE IF NOT EXISTS pf.sql (
|
|
id serial PRIMARY KEY,
|
|
source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE,
|
|
operation text NOT NULL, -- get_data | baseline | reference | scale | recode | clone | undo
|
|
sql text NOT NULL,
|
|
generated_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE (source_id, operation)
|
|
);
|