diff --git a/CLAUDE.md b/CLAUDE.md index 0706797..4438439 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,7 @@ Data transport architecture options: `pf_perspective_options.md` - **Frontend:** React + Vite + Tailwind CSS in `ui/`; built output lands in `public/app/` - **Pivot:** [Perspective](https://github.com/perspective-dev/perspective) (`@perspective-dev/*` distribution, **not** FINOS `@finos/perspective`) 4.4.0 loaded from CDN at runtime — see `PERSPECTIVE.md` for config/deploy guidance - **Dev:** `npm run dev` (nodemon) in root; `npm run build` in `ui/` +- **Schema:** forward-only SQL migrations in `setup_sql/migrations/`, applied by `npm run migrate` and tracked in `pf.schema_version`. `server.js` refuses to start when the database is behind. See `setup_sql/README.md` --- @@ -33,7 +34,13 @@ lib/ sql_generator.js buildFilterClause, token substitution helpers utils.js setup_sql/ - 01_schema.sql pf schema DDL — run once to install + README.md migration workflow — read before changing the schema + migrations/ ordered .sql, applied once, tracked in pf.schema_version + gen_dim_period.sql parameterised calendar load (not a migration) + schema.generated.sql pg_dump reference snapshot; generated, never edited +scripts/ + migrate.js migration CLI (up | status | baseline) + schema-dump.js regenerates schema.generated.sql ui/src/ views/ Setup.jsx DB browser, source registration, col_meta editor @@ -55,6 +62,7 @@ ui/src/ - **`pf.log`** — audit log; every write gets one entry; `slice` + `params` stored as jsonb - **`pf.sql`** — generated SQL templates per source/operation; tokens substituted at request time - **`pf.dim_period`** — calendar lookup table (2018–2035); one row per month keyed on `sdat` (month start date); provides cal/fiscal year, quarter, and month columns; populated by `setup_sql/gen_dim_period.sql` with a configurable fiscal year start month +- **`pf.schema_version`** — applied migrations (filename + checksum); owned by the migration runner, never edited by hand ### Key token substitution tokens `{{fc_table}}`, `{{where_clause}}`, `{{exclude_clause}}`, `{{logid}}`, `{{pf_user}}`, `{{value_incr}}`, `{{units_incr}}`, `{{pct}}`, `{{set_clause}}`, `{{scale_factor}}`, `{{date_offset}}`, `{{filter_clause}}` @@ -123,6 +131,7 @@ Theme state lives in `ui/src/theme.jsx` — a React context (`ThemeContext`) wit - Source/version selection doesn't persist across page reload - Col_meta / version schema drift: if col_meta roles change after a version's forecast table is created, SQL and DDL go out of sync — workaround is to delete and recreate the version - Grain drift: changing `in_grain` after a load requires Generate SQL + a page reload, since the loaded table's index and columns are fixed at load time. `routes/log.js` derives the grain from live col_meta, so a grain changed mid-session yields `pf_gkeys` that don't match the loaded table and undo silently removes nothing +- Migrations are forward-only — there are no down migrations. Rolling back a schema change means writing a new migration that reverses it - Grain is static per source — a dimension left unflagged cannot be pivoted on. Dynamic per-cut grain (intersect the viewer's field set with the eligible set) is the additive next step; see `pf_spec.md` → §Display-grain pre-aggregation ## Deferred (not in v1) diff --git a/lib/migrations.js b/lib/migrations.js new file mode 100644 index 0000000..953685c --- /dev/null +++ b/lib/migrations.js @@ -0,0 +1,158 @@ +// Migration bookkeeping shared by the runner (scripts/migrate.js) and the +// startup check in server.js. +// +// Migrations are plain .sql files in setup_sql/migrations, applied in filename +// order and recorded in pf.schema_version. They are immutable once applied: the +// runner stores a checksum and refuses to proceed if a file has changed, since +// editing an applied migration means databases silently disagree about what the +// schema is. +// +// pf.dim_period is deliberately not a migration — it is a parameterised data +// load (fiscal year start month), so it stays in setup_sql/gen_dim_period.sql. + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +const MIGRATIONS_DIR = path.join(__dirname, '..', 'setup_sql', 'migrations'); + +// Bookkeeping table has to exist before the first migration can be recorded, so +// the runner creates it rather than a migration doing so. +const BOOKKEEPING_DDL = ` + CREATE SCHEMA IF NOT EXISTS pf; + CREATE TABLE IF NOT EXISTS pf.schema_version ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + applied_by text + ); +`; + +function checksum(sql) { + return crypto.createHash('sha256').update(sql).digest('hex').slice(0, 16); +} + +// every .sql file on disk, in apply order +function readMigrations() { + if (!fs.existsSync(MIGRATIONS_DIR)) return []; + return fs.readdirSync(MIGRATIONS_DIR) + .filter(f => f.endsWith('.sql')) + .sort() + .map(filename => { + const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, filename), 'utf8'); + return { filename, sql, checksum: checksum(sql) }; + }); +} + +async function ensureBookkeeping(client) { + await client.query(BOOKKEEPING_DDL); +} + +async function readApplied(client) { + const { rows } = await client.query( + `SELECT filename, checksum, applied_at FROM pf.schema_version ORDER BY filename` + ); + return rows; +} + +// Compare disk against the database. Returns pending migrations plus any +// integrity problems worth refusing to run on. +async function status(client) { + await ensureBookkeeping(client); + const onDisk = readMigrations(); + const applied = await readApplied(client); + const appliedBy = new Map(applied.map(r => [r.filename, r])); + const diskBy = new Map(onDisk.map(m => [m.filename, m])); + + const pending = onDisk.filter(m => !appliedBy.has(m.filename)); + // an applied file whose contents changed — the schema is now undefined + const modified = onDisk + .filter(m => appliedBy.has(m.filename) && appliedBy.get(m.filename).checksum !== m.checksum) + .map(m => m.filename); + // recorded as applied but no longer on disk — someone deleted or renamed it + const missing = applied.filter(r => !diskBy.has(r.filename)).map(r => r.filename); + // a pending migration ordered before an applied one would apply out of sequence + const lastApplied = applied.length ? applied[applied.length - 1].filename : null; + const outOfOrder = lastApplied ? pending.filter(m => m.filename < lastApplied).map(m => m.filename) : []; + + return { onDisk, applied, pending, modified, missing, outOfOrder }; +} + +// Apply pending migrations, each in its own transaction so a failure leaves +// earlier ones committed and the failing one fully rolled back. +async function migrate(client, { user, log = console.log } = {}) { + const st = await status(client); + if (st.modified.length) { + throw new Error( + `Applied migrations have been modified: ${st.modified.join(', ')}. ` + + `Migrations are immutable — revert the edit and add a new migration instead.` + ); + } + if (st.missing.length) { + throw new Error( + `Migrations recorded as applied are missing from disk: ${st.missing.join(', ')}.` + ); + } + if (st.outOfOrder.length) { + throw new Error( + `Migrations would apply out of order: ${st.outOfOrder.join(', ')} sort before ` + + `already-applied migrations. Renumber them after the latest applied migration.` + ); + } + if (!st.pending.length) { + log('Schema is up to date — no migrations to apply.'); + return []; + } + + const done = []; + for (const m of st.pending) { + log(`applying ${m.filename} …`); + try { + await client.query('BEGIN'); + await client.query(m.sql); + await client.query( + `INSERT INTO pf.schema_version (filename, checksum, applied_by) VALUES ($1, $2, $3)`, + [m.filename, m.checksum, user || null] + ); + await client.query('COMMIT'); + done.push(m.filename); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + throw new Error(`${m.filename} failed: ${err.message}`); + } + } + log(`Applied ${done.length} migration(s).`); + return done; +} + +// Record migrations as applied without running them — for a database whose schema +// already matches, from before migrations existed. +async function baseline(client, { user, upTo, log = console.log } = {}) { + const st = await status(client); + const target = upTo + ? st.pending.filter(m => m.filename <= upTo) + : st.pending; + if (!target.length) { + log('Nothing to baseline — no pending migrations.'); + return []; + } + await client.query('BEGIN'); + try { + for (const m of target) { + await client.query( + `INSERT INTO pf.schema_version (filename, checksum, applied_by) + VALUES ($1, $2, $3) + ON CONFLICT (filename) DO NOTHING`, + [m.filename, m.checksum, user ? `${user} (baseline)` : 'baseline'] + ); + } + await client.query('COMMIT'); + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + throw err; + } + log(`Marked ${target.length} migration(s) as applied without running them.`); + return target.map(m => m.filename); +} + +module.exports = { MIGRATIONS_DIR, readMigrations, status, migrate, baseline, checksum }; diff --git a/package.json b/package.json index e4583d9..0db7ec3 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,11 @@ "scripts": { "start": "node server.js", "dev": "nodemon server.js", - "build": "cd ui && npm run build" + "build": "cd ui && npm run build", + "migrate": "node scripts/migrate.js up", + "migrate:status": "node scripts/migrate.js status", + "migrate:baseline": "node scripts/migrate.js baseline", + "schema:dump": "node scripts/schema-dump.js" }, "dependencies": { "apache-arrow": "^21.1.0", diff --git a/pf_spec.md b/pf_spec.md index 09ce5f2..b0265ab 100644 --- a/pf_spec.md +++ b/pf_spec.md @@ -189,9 +189,22 @@ CREATE TABLE pf.sql ( ``` setup_sql/ - 01_schema.sql -- CREATE SCHEMA pf; create all metadata tables (source, col_meta, version, log, sql) + migrations/ -- forward-only .sql, applied once each, tracked in pf.schema_version + 0001_initial_schema.sql CREATE SCHEMA pf + metadata tables + 0002_source_default_layout.sql + 0003_col_meta_dim_group_period.sql + 0004_col_meta_in_grain.sql + gen_dim_period.sql -- parameterised calendar load (fiscal year start month); not a migration + schema.generated.sql -- pg_dump reference snapshot; generated, never edited or applied + README.md -- migration workflow ``` +Install with `npm run migrate`, then run `gen_dim_period.sql`. For a database that +already matches the schema, `npm run migrate:baseline` records the migrations as +applied without running them. `server.js` refuses to start when the database is +behind, so drift surfaces at boot rather than as a column-not-found error inside +an unrelated request. + Source registration, col_meta configuration, SQL generation, version creation, and forecast table DDL all happen via API. --- diff --git a/scripts/migrate.js b/scripts/migrate.js new file mode 100644 index 0000000..8c9e62a --- /dev/null +++ b/scripts/migrate.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Migration CLI. +// +// npm run migrate apply pending migrations +// npm run migrate:status show what is applied and what is pending +// npm run migrate:baseline record pending migrations as applied WITHOUT running +// them — for a database that already matches +// +// Pass --up-to= to baseline only through a given migration. + +require('dotenv').config(); +const os = require('os'); +const { Client } = require('pg'); +const { status, migrate, baseline } = require('../lib/migrations'); + +const args = process.argv.slice(2); +const command = args.find(a => !a.startsWith('--')) || 'up'; +const upTo = (args.find(a => a.startsWith('--up-to=')) || '').split('=')[1] || null; + +function connect() { + return new Client({ + host: process.env.DB_HOST, + port: parseInt(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + ssl: false + }); +} + +async function showStatus(client) { + const st = await status(client); + console.log(`\n${process.env.DB_NAME} (${st.applied.length} applied, ${st.pending.length} pending)\n`); + for (const m of st.onDisk) { + const hit = st.applied.find(a => a.filename === m.filename); + const mark = !hit ? 'PENDING' + : st.modified.includes(m.filename) ? 'MODIFIED' + : 'applied'; + const when = hit ? hit.applied_at.toISOString().slice(0, 19).replace('T', ' ') : ''; + console.log(` ${mark.padEnd(9)} ${m.filename.padEnd(40)} ${when}`); + } + for (const f of st.missing) console.log(` MISSING ${f.padEnd(40)} recorded as applied but not on disk`); + if (st.modified.length) console.log(`\n ! modified after being applied: ${st.modified.join(', ')}`); + if (st.outOfOrder.length) console.log(` ! would apply out of order: ${st.outOfOrder.join(', ')}`); + console.log(''); + return st; +} + +(async () => { + const client = connect(); + await client.connect(); + try { + const user = `${os.userInfo().username}@${os.hostname()}`; + if (command === 'status') { + const st = await showStatus(client); + process.exitCode = (st.modified.length || st.missing.length) ? 1 : 0; + } else if (command === 'baseline') { + await baseline(client, { user, upTo }); + await showStatus(client); + } else if (command === 'up') { + await migrate(client, { user }); + } else { + console.error(`Unknown command "${command}" — expected up, status, or baseline.`); + process.exitCode = 2; + } + } catch (err) { + console.error(`\nmigration error: ${err.message}\n`); + process.exitCode = 1; + } finally { + await client.end(); + } +})(); diff --git a/scripts/schema-dump.js b/scripts/schema-dump.js new file mode 100644 index 0000000..7ce0d8e --- /dev/null +++ b/scripts/schema-dump.js @@ -0,0 +1,50 @@ +#!/usr/bin/env node +// Regenerate setup_sql/schema.generated.sql — a readable snapshot of the current +// pf schema, for reference only. Migrations are the source of truth; this file is +// generated so it cannot drift from the database the way a hand-maintained +// schema file does. Never edit it, and never apply it to create a database. + +require('dotenv').config(); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const out = path.join(__dirname, '..', 'setup_sql', 'schema.generated.sql'); + +const result = spawnSync('pg_dump', [ + '--schema-only', '--no-owner', '--no-privileges', + '--schema=pf', + // per-version forecast tables are created at runtime — including them would + // make this file churn every time a version is created or dropped + '--exclude-table=pf.fc_*', + '-f', out +], { + stdio: 'inherit', + env: { + ...process.env, + PGHOST: process.env.DB_HOST, + PGPORT: process.env.DB_PORT || '5432', + PGDATABASE: process.env.DB_NAME, + PGUSER: process.env.DB_USER, + PGPASSWORD: process.env.DB_PASSWORD + } +}); + +if (result.error) { + console.error(`pg_dump failed: ${result.error.message}`); + process.exit(1); +} +if (result.status !== 0) process.exit(result.status); + +// Strip the lines pg_dump varies between runs — a random \restrict token and the +// server/client version banner — so regenerating an unchanged schema produces an +// identical file instead of a spurious diff. +const cleaned = fs.readFileSync(out, 'utf8') + .split('\n') + .filter(l => !/^\\(un)?restrict /.test(l)) + .filter(l => !/^-- Dumped (from|by) /.test(l)) + .join('\n') + .replace(/\n{3,}/g, '\n\n'); +fs.writeFileSync(out, cleaned); + +console.log(`wrote ${path.relative(process.cwd(), out)} (${cleaned.split('\n').length} lines)`); diff --git a/server.js b/server.js index f6776d5..cec2441 100644 --- a/server.js +++ b/server.js @@ -33,5 +33,38 @@ app.use('/api', require('./routes/operations')(pool)); app.use('/api', require('./routes/log')(pool)); +// Refuse to start against a database whose schema is behind the migrations on +// disk. Without this the mismatch surfaces later as a confusing query error +// ('column "x" does not exist') deep inside an unrelated request. +// Set PF_SKIP_MIGRATION_CHECK=1 to bypass. +async function checkSchema() { + if (process.env.PF_SKIP_MIGRATION_CHECK === '1') return; + const { status } = require('./lib/migrations'); + const client = await pool.connect(); + try { + const st = await status(client); + const problems = [ + st.pending.length && `${st.pending.length} pending: ${st.pending.map(m => m.filename).join(', ')}`, + st.modified.length && `modified after being applied: ${st.modified.join(', ')}`, + st.missing.length && `applied but missing from disk: ${st.missing.join(', ')}` + ].filter(Boolean); + if (problems.length) { + console.error('\n Database schema is out of date:'); + for (const p of problems) console.error(` - ${p}`); + console.error('\n Run "npm run migrate" (or "npm run migrate:status" for detail).'); + console.error(' For a database that already matches, "npm run migrate:baseline".\n'); + throw new Error('schema out of date'); + } + console.log(`schema up to date (${st.applied.length} migrations applied)`); + } finally { + client.release(); + } +} + const port = process.env.PORT || 3010; -app.listen(port, '0.0.0.0', () => console.log(`pf_app started on port ${port}`)); +checkSchema() + .then(() => app.listen(port, '0.0.0.0', () => console.log(`pf_app started on port ${port}`))) + .catch((err) => { + if (err.message !== 'schema out of date') console.error('startup failed:', err.message); + process.exit(1); + }); diff --git a/setup_sql/01_schema.sql b/setup_sql/01_schema.sql deleted file mode 100644 index 2a9e0d9..0000000 --- a/setup_sql/01_schema.sql +++ /dev/null @@ -1,75 +0,0 @@ --- Pivot Forecast schema install --- Run once against target database: psql -d -f setup_sql/01_schema.sql - -CREATE SCHEMA IF NOT EXISTS pf; - -CREATE TABLE IF NOT EXISTS pf.source ( - id serial PRIMARY KEY, - schema text NOT NULL, - tname text NOT NULL, - label text, - status text NOT NULL DEFAULT 'active', -- active | archived - default_layout jsonb, -- Perspective view config used as the per-source default - created_at timestamptz NOT NULL DEFAULT now(), - created_by text, - UNIQUE (schema, tname) -); - --- backfill columns for existing installs -ALTER TABLE pf.source ADD COLUMN IF NOT EXISTS default_layout jsonb; - --- pf.dim_period: run setup_sql/gen_dim_period.sql to create and populate - -CREATE TABLE IF NOT EXISTS pf.col_meta ( - id serial PRIMARY KEY, - source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE, - cname text NOT NULL, - label text, - role text NOT NULL DEFAULT 'ignore', -- dimension | value | units | date | ignore - is_key boolean NOT NULL DEFAULT false, -- true = usable in WHERE slice - dim_group text, -- groups functionally dependent columns - dim_period_col text, -- pf.dim_period column this dimension derives from - in_grain boolean NOT NULL DEFAULT false, -- true = column defines the display grain - opos integer, - UNIQUE (source_id, cname) -); - --- backfill columns for existing installs (must follow the CREATE above) -ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_group text; -ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS dim_period_col text; -ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS in_grain boolean NOT NULL DEFAULT false; - -CREATE TABLE IF NOT EXISTS pf.version ( - id serial PRIMARY KEY, - source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE RESTRICT, - name text NOT NULL, - description text, - status text NOT NULL DEFAULT 'open', -- open | closed - exclude_iters jsonb NOT NULL DEFAULT '["reference"]'::jsonb, - created_at timestamptz NOT NULL DEFAULT now(), - created_by text, - closed_at timestamptz, - closed_by text, - UNIQUE (source_id, name) -); - -CREATE TABLE IF NOT EXISTS pf.log ( - id bigserial PRIMARY KEY, - version_id integer NOT NULL REFERENCES pf.version(id) ON DELETE CASCADE, - pf_user text NOT NULL, - stamp timestamptz NOT NULL DEFAULT now(), - operation text NOT NULL, -- baseline | reference | scale | recode | clone - slice jsonb, - params jsonb, - note text -); - --- generated operation SQL per source, stored after col_meta is configured -CREATE TABLE IF NOT EXISTS pf.sql ( - id serial PRIMARY KEY, - source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE, - operation text NOT NULL, -- get_data | baseline | reference | scale | recode | clone | undo - sql text NOT NULL, - generated_at timestamptz NOT NULL DEFAULT now(), - UNIQUE (source_id, operation) -); diff --git a/setup_sql/README.md b/setup_sql/README.md new file mode 100644 index 0000000..e681c52 --- /dev/null +++ b/setup_sql/README.md @@ -0,0 +1,64 @@ +# Database setup + +## Migrations + +`migrations/*.sql` are applied in filename order and recorded in +`pf.schema_version`. They are the **source of truth** for the `pf` schema — there +is no hand-maintained current-state file to drift out of sync. + +```bash +npm run migrate # apply pending migrations +npm run migrate:status # what is applied, what is pending +npm run migrate:baseline # record pending as applied WITHOUT running them +``` + +`server.js` refuses to start when the database is behind, so drift surfaces at +boot rather than as `column "x" does not exist` inside an unrelated request. Set +`PF_SKIP_MIGRATION_CHECK=1` to bypass. + +### Fresh database + +```bash +npm run migrate +psql -d -f setup_sql/gen_dim_period.sql +``` + +### Existing database that already matches + +Use `baseline` so the runner does not try to re-create tables that exist: + +```bash +npm run migrate:baseline +``` + +To baseline only part of the way — the schema matches through `0003` but not +`0004` — pass `--up-to` and then migrate the rest: + +```bash +node scripts/migrate.js baseline --up-to=0003_col_meta_dim_group_period.sql +npm run migrate +``` + +## Writing a migration + +- Name it `NNNN_short_description.sql`, numbered after the highest existing file. +- One concern per file. Keep it forward-only; there are no down migrations. +- **Applied migrations are immutable.** The runner stores a checksum and refuses + to proceed if a file changes after being applied, because editing one means + databases silently disagree about what the schema is. To fix a mistake, add a + new migration. +- No `IF NOT EXISTS` guards on new migrations. The bookkeeping already guarantees + each runs once, and the guards hide ordering mistakes — the reason the old + `01_schema.sql` had `ALTER`s sitting above the `CREATE TABLE` they depended on, + broken for anyone installing from scratch. `0004` is the one exception, since it + was applied by hand before migrations existed. + +## Not migrations + +- **`gen_dim_period.sql`** — creates and populates `pf.dim_period`. It is a + parameterised data load (configurable fiscal year start month), not a schema + change, so it stays a script you run deliberately. +- **`pf.fc_{tname}_{version_id}`** — per-version forecast tables, created and + dropped at runtime by `routes/versions.js` from `col_meta`. Never migrated. +- **`schema.generated.sql`** — a `pg_dump` snapshot for reading, refreshed with + `npm run schema:dump`. Generated, never edited, never applied. diff --git a/setup_sql/migrations/0001_initial_schema.sql b/setup_sql/migrations/0001_initial_schema.sql new file mode 100644 index 0000000..bd7d731 --- /dev/null +++ b/setup_sql/migrations/0001_initial_schema.sql @@ -0,0 +1,63 @@ +-- Initial pf schema: sources, column metadata, versions, audit log, generated SQL. +-- +-- This is the schema as it stood before the additive columns in later migrations. +-- Columns added afterwards are declared once, in their own migration — not here. + +CREATE SCHEMA IF NOT EXISTS pf; + +CREATE TABLE pf.source ( + id serial PRIMARY KEY, + schema text NOT NULL, + tname text NOT NULL, + label text, + status text NOT NULL DEFAULT 'active', -- active | archived + created_at timestamptz NOT NULL DEFAULT now(), + created_by text, + UNIQUE (schema, tname) +); + +CREATE TABLE pf.col_meta ( + id serial PRIMARY KEY, + source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE, + cname text NOT NULL, + label text, + role text NOT NULL DEFAULT 'ignore', -- dimension | value | units | date | filter | ignore + is_key boolean NOT NULL DEFAULT false, -- true = usable in WHERE slice + opos integer, + UNIQUE (source_id, cname) +); + +CREATE TABLE pf.version ( + id serial PRIMARY KEY, + source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE RESTRICT, + name text NOT NULL, + description text, + status text NOT NULL DEFAULT 'open', -- open | closed + exclude_iters jsonb NOT NULL DEFAULT '["reference"]'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + created_by text, + closed_at timestamptz, + closed_by text, + UNIQUE (source_id, name) +); + +CREATE TABLE pf.log ( + id bigserial PRIMARY KEY, + version_id integer NOT NULL REFERENCES pf.version(id) ON DELETE CASCADE, + pf_user text NOT NULL, + stamp timestamptz NOT NULL DEFAULT now(), + operation text NOT NULL, -- baseline | reference | scale | recode | clone + slice jsonb, + params jsonb, + note text +); + +-- generated operation SQL per source, stored after col_meta is configured +CREATE TABLE pf.sql ( + id serial PRIMARY KEY, + source_id integer NOT NULL REFERENCES pf.source(id) ON DELETE CASCADE, + operation text NOT NULL, -- get_data | get_agg | baseline | reference | scale | recode | clone | undo + sql text NOT NULL, + generated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (source_id, operation) +); diff --git a/setup_sql/migrations/0002_source_default_layout.sql b/setup_sql/migrations/0002_source_default_layout.sql new file mode 100644 index 0000000..6d02be5 --- /dev/null +++ b/setup_sql/migrations/0002_source_default_layout.sql @@ -0,0 +1,4 @@ +-- Per-source default Perspective view config, applied when a version has no +-- saved layout of its own. + +ALTER TABLE pf.source ADD COLUMN default_layout jsonb; diff --git a/setup_sql/migrations/0003_col_meta_dim_group_period.sql b/setup_sql/migrations/0003_col_meta_dim_group_period.sql new file mode 100644 index 0000000..14411e6 --- /dev/null +++ b/setup_sql/migrations/0003_col_meta_dim_group_period.sql @@ -0,0 +1,7 @@ +-- dim_group groups functionally dependent columns (e.g. a date and the year/month +-- dimensions derived from it). dim_period_col maps such a dimension to a +-- pf.dim_period column, so date-adjacent values are derived by JOIN at load time +-- rather than copied raw from the source. + +ALTER TABLE pf.col_meta ADD COLUMN dim_group text; +ALTER TABLE pf.col_meta ADD COLUMN dim_period_col text; diff --git a/setup_sql/migrations/0004_col_meta_in_grain.sql b/setup_sql/migrations/0004_col_meta_in_grain.sql new file mode 100644 index 0000000..a1b37b1 --- /dev/null +++ b/setup_sql/migrations/0004_col_meta_in_grain.sql @@ -0,0 +1,8 @@ +-- in_grain flags dimension/date columns that define the display grain: the +-- forecast load is pre-aggregated to the flagged columns instead of shipping raw +-- rows. See pf_spec.md -> Display-grain pre-aggregation. +-- +-- Already applied by hand on the original dev database before migrations existed; +-- IF NOT EXISTS keeps replay safe there. New migrations should not need the guard. + +ALTER TABLE pf.col_meta ADD COLUMN IF NOT EXISTS in_grain boolean NOT NULL DEFAULT false; diff --git a/setup_sql/schema.generated.sql b/setup_sql/schema.generated.sql new file mode 100644 index 0000000..4dcdfd4 --- /dev/null +++ b/setup_sql/schema.generated.sql @@ -0,0 +1,384 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET transaction_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: pf; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pf; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: col_meta; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.col_meta ( + id integer NOT NULL, + source_id integer NOT NULL, + cname text NOT NULL, + label text, + role text DEFAULT 'ignore'::text NOT NULL, + is_key boolean DEFAULT false NOT NULL, + opos integer, + dim_group text, + dim_period_col text, + in_grain boolean DEFAULT false NOT NULL +); + +-- +-- Name: col_meta_id_seq; Type: SEQUENCE; Schema: pf; Owner: - +-- + +CREATE SEQUENCE pf.col_meta_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +-- +-- Name: col_meta_id_seq; Type: SEQUENCE OWNED BY; Schema: pf; Owner: - +-- + +ALTER SEQUENCE pf.col_meta_id_seq OWNED BY pf.col_meta.id; + +-- +-- Name: dim_period; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.dim_period ( + sdat date NOT NULL, + edat date NOT NULL, + drange daterange NOT NULL, + ndays integer NOT NULL, + cal_year integer NOT NULL, + cal_quarter integer NOT NULL, + cal_month integer NOT NULL, + cal_month_abbr text NOT NULL, + cal_month_name text NOT NULL, + cal_label text NOT NULL, + fisc_year integer NOT NULL, + fisc_quarter integer NOT NULL, + fisc_quarter_label text NOT NULL, + fisc_month integer NOT NULL, + fisc_month_abbr text NOT NULL, + fisc_month_name text NOT NULL, + fisc_label text NOT NULL, + period_key text NOT NULL +); + +-- +-- Name: log; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.log ( + id bigint NOT NULL, + version_id integer NOT NULL, + pf_user text NOT NULL, + stamp timestamp with time zone DEFAULT now() NOT NULL, + operation text NOT NULL, + slice jsonb, + params jsonb, + note text +); + +-- +-- Name: log_id_seq; Type: SEQUENCE; Schema: pf; Owner: - +-- + +CREATE SEQUENCE pf.log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +-- +-- Name: log_id_seq; Type: SEQUENCE OWNED BY; Schema: pf; Owner: - +-- + +ALTER SEQUENCE pf.log_id_seq OWNED BY pf.log.id; + +-- +-- Name: schema_version; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.schema_version ( + filename text NOT NULL, + checksum text NOT NULL, + applied_at timestamp with time zone DEFAULT now() NOT NULL, + applied_by text +); + +-- +-- Name: source; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.source ( + id integer NOT NULL, + schema text NOT NULL, + tname text NOT NULL, + label text, + status text DEFAULT 'active'::text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + created_by text, + default_layout jsonb +); + +-- +-- Name: source_id_seq; Type: SEQUENCE; Schema: pf; Owner: - +-- + +CREATE SEQUENCE pf.source_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +-- +-- Name: source_id_seq; Type: SEQUENCE OWNED BY; Schema: pf; Owner: - +-- + +ALTER SEQUENCE pf.source_id_seq OWNED BY pf.source.id; + +-- +-- Name: sql; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.sql ( + id integer NOT NULL, + source_id integer NOT NULL, + operation text NOT NULL, + sql text NOT NULL, + generated_at timestamp with time zone DEFAULT now() NOT NULL +); + +-- +-- Name: sql_id_seq; Type: SEQUENCE; Schema: pf; Owner: - +-- + +CREATE SEQUENCE pf.sql_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +-- +-- Name: sql_id_seq; Type: SEQUENCE OWNED BY; Schema: pf; Owner: - +-- + +ALTER SEQUENCE pf.sql_id_seq OWNED BY pf.sql.id; + +-- +-- Name: version; Type: TABLE; Schema: pf; Owner: - +-- + +CREATE TABLE pf.version ( + id integer NOT NULL, + source_id integer NOT NULL, + name text NOT NULL, + description text, + status text DEFAULT 'open'::text NOT NULL, + exclude_iters jsonb DEFAULT '["reference"]'::jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + created_by text, + closed_at timestamp with time zone, + closed_by text +); + +-- +-- Name: version_id_seq; Type: SEQUENCE; Schema: pf; Owner: - +-- + +CREATE SEQUENCE pf.version_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +-- +-- Name: version_id_seq; Type: SEQUENCE OWNED BY; Schema: pf; Owner: - +-- + +ALTER SEQUENCE pf.version_id_seq OWNED BY pf.version.id; + +-- +-- Name: col_meta id; Type: DEFAULT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.col_meta ALTER COLUMN id SET DEFAULT nextval('pf.col_meta_id_seq'::regclass); + +-- +-- Name: log id; Type: DEFAULT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.log ALTER COLUMN id SET DEFAULT nextval('pf.log_id_seq'::regclass); + +-- +-- Name: source id; Type: DEFAULT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.source ALTER COLUMN id SET DEFAULT nextval('pf.source_id_seq'::regclass); + +-- +-- Name: sql id; Type: DEFAULT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.sql ALTER COLUMN id SET DEFAULT nextval('pf.sql_id_seq'::regclass); + +-- +-- Name: version id; Type: DEFAULT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.version ALTER COLUMN id SET DEFAULT nextval('pf.version_id_seq'::regclass); + +-- +-- Name: col_meta col_meta_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.col_meta + ADD CONSTRAINT col_meta_pkey PRIMARY KEY (id); + +-- +-- Name: col_meta col_meta_source_id_cname_key; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.col_meta + ADD CONSTRAINT col_meta_source_id_cname_key UNIQUE (source_id, cname); + +-- +-- Name: dim_period dim_period_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.dim_period + ADD CONSTRAINT dim_period_pkey PRIMARY KEY (sdat); + +-- +-- Name: log log_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.log + ADD CONSTRAINT log_pkey PRIMARY KEY (id); + +-- +-- Name: schema_version schema_version_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.schema_version + ADD CONSTRAINT schema_version_pkey PRIMARY KEY (filename); + +-- +-- Name: source source_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.source + ADD CONSTRAINT source_pkey PRIMARY KEY (id); + +-- +-- Name: source source_schema_tname_key; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.source + ADD CONSTRAINT source_schema_tname_key UNIQUE (schema, tname); + +-- +-- Name: sql sql_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.sql + ADD CONSTRAINT sql_pkey PRIMARY KEY (id); + +-- +-- Name: sql sql_source_id_operation_key; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.sql + ADD CONSTRAINT sql_source_id_operation_key UNIQUE (source_id, operation); + +-- +-- Name: version version_pkey; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.version + ADD CONSTRAINT version_pkey PRIMARY KEY (id); + +-- +-- Name: version version_source_id_name_key; Type: CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.version + ADD CONSTRAINT version_source_id_name_key UNIQUE (source_id, name); + +-- +-- Name: dim_period_cal_idx; Type: INDEX; Schema: pf; Owner: - +-- + +CREATE INDEX dim_period_cal_idx ON pf.dim_period USING btree (cal_year, cal_month); + +-- +-- Name: dim_period_drange_idx; Type: INDEX; Schema: pf; Owner: - +-- + +CREATE INDEX dim_period_drange_idx ON pf.dim_period USING gist (drange); + +-- +-- Name: dim_period_fisc_idx; Type: INDEX; Schema: pf; Owner: - +-- + +CREATE INDEX dim_period_fisc_idx ON pf.dim_period USING btree (fisc_year, fisc_month); + +-- +-- Name: col_meta col_meta_source_id_fkey; Type: FK CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.col_meta + ADD CONSTRAINT col_meta_source_id_fkey FOREIGN KEY (source_id) REFERENCES pf.source(id) ON DELETE CASCADE; + +-- +-- Name: log log_version_id_fkey; Type: FK CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.log + ADD CONSTRAINT log_version_id_fkey FOREIGN KEY (version_id) REFERENCES pf.version(id) ON DELETE CASCADE; + +-- +-- Name: sql sql_source_id_fkey; Type: FK CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.sql + ADD CONSTRAINT sql_source_id_fkey FOREIGN KEY (source_id) REFERENCES pf.source(id) ON DELETE CASCADE; + +-- +-- Name: version version_source_id_fkey; Type: FK CONSTRAINT; Schema: pf; Owner: - +-- + +ALTER TABLE ONLY pf.version + ADD CONSTRAINT version_source_id_fkey FOREIGN KEY (source_id) REFERENCES pf.source(id) ON DELETE RESTRICT; + +-- +-- PostgreSQL database dump complete +-- +