pf_app/routes/log.js
Paul Trowbridge 654a368672 Add static display-grain pre-aggregation (col_meta.in_grain)
Ship rows pre-aggregated to the grain the pivot displays instead of raw
forecast rows. This is Path B from pf_perspective_options.md: it keeps
Perspective's native WASM engine — so expand/collapse/depth/sort/filter
all still work — and fixes load time by cutting rows, not transport.

Measured on pf.fc_osm_stack_20 at pending_rep x customer x smon:
534,902 -> 6,154 rows (~87x), pf_gkey unique across all 6,154, and both
measures reconcile exactly to the raw totals.

The grain is static: flagged once per source in Setup and baked into the
stored pf.sql templates, so load and operations agree by construction.
Sources with no flagged column keep the previous raw-row behaviour, so
this is backward compatible.

- pf.col_meta gains in_grain; grainOf() in lib/sql_generator.js is the
  single definition of the grain and is reused by routes/log.js.
- New get_agg template + GET /api/versions/:id/agg, generated only when a
  grain is defined. Regenerating drops templates no longer produced, so
  clearing the grain falls back to /data.
- scale/recode/clone now aggregate their own new rows to grain before
  returning. Because pf_logid is part of pf_gkey those keys are always
  new, so table.update() appends and the view re-sums — the Excel
  pivot-cache pattern, no bucket recomputation.
- Undo reports pf_gkeys (RETURNING cannot take DISTINCT, so the delete
  feeds a CTE that reduces to distinct keys); the client removes those
  index values and the view re-sums.
- pf_gkey is concat_ws(chr(31), COALESCE(col::text, chr(30)), ...).
  The separator and NULL sentinel are load-bearing: plain concat_ws skips
  NULLs, so ('a',NULL) and (NULL,'a') would collide and silently merge two
  groups into one indexed row.
- Forecast.jsx reads col_meta first to pick /agg vs /data; the Arrow
  streaming logic is extracted to fetchArrow() since both share it.
- Setup.jsx gains a grain checkbox and shows the resulting grain.
- 01_schema.sql: move the col_meta ALTERs after its CREATE TABLE — they
  referenced the table before it existed on a fresh install.

All six generated statements verified to plan against the real forecast
table; the in_grain column has been added to the dev database.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-18 16:25:38 -04:00

135 lines
5.7 KiB
JavaScript

const express = require('express');
const { grainOf } = require('../lib/sql_generator');
const { fcTable } = require('../lib/utils');
module.exports = function(pool) {
const router = express.Router();
// list log entries for a version, newest first, with row counts and value/units totals
router.get('/versions/:id/log', async (req, res) => {
const versionId = parseInt(req.params.id);
try {
const verResult = await pool.query(
`SELECT v.*, s.tname, s.id AS source_id FROM pf.version v JOIN pf.source s ON s.id = v.source_id WHERE v.id = $1`,
[versionId]
);
if (!verResult.rows.length) return res.status(404).json({ error: 'Version not found' });
const { tname, source_id } = verResult.rows[0];
const table = fcTable(tname, versionId);
const colMeta = await pool.query(
`SELECT cname, role FROM pf.col_meta WHERE source_id = $1 AND role IN ('value', 'units')`,
[source_id]
);
const valueCol = colMeta.rows.find(c => c.role === 'value')?.cname;
const unitsCol = colMeta.rows.find(c => c.role === 'units')?.cname;
const aggCols = [
`count(f.pf_id)::int AS row_count`,
valueCol ? `sum(f."${valueCol}")::float8 AS value_total` : `NULL::float8 AS value_total`,
unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total`
].join(', ');
const result = await pool.query(`
SELECT l.*, ${aggCols},
$2::text AS value_col,
$3::text AS units_col
FROM pf.log l
LEFT JOIN ${table} f ON f.pf_logid = l.id
WHERE l.version_id = $1
GROUP BY l.id
ORDER BY l.id DESC
`, [versionId, valueCol || null, unitsCol || null]);
res.json(result.rows);
} catch (err) {
console.error(err);
res.status(err.status || 500).json({ error: err.message });
}
});
// undo a log entry — delete all fc rows with this logid, then delete the log entry
router.delete('/log/:logid', async (req, res) => {
const logId = parseInt(req.params.logid);
try {
const logResult = await pool.query(`
SELECT l.*, v.status, s.tname, v.id AS version_id, v.source_id
FROM pf.log l
JOIN pf.version v ON v.id = l.version_id
JOIN pf.source s ON s.id = v.source_id
WHERE l.id = $1
`, [logId]);
if (!logResult.rows.length) return res.status(404).json({ error: 'Log entry not found' });
const log = logResult.rows[0];
if (log.status === 'closed') return res.status(403).json({ error: 'Version is closed' });
const table = fcTable(log.tname, log.version_id);
// In grain mode the client's table is indexed on pf_gkey, so undo has to
// report the grain keys to remove rather than raw pf_ids. The keys are
// distinct while rows_deleted still counts the raw rows removed.
const colMeta = await pool.query(
`SELECT cname, role, in_grain, opos FROM pf.col_meta WHERE source_id = $1 ORDER BY opos`,
[log.source_id]
);
const grain = grainOf(colMeta.rows);
const client = await pool.connect();
try {
await client.query('BEGIN');
const deleted = grain
? await client.query(`
WITH
del AS (
DELETE FROM ${table}
WHERE pf_logid = $1
RETURNING ${grain.groupCols().join(', ')}
)
SELECT
count(*)::int AS rows_deleted
,array_agg(DISTINCT ${grain.key()}) AS pf_gkeys
FROM del
`, [logId])
: await client.query(
`DELETE FROM ${table} WHERE pf_logid = $1 RETURNING pf_id`, [logId]
);
await client.query('DELETE FROM pf.log WHERE id = $1', [logId]);
await client.query('COMMIT');
res.json(grain
? {
rows_deleted: deleted.rows[0].rows_deleted,
pf_gkeys: deleted.rows[0].pf_gkeys || []
}
: {
rows_deleted: deleted.rowCount,
pf_ids: deleted.rows.map(r => r.pf_id)
});
} catch (err) {
await client.query('ROLLBACK');
throw err;
} finally {
client.release();
}
} catch (err) {
console.error(err);
res.status(err.status || 500).json({ error: err.message });
}
});
// update the note on a log entry
router.patch('/log/:logid', async (req, res) => {
const logId = parseInt(req.params.logid);
const { note } = req.body;
try {
const result = await pool.query(
`UPDATE pf.log SET note = $1 WHERE id = $2 RETURNING *`, [note ?? null, logId]
);
if (!result.rows.length) return res.status(404).json({ error: 'Log entry not found' });
res.json(result.rows[0]);
} catch (err) {
console.error(err);
res.status(err.status || 500).json({ error: err.message });
}
});
return router;
};