Ship rows pre-aggregated to the grain the pivot displays instead of raw
forecast rows. This is Path B from pf_perspective_options.md: it keeps
Perspective's native WASM engine — so expand/collapse/depth/sort/filter
all still work — and fixes load time by cutting rows, not transport.
Measured on pf.fc_osm_stack_20 at pending_rep x customer x smon:
534,902 -> 6,154 rows (~87x), pf_gkey unique across all 6,154, and both
measures reconcile exactly to the raw totals.
The grain is static: flagged once per source in Setup and baked into the
stored pf.sql templates, so load and operations agree by construction.
Sources with no flagged column keep the previous raw-row behaviour, so
this is backward compatible.
- pf.col_meta gains in_grain; grainOf() in lib/sql_generator.js is the
single definition of the grain and is reused by routes/log.js.
- New get_agg template + GET /api/versions/:id/agg, generated only when a
grain is defined. Regenerating drops templates no longer produced, so
clearing the grain falls back to /data.
- scale/recode/clone now aggregate their own new rows to grain before
returning. Because pf_logid is part of pf_gkey those keys are always
new, so table.update() appends and the view re-sums — the Excel
pivot-cache pattern, no bucket recomputation.
- Undo reports pf_gkeys (RETURNING cannot take DISTINCT, so the delete
feeds a CTE that reduces to distinct keys); the client removes those
index values and the view re-sums.
- pf_gkey is concat_ws(chr(31), COALESCE(col::text, chr(30)), ...).
The separator and NULL sentinel are load-bearing: plain concat_ws skips
NULLs, so ('a',NULL) and (NULL,'a') would collide and silently merge two
groups into one indexed row.
- Forecast.jsx reads col_meta first to pick /agg vs /data; the Arrow
streaming logic is extracted to fetchArrow() since both share it.
- Setup.jsx gains a grain checkbox and shows the resulting grain.
- 01_schema.sql: move the col_meta ALTERs after its CREATE TABLE — they
referenced the table before it existed on a fresh install.
All six generated statements verified to plan against the real forecast
table; the in_grain column has been added to the dev database.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
135 lines
5.7 KiB
JavaScript
135 lines
5.7 KiB
JavaScript
const express = require('express');
|
|
const { grainOf } = require('../lib/sql_generator');
|
|
const { fcTable } = require('../lib/utils');
|
|
|
|
module.exports = function(pool) {
|
|
const router = express.Router();
|
|
|
|
// list log entries for a version, newest first, with row counts and value/units totals
|
|
router.get('/versions/:id/log', async (req, res) => {
|
|
const versionId = parseInt(req.params.id);
|
|
try {
|
|
const verResult = await pool.query(
|
|
`SELECT v.*, s.tname, s.id AS source_id FROM pf.version v JOIN pf.source s ON s.id = v.source_id WHERE v.id = $1`,
|
|
[versionId]
|
|
);
|
|
if (!verResult.rows.length) return res.status(404).json({ error: 'Version not found' });
|
|
const { tname, source_id } = verResult.rows[0];
|
|
const table = fcTable(tname, versionId);
|
|
|
|
const colMeta = await pool.query(
|
|
`SELECT cname, role FROM pf.col_meta WHERE source_id = $1 AND role IN ('value', 'units')`,
|
|
[source_id]
|
|
);
|
|
const valueCol = colMeta.rows.find(c => c.role === 'value')?.cname;
|
|
const unitsCol = colMeta.rows.find(c => c.role === 'units')?.cname;
|
|
|
|
const aggCols = [
|
|
`count(f.pf_id)::int AS row_count`,
|
|
valueCol ? `sum(f."${valueCol}")::float8 AS value_total` : `NULL::float8 AS value_total`,
|
|
unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total`
|
|
].join(', ');
|
|
|
|
const result = await pool.query(`
|
|
SELECT l.*, ${aggCols},
|
|
$2::text AS value_col,
|
|
$3::text AS units_col
|
|
FROM pf.log l
|
|
LEFT JOIN ${table} f ON f.pf_logid = l.id
|
|
WHERE l.version_id = $1
|
|
GROUP BY l.id
|
|
ORDER BY l.id DESC
|
|
`, [versionId, valueCol || null, unitsCol || null]);
|
|
res.json(result.rows);
|
|
} catch (err) {
|
|
console.error(err);
|
|
res.status(err.status || 500).json({ error: err.message });
|
|
}
|
|
});
|
|
|
|
// undo a log entry — delete all fc rows with this logid, then delete the log entry
|
|
router.delete('/log/:logid', async (req, res) => {
|
|
const logId = parseInt(req.params.logid);
|
|
try {
|
|
const logResult = await pool.query(`
|
|
SELECT l.*, v.status, s.tname, v.id AS version_id, v.source_id
|
|
FROM pf.log l
|
|
JOIN pf.version v ON v.id = l.version_id
|
|
JOIN pf.source s ON s.id = v.source_id
|
|
WHERE l.id = $1
|
|
`, [logId]);
|
|
if (!logResult.rows.length) return res.status(404).json({ error: 'Log entry not found' });
|
|
const log = logResult.rows[0];
|
|
if (log.status === 'closed') return res.status(403).json({ error: 'Version is closed' });
|
|
const table = fcTable(log.tname, log.version_id);
|
|
|
|
// In grain mode the client's table is indexed on pf_gkey, so undo has to
|
|
// report the grain keys to remove rather than raw pf_ids. The keys are
|
|
// distinct while rows_deleted still counts the raw rows removed.
|
|
const colMeta = await pool.query(
|
|
`SELECT cname, role, in_grain, opos FROM pf.col_meta WHERE source_id = $1 ORDER BY opos`,
|
|
[log.source_id]
|
|
);
|
|
const grain = grainOf(colMeta.rows);
|
|
|
|
const client = await pool.connect();
|
|
try {
|
|
await client.query('BEGIN');
|
|
const deleted = grain
|
|
? await client.query(`
|
|
WITH
|
|
del AS (
|
|
DELETE FROM ${table}
|
|
WHERE pf_logid = $1
|
|
RETURNING ${grain.groupCols().join(', ')}
|
|
)
|
|
SELECT
|
|
count(*)::int AS rows_deleted
|
|
,array_agg(DISTINCT ${grain.key()}) AS pf_gkeys
|
|
FROM del
|
|
`, [logId])
|
|
: await client.query(
|
|
`DELETE FROM ${table} WHERE pf_logid = $1 RETURNING pf_id`, [logId]
|
|
);
|
|
await client.query('DELETE FROM pf.log WHERE id = $1', [logId]);
|
|
await client.query('COMMIT');
|
|
res.json(grain
|
|
? {
|
|
rows_deleted: deleted.rows[0].rows_deleted,
|
|
pf_gkeys: deleted.rows[0].pf_gkeys || []
|
|
}
|
|
: {
|
|
rows_deleted: deleted.rowCount,
|
|
pf_ids: deleted.rows.map(r => r.pf_id)
|
|
});
|
|
} catch (err) {
|
|
await client.query('ROLLBACK');
|
|
throw err;
|
|
} finally {
|
|
client.release();
|
|
}
|
|
} catch (err) {
|
|
console.error(err);
|
|
res.status(err.status || 500).json({ error: err.message });
|
|
}
|
|
});
|
|
|
|
// update the note on a log entry
|
|
router.patch('/log/:logid', async (req, res) => {
|
|
const logId = parseInt(req.params.logid);
|
|
const { note } = req.body;
|
|
try {
|
|
const result = await pool.query(
|
|
`UPDATE pf.log SET note = $1 WHERE id = $2 RETURNING *`, [note ?? null, logId]
|
|
);
|
|
if (!result.rows.length) return res.status(404).json({ error: 'Log entry not found' });
|
|
res.json(result.rows[0]);
|
|
} catch (err) {
|
|
console.error(err);
|
|
res.status(err.status || 500).json({ error: err.message });
|
|
}
|
|
});
|
|
|
|
return router;
|
|
};
|