Stop totalling log entries the change log does not show

Filtering the loads out in the dialog left the server still joining the
whole forecast table for them and discarding the answer. ?kind=adjustments
moves the filter into the WHERE, so they never enter the join: on version 29
the join goes from 2,556,821 rows to 25, and the query from 1666ms to 719ms.

Still 719ms, because nothing indexes pf_logid and it stays a sequential scan
of 2.5M rows. New forecast tables now get an index on it. That column is how
every entry-level operation finds its rows -- undo deletes by it, this
aggregate groups by it, and it is part of the grain key -- so the scan was
being paid on all of them.

Existing tables predate the index and still scan; fc_osm_skinny_29 would
want it added by hand.

The route keeps returning everything by default: Baseline.jsx lists the
segments from the same endpoint, and the ledger's tag lookup needs every
entry to label its lines.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-09-16 23:31:49 -04:00
parent ad02aba8fb
commit 5376c25e04
3 changed files with 23 additions and 2 deletions

View File

@ -30,6 +30,17 @@ module.exports = function(pool) {
unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total` unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total`
].join(', '); ].join(', ');
// ?kind=adjustments drops the baseline and reference entries. That is not
// only about what gets listed: the aggregate below joins the whole
// forecast table, and on a real version the load entries own almost
// every row of it -- 2.5M against a few thousand for the adjustments.
// Filtering in the WHERE keeps them out of the join rather than
// totalling them and discarding the answer.
const adjustmentsOnly = req.query.kind === 'adjustments';
const opFilter = adjustmentsOnly
? `AND l.operation NOT IN ('baseline', 'reference')`
: '';
const result = await pool.query(` const result = await pool.query(`
SELECT l.*, ${aggCols}, SELECT l.*, ${aggCols},
$2::text AS value_col, $2::text AS value_col,
@ -37,6 +48,7 @@ module.exports = function(pool) {
FROM pf.log l FROM pf.log l
LEFT JOIN ${table} f ON f.pf_logid = l.id LEFT JOIN ${table} f ON f.pf_logid = l.id
WHERE l.version_id = $1 WHERE l.version_id = $1
${opFilter}
GROUP BY l.id GROUP BY l.id
ORDER BY l.id DESC ORDER BY l.id DESC
`, [versionId, valueCol || null, unitsCol || null]); `, [versionId, valueCol || null, unitsCol || null]);

View File

@ -102,6 +102,14 @@ ${colDefs},
`; `;
await client.query(ddl); await client.query(ddl);
// pf_logid is how every entry-level operation finds its rows: undo
// deletes by it, the change log aggregates by it, and it is part of the
// grain key. Without an index each of those is a sequential scan of the
// whole forecast table -- 2.5M rows to total two adjustments.
await client.query(
`CREATE INDEX ${table.split('.').pop()}_logid_idx ON ${table} (pf_logid)`
);
await client.query('COMMIT'); await client.query('COMMIT');
res.status(201).json({ ...version, fc_table: table }); res.status(201).json({ ...version, fc_table: table });
} catch (err) { } catch (err) {

View File

@ -1162,12 +1162,13 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
setShowLog(true) setShowLog(true)
setLogLoading(true) setLogLoading(true)
try { try {
const data = await fetch(`/api/versions/${versionId}/log`).then(r => r.json())
// Baseline and reference loads are segment construction, not forecasting. // Baseline and reference loads are segment construction, not forecasting.
// They are managed in the Baseline view, where they can be edited in place // They are managed in the Baseline view, where they can be edited in place
// and their date ranges seen; listing them here only buries the adjustments // and their date ranges seen; listing them here only buries the adjustments
// this log is for -- and offers an undo that would silently gut the version. // this log is for -- and offers an undo that would silently gut the version.
setLogEntries(data.filter(e => !['baseline', 'reference'].includes(e.operation))) // Asked for by kind so the server does not total rows we are not showing.
const data = await fetch(`/api/versions/${versionId}/log?kind=adjustments`).then(r => r.json())
setLogEntries(data)
} catch (err) { } catch (err) {
flash(err.message, 'error') flash(err.message, 'error')
} finally { } finally {