Stop totalling log entries the change log does not show
Filtering the loads out in the dialog left the server still joining the whole forecast table for them and discarding the answer. ?kind=adjustments moves the filter into the WHERE, so they never enter the join: on version 29 the join goes from 2,556,821 rows to 25, and the query from 1666ms to 719ms. Still 719ms, because nothing indexes pf_logid and it stays a sequential scan of 2.5M rows. New forecast tables now get an index on it. That column is how every entry-level operation finds its rows -- undo deletes by it, this aggregate groups by it, and it is part of the grain key -- so the scan was being paid on all of them. Existing tables predate the index and still scan; fc_osm_skinny_29 would want it added by hand. The route keeps returning everything by default: Baseline.jsx lists the segments from the same endpoint, and the ledger's tag lookup needs every entry to label its lines. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ad02aba8fb
commit
5376c25e04
@ -30,6 +30,17 @@ module.exports = function(pool) {
|
|||||||
unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total`
|
unitsCol ? `sum(f."${unitsCol}")::float8 AS units_total` : `NULL::float8 AS units_total`
|
||||||
].join(', ');
|
].join(', ');
|
||||||
|
|
||||||
|
// ?kind=adjustments drops the baseline and reference entries. That is not
|
||||||
|
// only about what gets listed: the aggregate below joins the whole
|
||||||
|
// forecast table, and on a real version the load entries own almost
|
||||||
|
// every row of it -- 2.5M against a few thousand for the adjustments.
|
||||||
|
// Filtering in the WHERE keeps them out of the join rather than
|
||||||
|
// totalling them and discarding the answer.
|
||||||
|
const adjustmentsOnly = req.query.kind === 'adjustments';
|
||||||
|
const opFilter = adjustmentsOnly
|
||||||
|
? `AND l.operation NOT IN ('baseline', 'reference')`
|
||||||
|
: '';
|
||||||
|
|
||||||
const result = await pool.query(`
|
const result = await pool.query(`
|
||||||
SELECT l.*, ${aggCols},
|
SELECT l.*, ${aggCols},
|
||||||
$2::text AS value_col,
|
$2::text AS value_col,
|
||||||
@ -37,6 +48,7 @@ module.exports = function(pool) {
|
|||||||
FROM pf.log l
|
FROM pf.log l
|
||||||
LEFT JOIN ${table} f ON f.pf_logid = l.id
|
LEFT JOIN ${table} f ON f.pf_logid = l.id
|
||||||
WHERE l.version_id = $1
|
WHERE l.version_id = $1
|
||||||
|
${opFilter}
|
||||||
GROUP BY l.id
|
GROUP BY l.id
|
||||||
ORDER BY l.id DESC
|
ORDER BY l.id DESC
|
||||||
`, [versionId, valueCol || null, unitsCol || null]);
|
`, [versionId, valueCol || null, unitsCol || null]);
|
||||||
|
|||||||
@ -102,6 +102,14 @@ ${colDefs},
|
|||||||
`;
|
`;
|
||||||
await client.query(ddl);
|
await client.query(ddl);
|
||||||
|
|
||||||
|
// pf_logid is how every entry-level operation finds its rows: undo
|
||||||
|
// deletes by it, the change log aggregates by it, and it is part of the
|
||||||
|
// grain key. Without an index each of those is a sequential scan of the
|
||||||
|
// whole forecast table -- 2.5M rows to total two adjustments.
|
||||||
|
await client.query(
|
||||||
|
`CREATE INDEX ${table.split('.').pop()}_logid_idx ON ${table} (pf_logid)`
|
||||||
|
);
|
||||||
|
|
||||||
await client.query('COMMIT');
|
await client.query('COMMIT');
|
||||||
res.status(201).json({ ...version, fc_table: table });
|
res.status(201).json({ ...version, fc_table: table });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@ -1162,12 +1162,13 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
|
|||||||
setShowLog(true)
|
setShowLog(true)
|
||||||
setLogLoading(true)
|
setLogLoading(true)
|
||||||
try {
|
try {
|
||||||
const data = await fetch(`/api/versions/${versionId}/log`).then(r => r.json())
|
|
||||||
// Baseline and reference loads are segment construction, not forecasting.
|
// Baseline and reference loads are segment construction, not forecasting.
|
||||||
// They are managed in the Baseline view, where they can be edited in place
|
// They are managed in the Baseline view, where they can be edited in place
|
||||||
// and their date ranges seen; listing them here only buries the adjustments
|
// and their date ranges seen; listing them here only buries the adjustments
|
||||||
// this log is for -- and offers an undo that would silently gut the version.
|
// this log is for -- and offers an undo that would silently gut the version.
|
||||||
setLogEntries(data.filter(e => !['baseline', 'reference'].includes(e.operation)))
|
// Asked for by kind so the server does not total rows we are not showing.
|
||||||
|
const data = await fetch(`/api/versions/${versionId}/log?kind=adjustments`).then(r => r.json())
|
||||||
|
setLogEntries(data)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
flash(err.message, 'error')
|
flash(err.message, 'error')
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user