pt/pf_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete recode values from the version, not the source view

Browse Source 
Pointing completion at the source made every keystroke a 76-second query:
gs.osm_skinny is a plain view over rlarp.osm_stack, so ILIKE '%1601%'
scanned 6.9M rows and read 1.3M buffers off disk, 64s of it I/O. Nothing had
called that endpoint before, so the cost only appeared once a debounced
input was wired to it.

The source is the wrong list anyway. It reaches back over all of history and
would offer parts discontinued years ago; the version holds what was
actually loaded, which is what the forecast is being written against.

So GET /versions/:id/values/:col reads the version's own forecast table --
2.0s for 11,290 parts on fc_osm_skinny_29 -- and holds the result in memory,
keyed on that version's latest pf.log id. Any load, adjustment or undo moves
the id and the next request rebuilds, so nothing has to remember to
invalidate. Filtering happens over the cached array, so typing costs one
small max(id) query.

The column name is interpolated into the DISTINCT, so it is checked against
col_meta first.

The source-side endpoint keeps its new q/limit but no longer has a caller;
its ILIKE path against a view is the expensive one and should stay unused.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-09-17 00:03:05 -04:00
parent b65da53360
commit 812678bb7f
3 changed files with 75 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
routes/versions.js
View File

@ -127,6 +127,70 @@ ${colDefs},
// where this version's writes actually land: the physical forecast table,
// its current row count, and the source table rows are read from.
// Surfaced in the status bar so the write target is never a mystery.
// Distinct values of a dimension as they appear in one version's forecast
// table, for completing recode and clone.
//
// Deliberately not the source: the source view reaches back over all of
// history, so completing from it offers parts discontinued years ago. The
// version holds what was actually loaded, which is what a forecast is being
// written against.
//
// Held in memory because the scan is not cheap -- 2.0s for 11,290 parts across
// 2.5M rows on fc_osm_skinny_29 -- and completion is typed into. Keyed on the
// version's latest log id, so any load, adjustment or undo rebuilds it on the
// next request without anything having to remember to invalidate.
const valueCache = new Map();
router.get('/versions/:id/values/:col', async (req, res) => {
const versionId = parseInt(req.params.id);
const col = req.params.col;
try {
const verResult = await pool.query(`
SELECT v.id, s.tname, s.id AS source_id
FROM pf.version v JOIN pf.source s ON s.id = v.source_id
WHERE v.id = $1
`, [versionId]);
if (!verResult.rows.length) return res.status(404).json({ error: 'Version not found' });
const { tname, source_id } = verResult.rows[0];
// the column name is interpolated, so it has to be one col_meta names
const okCol = await pool.query(`
SELECT 1 FROM pf.col_meta
WHERE source_id = $1 AND cname = $2 AND role IN ('dimension', 'date')
`, [source_id, col]);
if (!okCol.rows.length) {
return res.status(400).json({ error: `"${col}" is not a dimension on this source` });
}
const table = fcTable(tname, versionId);
const { rows: [{ rev }] } = await pool.query(
`SELECT coalesce(max(id), 0)::text AS rev FROM pf.log WHERE version_id = $1`,
[versionId]
);
const key = `${versionId}:${col}`;
let entry = valueCache.get(key);
if (!entry || entry.rev !== rev) {
const { rows } = await pool.query(
`SELECT DISTINCT "${col}"::text AS val FROM ${table}
WHERE "${col}" IS NOT NULL ORDER BY 1`
);
entry = { rev, values: rows.map(r => r.val) };
valueCache.set(key, entry);
}
const q = (req.query.q || '').trim().toLowerCase();
const limit = Math.min(parseInt(req.query.limit) || 50, 500);
const picked = q
? entry.values.filter(v => v.toLowerCase().includes(q))
: entry.values;
res.json(picked.slice(0, limit));
} catch (err) {
console.error(err);
res.status(err.status || 500).json({ error: err.message });
}
});
router.get('/versions/:id/table-info', async (req, res) => {
try {
const verResult = await pool.query(`

20
ui/src/components/OperationPanel.jsx
View File

@ -541,24 +541,24 @@ function ScaleLedger({ currentTotals, scaleInputs, setScaleInputs, scalePlug, se
// Completion for a key dimension. The list is fetched as you type rather than up
// front: part alone has 11,290 distinct values, and a native datalist given all of
// them is slow to open and no easier to read than a short filtered one.
function DimValueInput({ col, sourceId, value, onChange, onBlur, className }) {
function DimValueInput({ col, versionId, value, onChange, onBlur, className }) {
const [options, setOptions] = useState([])
const listId = `pf-vals-${col.cname}`
useEffect(() => {
if (!sourceId || !col.is_key) return
if (!versionId || !col.is_key) return
let cancelled = false
// wait for a pause in typing; every keystroke would otherwise be a query
// wait for a pause in typing; every keystroke would otherwise be a request
const t = setTimeout(async () => {
try {
const url = `/api/sources/${sourceId}/values/${encodeURIComponent(col.cname)}`
const url = `/api/versions/${versionId}/values/${encodeURIComponent(col.cname)}`
+ `?limit=50${value ? `&q=${encodeURIComponent(value)}` : ''}`
const rows = await fetch(url).then(r => r.ok ? r.json() : [])
if (!cancelled) setOptions(Array.isArray(rows) ? rows : [])
} catch { if (!cancelled) setOptions([]) }
}, 200)
return () => { cancelled = true; clearTimeout(t) }
}, [sourceId, col.cname, col.is_key, value])
}, [versionId, col.cname, col.is_key, value])
return (
<>
@ -580,7 +580,7 @@ function DimValueInput({ col, sourceId, value, onChange, onBlur, className }) {
// ── 2b. Recode / clone form ─────────────────────────────────────────────────
// Same pairing: the dimension's current value sits beside the box that replaces it.
function DimForm({ dimCols, setObj, setSet, slices, lookupDerivedCols, sourceId, extra }) {
function DimForm({ dimCols, setObj, setSet, slices, lookupDerivedCols, versionId, extra }) {
const multi = slices.length > 1
const first = slices[0] || {}
return (
@ -605,7 +605,7 @@ function DimForm({ dimCols, setObj, setSet, slices, lookupDerivedCols, sourceId,
<td className="pl-2 py-0.5">
<DimValueInput
col={c}
sourceId={sourceId}
versionId={versionId}
value={setObj[c.cname] || ''}
onChange={e => setSet(s => ({ ...s, [c.cname]: e.target.value }))}
onBlur={c.is_key && c.dim_group
@ -702,7 +702,7 @@ export default function OperationPanel({
cloneSet, setCloneSet,
cloneScale, setCloneScale,
cloneNote, setCloneNote,
dimCols, lookupDerivedCols, sourceId,
dimCols, lookupDerivedCols, versionId,
buildPayload, submitOp,
}) {
const hasSlice = slices.length > 0
@ -763,12 +763,12 @@ export default function OperationPanel({
)}
{activeOp === 'recode' && (
<DimForm dimCols={dimCols} setObj={recodeSet} setSet={setRecodeSet}
slices={slices} lookupDerivedCols={lookupDerivedCols} sourceId={sourceId}
slices={slices} lookupDerivedCols={lookupDerivedCols} versionId={versionId}
extra={<MovingTotal currentTotals={currentTotals} verb="Moving" />} />
)}
{activeOp === 'clone' && (
<DimForm dimCols={dimCols} setObj={cloneSet} setSet={setCloneSet}
slices={slices} lookupDerivedCols={lookupDerivedCols} sourceId={sourceId}
slices={slices} lookupDerivedCols={lookupDerivedCols} versionId={versionId}
extra={
<div className="flex flex-col gap-2">
<div className="flex items-center gap-2">

2
ui/src/views/Forecast.jsx
View File

@ -1257,7 +1257,7 @@ export default function Forecast({ sources = [], sourceId, versions = [], versio
cloneSet, setCloneSet,
cloneScale, setCloneScale,
cloneNote, setCloneNote,
dimCols, lookupDerivedCols, sourceId,
dimCols, lookupDerivedCols, versionId,
buildPayload, submitOp,
}