From c9b830b2864a2e83571f76170054a06d7e845751 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 15 Apr 2026 21:02:54 -0400 Subject: [PATCH] Add per-record overrides that survive reprocess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schema: - records.overrides JSONB column (ALTER TABLE, already applied) - apply_transformations merges overrides on top: data || rules || overrides - generate_source_view always includes id and _overridden columns - set_record_overrides(id, overrides): stores and immediately merges into transformed - clear_record_overrides(id): clears overrides then reprocesses record API: - PUT /records/:id/overrides — set overrides - DELETE /records/:id/overrides — clear and reprocess UI (Records page): - Rows are clickable; overridden rows highlighted amber - Side panel shows all transformed fields as editable inputs - Overridden fields highlighted amber with pencil indicator - Save stores overrides; Clear removes them and restores computed values - id and _overridden hidden from table display Co-Authored-By: Claude Sonnet 4.6 --- api/routes/records.js | 34 ++++ database/functions.sql | 2 +- database/queries/records.sql | 21 ++ database/queries/sources.sql | 2 +- ui/src/api.js | 3 + ui/src/pages/Records.jsx | 378 +++++++++++++++++++++++++---------- 6 files changed, 327 insertions(+), 113 deletions(-) diff --git a/api/routes/records.js b/api/routes/records.js index 63eb63e..b80399e 100644 --- a/api/routes/records.js +++ b/api/routes/records.js @@ -49,6 +49,40 @@ module.exports = (pool) => { } }); + // Set overrides for a record and immediately merge into transformed + router.put('/:id/overrides', async (req, res, next) => { + try { + const { overrides } = req.body; + if (!overrides || typeof overrides !== 'object') + return res.status(400).json({ error: 'overrides object required' }); + const result = await pool.query( + `SELECT * FROM set_record_overrides(${lit(parseInt(req.params.id))}, ${lit(overrides)})` + ); + if (result.rows.length === 0) return res.status(404).json({ error: 'Record not found' }); + res.json(result.rows[0]); + } catch (err) { + next(err); + } + }); + + // Clear overrides and reprocess that record to restore computed values + router.delete('/:id/overrides', async (req, res, next) => { + try { + const rec = await pool.query( + `SELECT * FROM clear_record_overrides(${lit(parseInt(req.params.id))})` + ); + if (rec.rows.length === 0) return res.status(404).json({ error: 'Record not found' }); + // Reprocess this record so transformed reflects rules/mappings without overrides + await pool.query( + `SELECT apply_transformations(${lit(rec.rows[0].source_name)}, ARRAY[${lit(parseInt(req.params.id))}::int], true)` + ); + const updated = await pool.query(`SELECT * FROM get_record(${lit(parseInt(req.params.id))})`); + res.json(updated.rows[0]); + } catch (err) { + next(err); + } + }); + // Delete record router.delete('/:id', async (req, res, next) => { try { diff --git a/database/functions.sql b/database/functions.sql index 3eace82..3301f7a 100644 --- a/database/functions.sql +++ b/database/functions.sql @@ -284,7 +284,7 @@ record_additions AS ( -- Update all qualifying records; records with no rule matches get transformed = data updated AS ( UPDATE dataflow.records rec - SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb), + SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb) || COALESCE(rec.overrides, '{}'::jsonb), transformed_at = CURRENT_TIMESTAMP FROM qualifying q LEFT JOIN record_additions ra ON ra.id = q.id diff --git a/database/queries/records.sql b/database/queries/records.sql index ea9b2af..f48e26b 100644 --- a/database/queries/records.sql +++ b/database/queries/records.sql @@ -39,6 +39,27 @@ RETURNS SETOF dataflow.records AS $$ LIMIT p_limit; $$ LANGUAGE sql STABLE; +-- ── Overrides ───────────────────────────────────────────────────────────────── + +-- Store manual overrides and immediately merge into transformed +CREATE OR REPLACE FUNCTION set_record_overrides(p_id INT, p_overrides JSONB) +RETURNS dataflow.records AS $$ + UPDATE dataflow.records + SET overrides = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END, + transformed = COALESCE(transformed, data) || COALESCE(p_overrides, '{}'::jsonb) + WHERE id = p_id + RETURNING *; +$$ LANGUAGE sql; + +-- Clear overrides; caller should reprocess to restore computed transformed value +CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INT) +RETURNS dataflow.records AS $$ + UPDATE dataflow.records + SET overrides = NULL + WHERE id = p_id + RETURNING *; +$$ LANGUAGE sql; + -- ── Delete ──────────────────────────────────────────────────────────────────── CREATE OR REPLACE FUNCTION delete_record(p_id BIGINT) diff --git a/database/queries/sources.sql b/database/queries/sources.sql index 7b2cefd..0c8cc32 100644 --- a/database/queries/sources.sql +++ b/database/queries/sources.sql @@ -188,7 +188,7 @@ BEGIN v_view := 'dfv.' || quote_ident(p_source_name); EXECUTE format('DROP VIEW IF EXISTS %s', v_view); v_sql := format( - 'CREATE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL', + 'CREATE VIEW %s AS SELECT id, overrides IS NOT NULL AS _overridden, %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL', v_view, v_cols, p_source_name ); EXECUTE v_sql; diff --git a/ui/src/api.js b/ui/src/api.js index ac2a106..a9c5a50 100644 --- a/ui/src/api.js +++ b/ui/src/api.js @@ -116,4 +116,7 @@ export const api = { // Records getRecords: (source, limit = 100, offset = 0) => request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`), + getRecord: (id) => request('GET', `/records/${id}`), + setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }), + clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`), } diff --git a/ui/src/pages/Records.jsx b/ui/src/pages/Records.jsx index 8f1476e..d8a96fa 100644 --- a/ui/src/pages/Records.jsx +++ b/ui/src/pages/Records.jsx @@ -2,6 +2,7 @@ import { useState, useEffect, useRef } from 'react' import { api } from '../api' const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/ +const HIDDEN_COLS = new Set(['id', '_overridden']) function formatVal(val) { if (val === null || val === undefined) return null @@ -30,12 +31,20 @@ export default function Records({ source }) { const debounceRef = useRef(null) const LIMIT = 100 + // Override panel + const [selectedRecord, setSelectedRecord] = useState(null) // full record from API + const [overrideDraft, setOverrideDraft] = useState({}) // { field: newValue } + const [panelLoading, setPanelLoading] = useState(false) + const [panelSaving, setPanelSaving] = useState(false) + const [panelMsg, setPanelMsg] = useState(null) + useEffect(() => { if (!source) return setOffset(0) setSort({ col: null, dir: 'asc' }) setFilters([]) setViewError(null) + setSelectedRecord(null) load(0, null, 'asc', []) }, [source]) @@ -46,8 +55,7 @@ export default function Records({ source }) { const res = await api.getViewData(source, LIMIT, off, col, dir, active) setExists(res.exists) setRows(res.rows) - if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0])) - else if (res.rows.length > 0) setCols(Object.keys(res.rows[0])) + if (res.rows.length > 0) setCols(Object.keys(res.rows[0])) } catch (err) { setViewError(err.message) } finally { @@ -70,7 +78,8 @@ export default function Records({ source }) { } function addFilter() { - setFilters(f => [...f, { col: cols[0] || '', pattern: '' }]) + const visCols = cols.filter(c => !HIDDEN_COLS.has(c)) + setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }]) } function removeFilter(i) { @@ -90,131 +99,278 @@ export default function Records({ source }) { function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) } function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) } + async function openPanel(row) { + const id = row.id + if (!id) return + setPanelLoading(true) + setPanelMsg(null) + setSelectedRecord(null) + setOverrideDraft({}) + try { + const rec = await api.getRecord(id) + setSelectedRecord(rec) + setOverrideDraft(rec.overrides || {}) + } catch (err) { + setPanelMsg({ text: err.message, ok: false }) + } finally { + setPanelLoading(false) + } + } + + function closePanel() { + setSelectedRecord(null) + setOverrideDraft({}) + setPanelMsg(null) + } + + async function handleSaveOverrides() { + if (!selectedRecord) return + setPanelSaving(true) + setPanelMsg(null) + try { + const updated = await api.setRecordOverrides(selectedRecord.id, overrideDraft) + setSelectedRecord(updated) + setOverrideDraft(updated.overrides || {}) + setPanelMsg({ text: 'Saved.', ok: true }) + // Refresh the row in the table + setRows(rs => rs.map(r => r.id === updated.id + ? { ...r, _overridden: updated.overrides != null } + : r + )) + } catch (err) { + setPanelMsg({ text: err.message, ok: false }) + } finally { + setPanelSaving(false) + } + } + + async function handleClearOverrides() { + if (!selectedRecord) return + setPanelSaving(true) + setPanelMsg(null) + try { + const updated = await api.clearRecordOverrides(selectedRecord.id) + setSelectedRecord(updated) + setOverrideDraft({}) + setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true }) + setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r)) + } catch (err) { + setPanelMsg({ text: err.message, ok: false }) + } finally { + setPanelSaving(false) + } + } + if (!source) return
Select a source first.
- const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols + const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c)) + const visCols = cols.filter(c => !HIDDEN_COLS.has(c)) + + // Fields available for override: keys from transformed + const transformedFields = selectedRecord?.transformed + ? Object.keys(selectedRecord.transformed) + : [] return ( -
-
-

Records — {source}

- {exists && rows.length > 0 && ( - dfv.{source} - )} -
- - {/* Filter bar */} - {exists !== false && displayCols.length > 0 && ( -
- {filters.map((f, i) => ( -
- - ~* - updateFilter(i, 'pattern', e.target.value)} - /> - -
- ))} - - {filters.length > 0 && ( - +
+
+
+

Records — {source}

+ {exists && rows.length > 0 && ( + dfv.{source} )}
- )} - {loading &&

Loading…

} + {/* Filter bar */} + {exists !== false && visCols.length > 0 && ( +
+ {filters.map((f, i) => ( +
+ + ~* + updateFilter(i, 'pattern', e.target.value)} + /> + +
+ ))} + + {filters.length > 0 && ( + + )} +
+ )} - {!loading && viewError && ( -

View error: {viewError} — check field types in Sources.

- )} + {loading &&

Loading…

} - {!loading && exists === false && ( -

- No view generated yet. Go to Sources, check fields as In view, then click Generate view. -

- )} + {!loading && viewError && ( +

View error: {viewError} — check field types in Sources.

+ )} - {!loading && exists && rows.length === 0 && ( -

- {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'} -

- )} + {!loading && exists === false && ( +

+ No view generated yet. Go to Sources, check fields as In view, then click Generate view. +

+ )} - {!loading && exists && rows.length > 0 && ( - <> -
- - - - {displayCols.map(col => { - const active = sort.col === col - return ( - - ) - })} - - - - {rows.map((row, i) => ( - - {displayCols.map((col, j) => { - const formatted = formatVal(row[col]) + {!loading && exists && rows.length === 0 && ( +

+ {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'} +

+ )} + + {!loading && exists && rows.length > 0 && ( + <> +
+
toggleSort(col)} - className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600" - > - {col} - - {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'} - -
+ + + {displayCols.map(col => { + const active = sort.col === col return ( - + ) })} - ))} - -
- {formatted === null ? — : formatted} - toggleSort(col)} + className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600" + > + {col} + + {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'} + +
+ + + {rows.map((row, i) => { + const isOverridden = row._overridden + const isSelected = selectedRecord?.id === row.id + return ( + openPanel(row)} + className={`border-t border-gray-50 cursor-pointer transition-colors + ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`} + > + {displayCols.map((col, j) => { + const formatted = formatVal(row[col]) + return ( + + {formatted === null ? — : formatted} + + ) + })} + + ) + })} + + +
+ +
+ + {offset + 1}–{offset + rows.length} + +
+ + )} +
+ + {/* Override panel */} + {(selectedRecord || panelLoading) && ( +
+
+ Override +
-
- - {offset + 1}–{offset + rows.length} - -
- + {panelLoading &&

Loading…

} + + {selectedRecord && !panelLoading && ( +
+ {panelMsg && ( +
+ {panelMsg.text} +
+ )} + +
+ Click any field to override its value. Overrides survive reprocess. +
+ +
+ {transformedFields.map(field => { + const currentVal = selectedRecord.transformed[field] + const isOverridden = field in overrideDraft + return ( +
+
+ {field} + {isOverridden && ✎} +
+ setOverrideDraft(d => ({ ...d, [field]: e.target.value }))} + onFocus={() => { + if (!(field in overrideDraft)) { + setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') })) + } + }} + /> +
+ ) + })} +
+ +
+ + {selectedRecord.overrides && ( + + )} +
+
+ )} +
)}
)