Add per-record overrides that survive reprocess

Schema:
- records.overrides JSONB column (ALTER TABLE, already applied)
- apply_transformations merges overrides on top: data || rules || overrides
- generate_source_view always includes id and _overridden columns
- set_record_overrides(id, overrides): stores and immediately merges into transformed
- clear_record_overrides(id): clears overrides then reprocesses record

API:
- PUT  /records/:id/overrides — set overrides
- DELETE /records/:id/overrides — clear and reprocess

UI (Records page):
- Rows are clickable; overridden rows highlighted amber
- Side panel shows all transformed fields as editable inputs
- Overridden fields highlighted amber with pencil indicator
- Save stores overrides; Clear removes them and restores computed values
- id and _overridden hidden from table display

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-04-15 21:02:54 -04:00
parent 24675feb49
commit c9b830b286
6 changed files with 327 additions and 113 deletions

View File

@ -49,6 +49,40 @@ module.exports = (pool) => {
}
});
// Set overrides for a record and immediately merge into transformed
router.put('/:id/overrides', async (req, res, next) => {
try {
const { overrides } = req.body;
if (!overrides || typeof overrides !== 'object')
return res.status(400).json({ error: 'overrides object required' });
const result = await pool.query(
`SELECT * FROM set_record_overrides(${lit(parseInt(req.params.id))}, ${lit(overrides)})`
);
if (result.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
res.json(result.rows[0]);
} catch (err) {
next(err);
}
});
// Clear overrides and reprocess that record to restore computed values
router.delete('/:id/overrides', async (req, res, next) => {
try {
const rec = await pool.query(
`SELECT * FROM clear_record_overrides(${lit(parseInt(req.params.id))})`
);
if (rec.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
// Reprocess this record so transformed reflects rules/mappings without overrides
await pool.query(
`SELECT apply_transformations(${lit(rec.rows[0].source_name)}, ARRAY[${lit(parseInt(req.params.id))}::int], true)`
);
const updated = await pool.query(`SELECT * FROM get_record(${lit(parseInt(req.params.id))})`);
res.json(updated.rows[0]);
} catch (err) {
next(err);
}
});
// Delete record
router.delete('/:id', async (req, res, next) => {
try {

View File

@ -284,7 +284,7 @@ record_additions AS (
-- Update all qualifying records; records with no rule matches get transformed = data
updated AS (
UPDATE dataflow.records rec
SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb),
SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb) || COALESCE(rec.overrides, '{}'::jsonb),
transformed_at = CURRENT_TIMESTAMP
FROM qualifying q
LEFT JOIN record_additions ra ON ra.id = q.id

View File

@ -39,6 +39,27 @@ RETURNS SETOF dataflow.records AS $$
LIMIT p_limit;
$$ LANGUAGE sql STABLE;
-- ── Overrides ─────────────────────────────────────────────────────────────────
-- Store manual overrides and immediately merge into transformed
CREATE OR REPLACE FUNCTION set_record_overrides(p_id INT, p_overrides JSONB)
RETURNS dataflow.records AS $$
UPDATE dataflow.records
SET overrides = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END,
transformed = COALESCE(transformed, data) || COALESCE(p_overrides, '{}'::jsonb)
WHERE id = p_id
RETURNING *;
$$ LANGUAGE sql;
-- Clear overrides; caller should reprocess to restore computed transformed value
CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INT)
RETURNS dataflow.records AS $$
UPDATE dataflow.records
SET overrides = NULL
WHERE id = p_id
RETURNING *;
$$ LANGUAGE sql;
-- ── Delete ────────────────────────────────────────────────────────────────────
CREATE OR REPLACE FUNCTION delete_record(p_id BIGINT)

View File

@ -188,7 +188,7 @@ BEGIN
v_view := 'dfv.' || quote_ident(p_source_name);
EXECUTE format('DROP VIEW IF EXISTS %s', v_view);
v_sql := format(
'CREATE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
'CREATE VIEW %s AS SELECT id, overrides IS NOT NULL AS _overridden, %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
v_view, v_cols, p_source_name
);
EXECUTE v_sql;

View File

@ -116,4 +116,7 @@ export const api = {
// Records
getRecords: (source, limit = 100, offset = 0) =>
request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`),
getRecord: (id) => request('GET', `/records/${id}`),
setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }),
clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`),
}

View File

@ -2,6 +2,7 @@ import { useState, useEffect, useRef } from 'react'
import { api } from '../api'
const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
const HIDDEN_COLS = new Set(['id', '_overridden'])
function formatVal(val) {
if (val === null || val === undefined) return null
@ -30,12 +31,20 @@ export default function Records({ source }) {
const debounceRef = useRef(null)
const LIMIT = 100
// Override panel
const [selectedRecord, setSelectedRecord] = useState(null) // full record from API
const [overrideDraft, setOverrideDraft] = useState({}) // { field: newValue }
const [panelLoading, setPanelLoading] = useState(false)
const [panelSaving, setPanelSaving] = useState(false)
const [panelMsg, setPanelMsg] = useState(null)
useEffect(() => {
if (!source) return
setOffset(0)
setSort({ col: null, dir: 'asc' })
setFilters([])
setViewError(null)
setSelectedRecord(null)
load(0, null, 'asc', [])
}, [source])
@ -46,8 +55,7 @@ export default function Records({ source }) {
const res = await api.getViewData(source, LIMIT, off, col, dir, active)
setExists(res.exists)
setRows(res.rows)
if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
} catch (err) {
setViewError(err.message)
} finally {
@ -70,7 +78,8 @@ export default function Records({ source }) {
}
function addFilter() {
setFilters(f => [...f, { col: cols[0] || '', pattern: '' }])
const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }])
}
function removeFilter(i) {
@ -90,131 +99,278 @@ export default function Records({ source }) {
function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) }
function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
async function openPanel(row) {
const id = row.id
if (!id) return
setPanelLoading(true)
setPanelMsg(null)
setSelectedRecord(null)
setOverrideDraft({})
try {
const rec = await api.getRecord(id)
setSelectedRecord(rec)
setOverrideDraft(rec.overrides || {})
} catch (err) {
setPanelMsg({ text: err.message, ok: false })
} finally {
setPanelLoading(false)
}
}
function closePanel() {
setSelectedRecord(null)
setOverrideDraft({})
setPanelMsg(null)
}
async function handleSaveOverrides() {
if (!selectedRecord) return
setPanelSaving(true)
setPanelMsg(null)
try {
const updated = await api.setRecordOverrides(selectedRecord.id, overrideDraft)
setSelectedRecord(updated)
setOverrideDraft(updated.overrides || {})
setPanelMsg({ text: 'Saved.', ok: true })
// Refresh the row in the table
setRows(rs => rs.map(r => r.id === updated.id
? { ...r, _overridden: updated.overrides != null }
: r
))
} catch (err) {
setPanelMsg({ text: err.message, ok: false })
} finally {
setPanelSaving(false)
}
}
async function handleClearOverrides() {
if (!selectedRecord) return
setPanelSaving(true)
setPanelMsg(null)
try {
const updated = await api.clearRecordOverrides(selectedRecord.id)
setSelectedRecord(updated)
setOverrideDraft({})
setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true })
setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r))
} catch (err) {
setPanelMsg({ text: err.message, ok: false })
} finally {
setPanelSaving(false)
}
}
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols
const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
// Fields available for override: keys from transformed
const transformedFields = selectedRecord?.transformed
? Object.keys(selectedRecord.transformed)
: []
return (
<div className="p-6">
<div className="flex items-center justify-between mb-4">
<h1 className="text-xl font-semibold text-gray-800">Records {source}</h1>
{exists && rows.length > 0 && (
<span className="text-xs text-gray-400 font-mono">dfv.{source}</span>
)}
</div>
{/* Filter bar */}
{exists !== false && displayCols.length > 0 && (
<div className="mb-4 flex flex-wrap gap-2 items-center">
{filters.map((f, i) => (
<div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
<select
className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
value={f.col}
onChange={e => updateFilter(i, 'col', e.target.value)}
>
{displayCols.map(c => <option key={c} value={c}>{c}</option>)}
</select>
<span className="text-xs text-gray-300 mx-0.5">~*</span>
<input
className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
placeholder="regex…"
value={f.pattern}
onChange={e => updateFilter(i, 'pattern', e.target.value)}
/>
<button
onClick={() => removeFilter(i)}
className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
>×</button>
</div>
))}
<button
onClick={addFilter}
className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
>
+ filter
</button>
{filters.length > 0 && (
<button
onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
className="text-xs text-gray-400 hover:text-red-500"
>
clear
</button>
<div className="flex h-full min-h-0 overflow-hidden">
<div className="flex-1 overflow-auto p-6 min-w-0">
<div className="flex items-center justify-between mb-4">
<h1 className="text-xl font-semibold text-gray-800">Records {source}</h1>
{exists && rows.length > 0 && (
<span className="text-xs text-gray-400 font-mono">dfv.{source}</span>
)}
</div>
)}
{loading && <p className="text-sm text-gray-400">Loading</p>}
{/* Filter bar */}
{exists !== false && visCols.length > 0 && (
<div className="mb-4 flex flex-wrap gap-2 items-center">
{filters.map((f, i) => (
<div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
<select
className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
value={f.col}
onChange={e => updateFilter(i, 'col', e.target.value)}
>
{visCols.map(c => <option key={c} value={c}>{c}</option>)}
</select>
<span className="text-xs text-gray-300 mx-0.5">~*</span>
<input
className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
placeholder="regex…"
value={f.pattern}
onChange={e => updateFilter(i, 'pattern', e.target.value)}
/>
<button
onClick={() => removeFilter(i)}
className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
>×</button>
</div>
))}
<button
onClick={addFilter}
className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
>
+ filter
</button>
{filters.length > 0 && (
<button
onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
className="text-xs text-gray-400 hover:text-red-500"
>
clear
</button>
)}
</div>
)}
{!loading && viewError && (
<p className="text-sm text-red-500">View error: {viewError} check field types in Sources.</p>
)}
{loading && <p className="text-sm text-gray-400">Loading</p>}
{!loading && exists === false && (
<p className="text-sm text-gray-400">
No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
</p>
)}
{!loading && viewError && (
<p className="text-sm text-red-500">View error: {viewError} check field types in Sources.</p>
)}
{!loading && exists && rows.length === 0 && (
<p className="text-sm text-gray-400">
{filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
</p>
)}
{!loading && exists === false && (
<p className="text-sm text-gray-400">
No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
</p>
)}
{!loading && exists && rows.length > 0 && (
<>
<div className="bg-white border border-gray-200 rounded overflow-auto mb-4">
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
{displayCols.map(col => {
const active = sort.col === col
return (
<th
key={col}
onClick={() => toggleSort(col)}
className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"
>
{col}
<span className="ml-1 text-gray-300">
{active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}
</span>
</th>
)
})}
</tr>
</thead>
<tbody>
{rows.map((row, i) => (
<tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
{displayCols.map((col, j) => {
const formatted = formatVal(row[col])
{!loading && exists && rows.length === 0 && (
<p className="text-sm text-gray-400">
{filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
</p>
)}
{!loading && exists && rows.length > 0 && (
<>
<div className="bg-white border border-gray-200 rounded overflow-auto mb-4">
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
{displayCols.map(col => {
const active = sort.col === col
return (
<td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
{formatted === null ? <span className="text-gray-300"></span> : formatted}
</td>
<th
key={col}
onClick={() => toggleSort(col)}
className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"
>
{col}
<span className="ml-1 text-gray-300">
{active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}
</span>
</th>
)
})}
</tr>
))}
</tbody>
</table>
</thead>
<tbody>
{rows.map((row, i) => {
const isOverridden = row._overridden
const isSelected = selectedRecord?.id === row.id
return (
<tr
key={i}
onClick={() => openPanel(row)}
className={`border-t border-gray-50 cursor-pointer transition-colors
${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}
>
{displayCols.map((col, j) => {
const formatted = formatVal(row[col])
return (
<td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
{formatted === null ? <span className="text-gray-300"></span> : formatted}
</td>
)
})}
</tr>
)
})}
</tbody>
</table>
</div>
<div className="flex items-center gap-3 text-sm text-gray-500">
<button onClick={prev} disabled={offset === 0}
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
Prev
</button>
<span>{offset + 1}{offset + rows.length}</span>
<button onClick={next} disabled={rows.length < LIMIT}
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
Next
</button>
</div>
</>
)}
</div>
{/* Override panel */}
{(selectedRecord || panelLoading) && (
<div className="w-80 border-l border-gray-200 bg-white flex flex-col overflow-hidden flex-shrink-0">
<div className="flex items-center justify-between px-3 py-2 border-b border-gray-100">
<span className="text-xs font-semibold text-gray-600 uppercase tracking-wide">Override</span>
<button onClick={closePanel} className="text-gray-300 hover:text-gray-500 leading-none text-lg">×</button>
</div>
<div className="flex items-center gap-3 text-sm text-gray-500">
<button onClick={prev} disabled={offset === 0}
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
Prev
</button>
<span>{offset + 1}{offset + rows.length}</span>
<button onClick={next} disabled={rows.length < LIMIT}
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
Next
</button>
</div>
</>
{panelLoading && <p className="text-xs text-gray-400 p-3">Loading</p>}
{selectedRecord && !panelLoading && (
<div className="flex-1 overflow-y-auto p-3 flex flex-col gap-3">
{panelMsg && (
<div className={`text-xs ${panelMsg.ok ? 'text-green-600' : 'text-red-500'}`}>
{panelMsg.text}
</div>
)}
<div className="text-xs text-gray-400">
Click any field to override its value. Overrides survive reprocess.
</div>
<div className="flex flex-col gap-1">
{transformedFields.map(field => {
const currentVal = selectedRecord.transformed[field]
const isOverridden = field in overrideDraft
return (
<div key={field} className={`rounded px-2 py-1.5 ${isOverridden ? 'bg-amber-50 border border-amber-200' : 'bg-gray-50'}`}>
<div className={`text-xs font-mono mb-0.5 ${isOverridden ? 'text-amber-700' : 'text-gray-400'}`}>
{field}
{isOverridden && <span className="ml-1 text-amber-500"></span>}
</div>
<input
className={`w-full text-xs font-mono bg-transparent border-0 focus:outline-none focus:ring-0 ${isOverridden ? 'text-amber-800' : 'text-gray-700'}`}
value={isOverridden ? overrideDraft[field] : (currentVal ?? '')}
onChange={e => setOverrideDraft(d => ({ ...d, [field]: e.target.value }))}
onFocus={() => {
if (!(field in overrideDraft)) {
setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') }))
}
}}
/>
</div>
)
})}
</div>
<div className="flex gap-2 pt-1">
<button
onClick={handleSaveOverrides}
disabled={panelSaving || Object.keys(overrideDraft).length === 0}
className="flex-1 text-xs bg-blue-600 text-white rounded px-3 py-1.5 hover:bg-blue-700 disabled:opacity-40">
{panelSaving ? 'Saving…' : 'Save overrides'}
</button>
{selectedRecord.overrides && (
<button
onClick={handleClearOverrides}
disabled={panelSaving}
className="text-xs border border-gray-200 rounded px-3 py-1.5 text-gray-500 hover:border-red-300 hover:text-red-500 disabled:opacity-40">
Clear
</button>
)}
</div>
</div>
)}
</div>
)}
</div>
)