Add per-record overrides that survive reprocess

Schema:
- records.overrides JSONB column (ALTER TABLE, already applied)
- apply_transformations merges overrides on top: data || rules || overrides
- generate_source_view always includes id and _overridden columns
- set_record_overrides(id, overrides): stores and immediately merges into transformed
- clear_record_overrides(id): clears overrides then reprocesses record

API:
- PUT  /records/:id/overrides — set overrides
- DELETE /records/:id/overrides — clear and reprocess

UI (Records page):
- Rows are clickable; overridden rows highlighted amber
- Side panel shows all transformed fields as editable inputs
- Overridden fields highlighted amber with pencil indicator
- Save stores overrides; Clear removes them and restores computed values
- id and _overridden hidden from table display

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/routes/records.js

@@ -49,6 +49,40 @@ module.exports = (pool) => {
         }
     });
 
+    // Set overrides for a record and immediately merge into transformed
+    router.put('/:id/overrides', async (req, res, next) => {
+        try {
+            const { overrides } = req.body;
+            if (!overrides || typeof overrides !== 'object')
+                return res.status(400).json({ error: 'overrides object required' });
+            const result = await pool.query(
+                `SELECT * FROM set_record_overrides(${lit(parseInt(req.params.id))}, ${lit(overrides)})`
+            );
+            if (result.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
+            res.json(result.rows[0]);
+        } catch (err) {
+            next(err);
+        }
+    });
+
+    // Clear overrides and reprocess that record to restore computed values
+    router.delete('/:id/overrides', async (req, res, next) => {
+        try {
+            const rec = await pool.query(
+                `SELECT * FROM clear_record_overrides(${lit(parseInt(req.params.id))})`
+            );
+            if (rec.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
+            // Reprocess this record so transformed reflects rules/mappings without overrides
+            await pool.query(
+                `SELECT apply_transformations(${lit(rec.rows[0].source_name)}, ARRAY[${lit(parseInt(req.params.id))}::int], true)`
+            );
+            const updated = await pool.query(`SELECT * FROM get_record(${lit(parseInt(req.params.id))})`);
+            res.json(updated.rows[0]);
+        } catch (err) {
+            next(err);
+        }
+    });
+
     // Delete record
     router.delete('/:id', async (req, res, next) => {
         try {

database/functions.sql

@@ -284,7 +284,7 @@ record_additions AS (
 -- Update all qualifying records; records with no rule matches get transformed = data
 updated AS (
     UPDATE dataflow.records rec
-    SET transformed    = rec.data || COALESCE(ra.additions, '{}'::jsonb),
+    SET transformed    = rec.data || COALESCE(ra.additions, '{}'::jsonb) || COALESCE(rec.overrides, '{}'::jsonb),
         transformed_at = CURRENT_TIMESTAMP
     FROM qualifying q
     LEFT JOIN record_additions ra ON ra.id = q.id

database/queries/records.sql

@@ -39,6 +39,27 @@ RETURNS SETOF dataflow.records AS $$
     LIMIT p_limit;
 $$ LANGUAGE sql STABLE;
 
+-- ── Overrides ─────────────────────────────────────────────────────────────────
+
+-- Store manual overrides and immediately merge into transformed
+CREATE OR REPLACE FUNCTION set_record_overrides(p_id INT, p_overrides JSONB)
+RETURNS dataflow.records AS $$
+    UPDATE dataflow.records
+    SET overrides  = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END,
+        transformed = COALESCE(transformed, data) || COALESCE(p_overrides, '{}'::jsonb)
+    WHERE id = p_id
+    RETURNING *;
+$$ LANGUAGE sql;
+
+-- Clear overrides; caller should reprocess to restore computed transformed value
+CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INT)
+RETURNS dataflow.records AS $$
+    UPDATE dataflow.records
+    SET overrides = NULL
+    WHERE id = p_id
+    RETURNING *;
+$$ LANGUAGE sql;
+
 -- ── Delete ────────────────────────────────────────────────────────────────────
 
 CREATE OR REPLACE FUNCTION delete_record(p_id BIGINT)

database/queries/sources.sql

@@ -188,7 +188,7 @@ BEGIN
     v_view := 'dfv.' || quote_ident(p_source_name);
     EXECUTE format('DROP VIEW IF EXISTS %s', v_view);
     v_sql := format(
-        'CREATE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
+        'CREATE VIEW %s AS SELECT id, overrides IS NOT NULL AS _overridden, %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
         v_view, v_cols, p_source_name
     );
     EXECUTE v_sql;

ui/src/api.js

@@ -116,4 +116,7 @@ export const api = {
   // Records
   getRecords: (source, limit = 100, offset = 0) =>
     request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`),
+  getRecord: (id) => request('GET', `/records/${id}`),
+  setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }),
+  clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`),
 }

ui/src/pages/Records.jsx

@@ -2,6 +2,7 @@ import { useState, useEffect, useRef } from 'react'
 import { api } from '../api'
 
 const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
+const HIDDEN_COLS = new Set(['id', '_overridden'])
 
 function formatVal(val) {
   if (val === null || val === undefined) return null
@@ -30,12 +31,20 @@ export default function Records({ source }) {
   const debounceRef = useRef(null)
   const LIMIT = 100
 
+  // Override panel
+  const [selectedRecord, setSelectedRecord] = useState(null)   // full record from API
+  const [overrideDraft, setOverrideDraft] = useState({})        // { field: newValue }
+  const [panelLoading, setPanelLoading] = useState(false)
+  const [panelSaving, setPanelSaving] = useState(false)
+  const [panelMsg, setPanelMsg] = useState(null)
+
   useEffect(() => {
     if (!source) return
     setOffset(0)
     setSort({ col: null, dir: 'asc' })
     setFilters([])
     setViewError(null)
+    setSelectedRecord(null)
     load(0, null, 'asc', [])
   }, [source])
 
@@ -46,8 +55,7 @@ export default function Records({ source }) {
       const res = await api.getViewData(source, LIMIT, off, col, dir, active)
       setExists(res.exists)
       setRows(res.rows)
-      if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
+      if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
-      else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
     } catch (err) {
       setViewError(err.message)
     } finally {
@@ -70,7 +78,8 @@ export default function Records({ source }) {
   }
 
   function addFilter() {
-    setFilters(f => [...f, { col: cols[0] || '', pattern: '' }])
+    const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+    setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }])
   }
 
   function removeFilter(i) {
@@ -90,131 +99,278 @@ export default function Records({ source }) {
   function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) }
   function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
 
+  async function openPanel(row) {
+    const id = row.id
+    if (!id) return
+    setPanelLoading(true)
+    setPanelMsg(null)
+    setSelectedRecord(null)
+    setOverrideDraft({})
+    try {
+      const rec = await api.getRecord(id)
+      setSelectedRecord(rec)
+      setOverrideDraft(rec.overrides || {})
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelLoading(false)
+    }
+  }
+
+  function closePanel() {
+    setSelectedRecord(null)
+    setOverrideDraft({})
+    setPanelMsg(null)
+  }
+
+  async function handleSaveOverrides() {
+    if (!selectedRecord) return
+    setPanelSaving(true)
+    setPanelMsg(null)
+    try {
+      const updated = await api.setRecordOverrides(selectedRecord.id, overrideDraft)
+      setSelectedRecord(updated)
+      setOverrideDraft(updated.overrides || {})
+      setPanelMsg({ text: 'Saved.', ok: true })
+      // Refresh the row in the table
+      setRows(rs => rs.map(r => r.id === updated.id
+        ? { ...r, _overridden: updated.overrides != null }
+        : r
+      ))
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelSaving(false)
+    }
+  }
+
+  async function handleClearOverrides() {
+    if (!selectedRecord) return
+    setPanelSaving(true)
+    setPanelMsg(null)
+    try {
+      const updated = await api.clearRecordOverrides(selectedRecord.id)
+      setSelectedRecord(updated)
+      setOverrideDraft({})
+      setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true })
+      setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r))
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelSaving(false)
+    }
+  }
+
   if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
 
-  const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols
+  const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
+  const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+
+  // Fields available for override: keys from transformed
+  const transformedFields = selectedRecord?.transformed
+    ? Object.keys(selectedRecord.transformed)
+    : []
 
   return (
-    <div className="p-6">
+    <div className="flex h-full min-h-0 overflow-hidden">
-      <div className="flex items-center justify-between mb-4">
+      <div className="flex-1 overflow-auto p-6 min-w-0">
-        <h1 className="text-xl font-semibold text-gray-800">Records — {source}</h1>
+        <div className="flex items-center justify-between mb-4">
-        {exists && rows.length > 0 && (
+          <h1 className="text-xl font-semibold text-gray-800">Records — {source}</h1>
-          <span className="text-xs text-gray-400 font-mono">dfv.{source}</span>
+          {exists && rows.length > 0 && (
-        )}
+            <span className="text-xs text-gray-400 font-mono">dfv.{source}</span>
-      </div>
-
-      {/* Filter bar */}
-      {exists !== false && displayCols.length > 0 && (
-        <div className="mb-4 flex flex-wrap gap-2 items-center">
-          {filters.map((f, i) => (
-            <div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
-              <select
-                className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
-                value={f.col}
-                onChange={e => updateFilter(i, 'col', e.target.value)}
-              >
-                {displayCols.map(c => <option key={c} value={c}>{c}</option>)}
-              </select>
-              <span className="text-xs text-gray-300 mx-0.5">~*</span>
-              <input
-                className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
-                placeholder="regex…"
-                value={f.pattern}
-                onChange={e => updateFilter(i, 'pattern', e.target.value)}
-              />
-              <button
-                onClick={() => removeFilter(i)}
-                className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
-              >×</button>
-            </div>
-          ))}
-          <button
-            onClick={addFilter}
-            className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
-          >
-            + filter
-          </button>
-          {filters.length > 0 && (
-            <button
-              onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
-              className="text-xs text-gray-400 hover:text-red-500"
-            >
-              clear
-            </button>
           )}
         </div>
-      )}
 
-      {loading && <p className="text-sm text-gray-400">Loading…</p>}
+        {/* Filter bar */}
+        {exists !== false && visCols.length > 0 && (
+          <div className="mb-4 flex flex-wrap gap-2 items-center">
+            {filters.map((f, i) => (
+              <div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
+                <select
+                  className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
+                  value={f.col}
+                  onChange={e => updateFilter(i, 'col', e.target.value)}
+                >
+                  {visCols.map(c => <option key={c} value={c}>{c}</option>)}
+                </select>
+                <span className="text-xs text-gray-300 mx-0.5">~*</span>
+                <input
+                  className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
+                  placeholder="regex…"
+                  value={f.pattern}
+                  onChange={e => updateFilter(i, 'pattern', e.target.value)}
+                />
+                <button
+                  onClick={() => removeFilter(i)}
+                  className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
+                >×</button>
+              </div>
+            ))}
+            <button
+              onClick={addFilter}
+              className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
+            >
+              + filter
+            </button>
+            {filters.length > 0 && (
+              <button
+                onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
+                className="text-xs text-gray-400 hover:text-red-500"
+              >
+                clear
+              </button>
+            )}
+          </div>
+        )}
 
-      {!loading && viewError && (
+        {loading && <p className="text-sm text-gray-400">Loading…</p>}
-        <p className="text-sm text-red-500">View error: {viewError} — check field types in Sources.</p>
-      )}
 
-      {!loading && exists === false && (
+        {!loading && viewError && (
-        <p className="text-sm text-gray-400">
+          <p className="text-sm text-red-500">View error: {viewError} — check field types in Sources.</p>
-          No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
+        )}
-        </p>
-      )}
 
-      {!loading && exists && rows.length === 0 && (
+        {!loading && exists === false && (
-        <p className="text-sm text-gray-400">
+          <p className="text-sm text-gray-400">
-          {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
+            No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
-        </p>
+          </p>
-      )}
+        )}
 
-      {!loading && exists && rows.length > 0 && (
+        {!loading && exists && rows.length === 0 && (
-        <>
+          <p className="text-sm text-gray-400">
-          <div className="bg-white border border-gray-200 rounded overflow-auto mb-4">
+            {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
-            <table className="w-full text-sm">
+          </p>
-              <thead>
+        )}
-                <tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
+
-                  {displayCols.map(col => {
+        {!loading && exists && rows.length > 0 && (
-                    const active = sort.col === col
+          <>
-                    return (
+            <div className="bg-white border border-gray-200 rounded overflow-auto mb-4">
-                      <th
+              <table className="w-full text-sm">
-                        key={col}
+                <thead>
-                        onClick={() => toggleSort(col)}
+                  <tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
-                        className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"
+                    {displayCols.map(col => {
-                      >
+                      const active = sort.col === col
-                        {col}
-                        <span className="ml-1 text-gray-300">
-                          {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}
-                        </span>
-                      </th>
-                    )
-                  })}
-                </tr>
-              </thead>
-              <tbody>
-                {rows.map((row, i) => (
-                  <tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
-                    {displayCols.map((col, j) => {
-                      const formatted = formatVal(row[col])
                       return (
-                        <td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
+                        <th
-                          {formatted === null ? <span className="text-gray-300">—</span> : formatted}
+                          key={col}
-                        </td>
+                          onClick={() => toggleSort(col)}
+                          className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"
+                        >
+                          {col}
+                          <span className="ml-1 text-gray-300">
+                            {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}
+                          </span>
+                        </th>
                       )
                     })}
                   </tr>
-                ))}
+                </thead>
-              </tbody>
+                <tbody>
-            </table>
+                  {rows.map((row, i) => {
+                    const isOverridden = row._overridden
+                    const isSelected = selectedRecord?.id === row.id
+                    return (
+                      <tr
+                        key={i}
+                        onClick={() => openPanel(row)}
+                        className={`border-t border-gray-50 cursor-pointer transition-colors
+                          ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}
+                      >
+                        {displayCols.map((col, j) => {
+                          const formatted = formatVal(row[col])
+                          return (
+                            <td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
+                              {formatted === null ? <span className="text-gray-300">—</span> : formatted}
+                            </td>
+                          )
+                        })}
+                      </tr>
+                    )
+                  })}
+                </tbody>
+              </table>
+            </div>
+
+            <div className="flex items-center gap-3 text-sm text-gray-500">
+              <button onClick={prev} disabled={offset === 0}
+                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+                ← Prev
+              </button>
+              <span>{offset + 1}–{offset + rows.length}</span>
+              <button onClick={next} disabled={rows.length < LIMIT}
+                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+                Next →
+              </button>
+            </div>
+          </>
+        )}
+      </div>
+
+      {/* Override panel */}
+      {(selectedRecord || panelLoading) && (
+        <div className="w-80 border-l border-gray-200 bg-white flex flex-col overflow-hidden flex-shrink-0">
+          <div className="flex items-center justify-between px-3 py-2 border-b border-gray-100">
+            <span className="text-xs font-semibold text-gray-600 uppercase tracking-wide">Override</span>
+            <button onClick={closePanel} className="text-gray-300 hover:text-gray-500 leading-none text-lg">×</button>
           </div>
 
-          <div className="flex items-center gap-3 text-sm text-gray-500">
+          {panelLoading && <p className="text-xs text-gray-400 p-3">Loading…</p>}
-            <button onClick={prev} disabled={offset === 0}
+
-              className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+          {selectedRecord && !panelLoading && (
-              ← Prev
+            <div className="flex-1 overflow-y-auto p-3 flex flex-col gap-3">
-            </button>
+              {panelMsg && (
-            <span>{offset + 1}–{offset + rows.length}</span>
+                <div className={`text-xs ${panelMsg.ok ? 'text-green-600' : 'text-red-500'}`}>
-            <button onClick={next} disabled={rows.length < LIMIT}
+                  {panelMsg.text}
-              className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+                </div>
-              Next →
+              )}
-            </button>
+
-          </div>
+              <div className="text-xs text-gray-400">
-        </>
+                Click any field to override its value. Overrides survive reprocess.
+              </div>
+
+              <div className="flex flex-col gap-1">
+                {transformedFields.map(field => {
+                  const currentVal = selectedRecord.transformed[field]
+                  const isOverridden = field in overrideDraft
+                  return (
+                    <div key={field} className={`rounded px-2 py-1.5 ${isOverridden ? 'bg-amber-50 border border-amber-200' : 'bg-gray-50'}`}>
+                      <div className={`text-xs font-mono mb-0.5 ${isOverridden ? 'text-amber-700' : 'text-gray-400'}`}>
+                        {field}
+                        {isOverridden && <span className="ml-1 text-amber-500">✎</span>}
+                      </div>
+                      <input
+                        className={`w-full text-xs font-mono bg-transparent border-0 focus:outline-none focus:ring-0 ${isOverridden ? 'text-amber-800' : 'text-gray-700'}`}
+                        value={isOverridden ? overrideDraft[field] : (currentVal ?? '')}
+                        onChange={e => setOverrideDraft(d => ({ ...d, [field]: e.target.value }))}
+                        onFocus={() => {
+                          if (!(field in overrideDraft)) {
+                            setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') }))
+                          }
+                        }}
+                      />
+                    </div>
+                  )
+                })}
+              </div>
+
+              <div className="flex gap-2 pt-1">
+                <button
+                  onClick={handleSaveOverrides}
+                  disabled={panelSaving || Object.keys(overrideDraft).length === 0}
+                  className="flex-1 text-xs bg-blue-600 text-white rounded px-3 py-1.5 hover:bg-blue-700 disabled:opacity-40">
+                  {panelSaving ? 'Saving…' : 'Save overrides'}
+                </button>
+                {selectedRecord.overrides && (
+                  <button
+                    onClick={handleClearOverrides}
+                    disabled={panelSaving}
+                    className="text-xs border border-gray-200 rounded px-3 py-1.5 text-gray-500 hover:border-red-300 hover:text-red-500 disabled:opacity-40">
+                    Clear
+                  </button>
+                )}
+              </div>
+            </div>
+          )}
+        </div>
       )}
     </div>
   )