Add per-record overrides that survive reprocess

Schema:
- records.overrides JSONB column (ALTER TABLE, already applied)
- apply_transformations merges overrides on top: data || rules || overrides
- generate_source_view always includes id and _overridden columns
- set_record_overrides(id, overrides): stores and immediately merges into transformed
- clear_record_overrides(id): clears overrides then reprocesses record

API:
- PUT  /records/:id/overrides — set overrides
- DELETE /records/:id/overrides — clear and reprocess

UI (Records page):
- Rows are clickable; overridden rows highlighted amber
- Side panel shows all transformed fields as editable inputs
- Overridden fields highlighted amber with pencil indicator
- Save stores overrides; Clear removes them and restores computed values
- id and _overridden hidden from table display

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/routes/records.js

@@ -49,6 +49,40 @@ module.exports = (pool) => {
         }
     });
 
+    // Set overrides for a record and immediately merge into transformed
+    router.put('/:id/overrides', async (req, res, next) => {
+        try {
+            const { overrides } = req.body;
+            if (!overrides || typeof overrides !== 'object')
+                return res.status(400).json({ error: 'overrides object required' });
+            const result = await pool.query(
+                `SELECT * FROM set_record_overrides(${lit(parseInt(req.params.id))}, ${lit(overrides)})`
+            );
+            if (result.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
+            res.json(result.rows[0]);
+        } catch (err) {
+            next(err);
+        }
+    });
+
+    // Clear overrides and reprocess that record to restore computed values
+    router.delete('/:id/overrides', async (req, res, next) => {
+        try {
+            const rec = await pool.query(
+                `SELECT * FROM clear_record_overrides(${lit(parseInt(req.params.id))})`
+            );
+            if (rec.rows.length === 0) return res.status(404).json({ error: 'Record not found' });
+            // Reprocess this record so transformed reflects rules/mappings without overrides
+            await pool.query(
+                `SELECT apply_transformations(${lit(rec.rows[0].source_name)}, ARRAY[${lit(parseInt(req.params.id))}::int], true)`
+            );
+            const updated = await pool.query(`SELECT * FROM get_record(${lit(parseInt(req.params.id))})`);
+            res.json(updated.rows[0]);
+        } catch (err) {
+            next(err);
+        }
+    });
+
     // Delete record
     router.delete('/:id', async (req, res, next) => {
         try {

database/functions.sql

@@ -284,7 +284,7 @@ record_additions AS (
 -- Update all qualifying records; records with no rule matches get transformed = data
 updated AS (
     UPDATE dataflow.records rec
-    SET transformed    = rec.data || COALESCE(ra.additions, '{}'::jsonb),
+    SET transformed    = rec.data || COALESCE(ra.additions, '{}'::jsonb) || COALESCE(rec.overrides, '{}'::jsonb),
         transformed_at = CURRENT_TIMESTAMP
     FROM qualifying q
     LEFT JOIN record_additions ra ON ra.id = q.id

database/queries/records.sql

@@ -39,6 +39,27 @@ RETURNS SETOF dataflow.records AS $$
     LIMIT p_limit;
 $$ LANGUAGE sql STABLE;
 
+-- ── Overrides ─────────────────────────────────────────────────────────────────
+
+-- Store manual overrides and immediately merge into transformed
+CREATE OR REPLACE FUNCTION set_record_overrides(p_id INT, p_overrides JSONB)
+RETURNS dataflow.records AS $$
+    UPDATE dataflow.records
+    SET overrides  = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END,
+        transformed = COALESCE(transformed, data) || COALESCE(p_overrides, '{}'::jsonb)
+    WHERE id = p_id
+    RETURNING *;
+$$ LANGUAGE sql;
+
+-- Clear overrides; caller should reprocess to restore computed transformed value
+CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INT)
+RETURNS dataflow.records AS $$
+    UPDATE dataflow.records
+    SET overrides = NULL
+    WHERE id = p_id
+    RETURNING *;
+$$ LANGUAGE sql;
+
 -- ── Delete ────────────────────────────────────────────────────────────────────
 
 CREATE OR REPLACE FUNCTION delete_record(p_id BIGINT)

database/queries/sources.sql

@@ -188,7 +188,7 @@ BEGIN
     v_view := 'dfv.' || quote_ident(p_source_name);
     EXECUTE format('DROP VIEW IF EXISTS %s', v_view);
     v_sql := format(
-        'CREATE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
+        'CREATE VIEW %s AS SELECT id, overrides IS NOT NULL AS _overridden, %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
         v_view, v_cols, p_source_name
     );
     EXECUTE v_sql;

ui/src/api.js

@@ -116,4 +116,7 @@ export const api = {
   // Records
   getRecords: (source, limit = 100, offset = 0) =>
     request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`),
+  getRecord: (id) => request('GET', `/records/${id}`),
+  setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }),
+  clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`),
 }

ui/src/pages/Records.jsx

@@ -2,6 +2,7 @@ import { useState, useEffect, useRef } from 'react'
 import { api } from '../api'
 
 const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
+const HIDDEN_COLS = new Set(['id', '_overridden'])
 
 function formatVal(val) {
   if (val === null || val === undefined) return null
@@ -30,12 +31,20 @@ export default function Records({ source }) {
   const debounceRef = useRef(null)
   const LIMIT = 100
 
+  // Override panel
+  const [selectedRecord, setSelectedRecord] = useState(null)   // full record from API
+  const [overrideDraft, setOverrideDraft] = useState({})        // { field: newValue }
+  const [panelLoading, setPanelLoading] = useState(false)
+  const [panelSaving, setPanelSaving] = useState(false)
+  const [panelMsg, setPanelMsg] = useState(null)
+
   useEffect(() => {
     if (!source) return
     setOffset(0)
     setSort({ col: null, dir: 'asc' })
     setFilters([])
     setViewError(null)
+    setSelectedRecord(null)
     load(0, null, 'asc', [])
   }, [source])
 
@@ -46,8 +55,7 @@ export default function Records({ source }) {
       const res = await api.getViewData(source, LIMIT, off, col, dir, active)
       setExists(res.exists)
       setRows(res.rows)
-      if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
+      if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
-      else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
     } catch (err) {
       setViewError(err.message)
     } finally {
@@ -70,7 +78,8 @@ export default function Records({ source }) {
   }
 
   function addFilter() {
-    setFilters(f => [...f, { col: cols[0] || '', pattern: '' }])
+    const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+    setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }])
   }
 
   function removeFilter(i) {
@@ -90,12 +99,81 @@ export default function Records({ source }) {
   function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) }
   function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
 
+  async function openPanel(row) {
+    const id = row.id
+    if (!id) return
+    setPanelLoading(true)
+    setPanelMsg(null)
+    setSelectedRecord(null)
+    setOverrideDraft({})
+    try {
+      const rec = await api.getRecord(id)
+      setSelectedRecord(rec)
+      setOverrideDraft(rec.overrides || {})
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelLoading(false)
+    }
+  }
+
+  function closePanel() {
+    setSelectedRecord(null)
+    setOverrideDraft({})
+    setPanelMsg(null)
+  }
+
+  async function handleSaveOverrides() {
+    if (!selectedRecord) return
+    setPanelSaving(true)
+    setPanelMsg(null)
+    try {
+      const updated = await api.setRecordOverrides(selectedRecord.id, overrideDraft)
+      setSelectedRecord(updated)
+      setOverrideDraft(updated.overrides || {})
+      setPanelMsg({ text: 'Saved.', ok: true })
+      // Refresh the row in the table
+      setRows(rs => rs.map(r => r.id === updated.id
+        ? { ...r, _overridden: updated.overrides != null }
+        : r
+      ))
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelSaving(false)
+    }
+  }
+
+  async function handleClearOverrides() {
+    if (!selectedRecord) return
+    setPanelSaving(true)
+    setPanelMsg(null)
+    try {
+      const updated = await api.clearRecordOverrides(selectedRecord.id)
+      setSelectedRecord(updated)
+      setOverrideDraft({})
+      setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true })
+      setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r))
+    } catch (err) {
+      setPanelMsg({ text: err.message, ok: false })
+    } finally {
+      setPanelSaving(false)
+    }
+  }
+
   if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
 
-  const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols
+  const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
+  const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+
+  // Fields available for override: keys from transformed
+  const transformedFields = selectedRecord?.transformed
+    ? Object.keys(selectedRecord.transformed)
+    : []
 
   return (
-    <div className="p-6">
+    <div className="flex h-full min-h-0 overflow-hidden">
+      <div className="flex-1 overflow-auto p-6 min-w-0">
         <div className="flex items-center justify-between mb-4">
           <h1 className="text-xl font-semibold text-gray-800">Records — {source}</h1>
           {exists && rows.length > 0 && (
@@ -104,7 +182,7 @@ export default function Records({ source }) {
         </div>
 
         {/* Filter bar */}
-      {exists !== false && displayCols.length > 0 && (
+        {exists !== false && visCols.length > 0 && (
           <div className="mb-4 flex flex-wrap gap-2 items-center">
             {filters.map((f, i) => (
               <div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
@@ -113,7 +191,7 @@ export default function Records({ source }) {
                   value={f.col}
                   onChange={e => updateFilter(i, 'col', e.target.value)}
                 >
-                {displayCols.map(c => <option key={c} value={c}>{c}</option>)}
+                  {visCols.map(c => <option key={c} value={c}>{c}</option>)}
                 </select>
                 <span className="text-xs text-gray-300 mx-0.5">~*</span>
                 <input
@@ -187,8 +265,16 @@ export default function Records({ source }) {
                   </tr>
                 </thead>
                 <tbody>
-                {rows.map((row, i) => (
+                  {rows.map((row, i) => {
-                  <tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
+                    const isOverridden = row._overridden
+                    const isSelected = selectedRecord?.id === row.id
+                    return (
+                      <tr
+                        key={i}
+                        onClick={() => openPanel(row)}
+                        className={`border-t border-gray-50 cursor-pointer transition-colors
+                          ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}
+                      >
                         {displayCols.map((col, j) => {
                           const formatted = formatVal(row[col])
                           return (
@@ -198,7 +284,8 @@ export default function Records({ source }) {
                           )
                         })}
                       </tr>
-                ))}
+                    )
+                  })}
                 </tbody>
               </table>
             </div>
@@ -217,5 +304,74 @@ export default function Records({ source }) {
           </>
         )}
       </div>
+
+      {/* Override panel */}
+      {(selectedRecord || panelLoading) && (
+        <div className="w-80 border-l border-gray-200 bg-white flex flex-col overflow-hidden flex-shrink-0">
+          <div className="flex items-center justify-between px-3 py-2 border-b border-gray-100">
+            <span className="text-xs font-semibold text-gray-600 uppercase tracking-wide">Override</span>
+            <button onClick={closePanel} className="text-gray-300 hover:text-gray-500 leading-none text-lg">×</button>
+          </div>
+
+          {panelLoading && <p className="text-xs text-gray-400 p-3">Loading…</p>}
+
+          {selectedRecord && !panelLoading && (
+            <div className="flex-1 overflow-y-auto p-3 flex flex-col gap-3">
+              {panelMsg && (
+                <div className={`text-xs ${panelMsg.ok ? 'text-green-600' : 'text-red-500'}`}>
+                  {panelMsg.text}
+                </div>
+              )}
+
+              <div className="text-xs text-gray-400">
+                Click any field to override its value. Overrides survive reprocess.
+              </div>
+
+              <div className="flex flex-col gap-1">
+                {transformedFields.map(field => {
+                  const currentVal = selectedRecord.transformed[field]
+                  const isOverridden = field in overrideDraft
+                  return (
+                    <div key={field} className={`rounded px-2 py-1.5 ${isOverridden ? 'bg-amber-50 border border-amber-200' : 'bg-gray-50'}`}>
+                      <div className={`text-xs font-mono mb-0.5 ${isOverridden ? 'text-amber-700' : 'text-gray-400'}`}>
+                        {field}
+                        {isOverridden && <span className="ml-1 text-amber-500">✎</span>}
+                      </div>
+                      <input
+                        className={`w-full text-xs font-mono bg-transparent border-0 focus:outline-none focus:ring-0 ${isOverridden ? 'text-amber-800' : 'text-gray-700'}`}
+                        value={isOverridden ? overrideDraft[field] : (currentVal ?? '')}
+                        onChange={e => setOverrideDraft(d => ({ ...d, [field]: e.target.value }))}
+                        onFocus={() => {
+                          if (!(field in overrideDraft)) {
+                            setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') }))
+                          }
+                        }}
+                      />
+                    </div>
+                  )
+                })}
+              </div>
+
+              <div className="flex gap-2 pt-1">
+                <button
+                  onClick={handleSaveOverrides}
+                  disabled={panelSaving || Object.keys(overrideDraft).length === 0}
+                  className="flex-1 text-xs bg-blue-600 text-white rounded px-3 py-1.5 hover:bg-blue-700 disabled:opacity-40">
+                  {panelSaving ? 'Saving…' : 'Save overrides'}
+                </button>
+                {selectedRecord.overrides && (
+                  <button
+                    onClick={handleClearOverrides}
+                    disabled={panelSaving}
+                    className="text-xs border border-gray-200 rounded px-3 py-1.5 text-gray-500 hover:border-red-300 hover:text-red-500 disabled:opacity-40">
+                    Clear
+                  </button>
+                )}
+              </div>
+            </div>
+          )}
+        </div>
+      )}
+    </div>
   )
 }