Records override panel: read-only transformed view + Mappings-style override cols

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/routes/sources.js

@@ -234,6 +234,19 @@ module.exports = (pool) => {
         }
     });
 
+    // Override keys — distinct field names used in overrides across all records for this source
+    router.get('/:name/override-keys', async (req, res, next) => {
+        try {
+            const result = await pool.query(
+                `SELECT DISTINCT jsonb_object_keys(overrides) AS key
+                 FROM dataflow.records
+                 WHERE source_name = ${lit(req.params.name)} AND overrides IS NOT NULL
+                 ORDER BY key`
+            );
+            res.json(result.rows.map(r => r.key));
+        } catch (err) { next(err); }
+    });
+
     // Pivot layouts
     router.get('/:name/layouts', async (req, res, next) => {
         try {

ui/src/api.js

@@ -135,6 +135,7 @@ export const api = {
   getRecords: (source, limit = 100, offset = 0) =>
     request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`),
   getRecord: (id) => request('GET', `/records/${id}`),
+  getOverrideKeys: (source) => request('GET', `/sources/${source}/override-keys`),
   setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }),
   clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`),
 }

ui/src/pages/Records.jsx

@@ -1,7 +1,7 @@
 import { useState, useEffect, useRef } from 'react'
 import { api } from '../api'
 
-function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = [], className, placeholder }) {
+function AutocompleteInput({ value, onChange, onEnter, suggestions = [], className, placeholder }) {
   const [open, setOpen] = useState(false)
   const [highlighted, setHighlighted] = useState(0)
   const [dropPos, setDropPos] = useState(null)
@@ -25,11 +25,11 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = []
   function handleKeyDown(e) {
     if (e.altKey && e.key === 'ArrowDown') { e.preventDefault(); openList(); return }
     if (open && filtered.length > 0) {
-      if (e.key === 'Tab')        { e.preventDefault(); setHighlighted(h => (h + 1) % filtered.length); return }
+      if (e.key === 'Tab')       { e.preventDefault(); setHighlighted(h => (h + 1) % filtered.length); return }
-      if (e.key === 'ArrowDown')  { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return }
+      if (e.key === 'ArrowDown') { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return }
-      if (e.key === 'ArrowUp')    { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return }
+      if (e.key === 'ArrowUp')   { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return }
-      if (e.key === 'Enter')      { e.preventDefault(); select(filtered[highlighted]); return }
+      if (e.key === 'Enter')     { e.preventDefault(); select(filtered[highlighted]); return }
-      if (e.key === 'Escape')     { setOpen(false); return }
+      if (e.key === 'Escape')    { setOpen(false); return }
     }
     if (e.key === 'Enter') onEnter?.()
   }
@@ -44,7 +44,6 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = []
       <input ref={inputRef} className={className} value={value} placeholder={placeholder}
         onChange={e => { onChange(e.target.value); if (e.target.value) openList() }}
         onKeyDown={handleKeyDown}
-        onFocus={onFocus}
         onBlur={e => { if (!listRef.current?.contains(e.relatedTarget)) setOpen(false) }}
       />
       {open && filtered.length > 0 && dropPos && (
@@ -52,7 +51,8 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = []
           style={{ position: 'fixed', top: dropPos.top, left: dropPos.left, minWidth: dropPos.minWidth, zIndex: 9999 }}
           className="bg-white border border-gray-200 rounded shadow-lg max-h-40 overflow-y-auto">
           {filtered.map((s, i) => (
-            <div key={s} className={`px-2 py-1 text-xs cursor-pointer whitespace-nowrap ${i === highlighted ? 'bg-blue-50 text-blue-700' : 'text-gray-700 hover:bg-gray-50'}`}
+            <div key={s}
+              className={`px-2 py-1 text-xs cursor-pointer whitespace-nowrap ${i === highlighted ? 'bg-blue-50 text-blue-700' : 'text-gray-700 hover:bg-gray-50'}`}
               onMouseDown={e => { e.preventDefault(); select(s) }}>{s}</div>
           ))}
         </div>
@@ -91,12 +91,16 @@ export default function Records({ source }) {
   const debounceRef = useRef(null)
   const LIMIT = 100
 
+  // Override cols — loaded from DB once per source, extended by user via +
+  const [overrideCols, setOverrideCols] = useState([])   // keys seen in overrides across all records
+  const [extraCols, setExtraCols] = useState([])         // new cols added this session via +
+  const [globalValues, setGlobalValues] = useState({})   // picklist suggestions
+
   // Override panel
   const [panelOpen, setPanelOpen] = useState(false)
-  const [selectedRow, setSelectedRow] = useState(null)          // raw view row (has id)
+  const [selectedRow, setSelectedRow] = useState(null)
-  const [selectedRecord, setSelectedRecord] = useState(null)    // full record from API
+  const [selectedRecord, setSelectedRecord] = useState(null)
-  const [overrideDraft, setOverrideDraft] = useState({})        // { field: newValue }
+  const [overrideDraft, setOverrideDraft] = useState({})
-  const [extraFields, setExtraFields] = useState([])           // [{field, value}] for new keys
   const [panelLoading, setPanelLoading] = useState(false)
   const [panelSaving, setPanelSaving] = useState(false)
   const [panelMsg, setPanelMsg] = useState(null)
@@ -110,7 +114,11 @@ export default function Records({ source }) {
     setSelectedRecord(null)
     setSelectedRow(null)
     setPanelOpen(false)
+    setOverrideCols([])
+    setExtraCols([])
     load(0, null, 'asc', [])
+    api.getOverrideKeys(source).then(setOverrideCols).catch(() => {})
+    api.getGlobalValues().then(setGlobalValues).catch(() => {})
   }, [source])
 
   async function load(off, col, dir, filt) {
@@ -165,12 +173,10 @@ export default function Records({ source }) {
   function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
 
   async function openPanel(row) {
-    // Open panel immediately, then load full record async
     setPanelOpen(true)
     setSelectedRow(row)
     setSelectedRecord(null)
     setOverrideDraft({})
-    setExtraFields([])
     setPanelMsg(null)
 
     const id = row.id
@@ -196,7 +202,6 @@ export default function Records({ source }) {
     setSelectedRow(null)
     setSelectedRecord(null)
     setOverrideDraft({})
-    setExtraFields([])
     setPanelMsg(null)
   }
 
@@ -205,15 +210,19 @@ export default function Records({ source }) {
     setPanelSaving(true)
     setPanelMsg(null)
     try {
-      const merged = { ...overrideDraft }
+      // Only send cols that have a non-empty value
-      extraFields.forEach(({ field, value }) => { if (field.trim()) merged[field.trim()] = value })
+      const toSave = Object.fromEntries(
-      const updated = await api.setRecordOverrides(selectedRecord.id, merged)
+        Object.entries(overrideDraft).filter(([, v]) => String(v).trim())
+      )
+      const updated = await api.setRecordOverrides(selectedRecord.id, toSave)
       setSelectedRecord(updated)
       setOverrideDraft(updated.overrides || {})
+      // Merge any new cols from extraCols into overrideCols
+      setOverrideCols(prev => [...new Set([...prev, ...extraCols.filter(c => c.trim())])])
+      setExtraCols([])
       setPanelMsg({ text: 'Saved.', ok: true })
-      // Refresh the row in the table
       setRows(rs => rs.map(r => r.id === updated.id
-        ? { ...r, _overridden: updated.overrides != null }
+        ? { ...r, _overridden: updated.overrides != null && Object.keys(updated.overrides).length > 0 }
         : r
       ))
     } catch (err) {
@@ -231,7 +240,7 @@ export default function Records({ source }) {
       const updated = await api.clearRecordOverrides(selectedRecord.id)
       setSelectedRecord(updated)
       setOverrideDraft({})
-      setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true })
+      setPanelMsg({ text: 'Cleared.', ok: true })
       setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r))
     } catch (err) {
       setPanelMsg({ text: err.message, ok: false })
@@ -245,26 +254,10 @@ export default function Records({ source }) {
   const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
   const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
 
-  const transformedFields = selectedRecord?.transformed ? Object.keys(selectedRecord.transformed) : []
+  // All override cols: known from DB + new ones added this session
+  const allOverrideCols = [...new Set([...overrideCols, ...extraCols])]
 
-  // Suggestions for new field names: all display cols + any already-overridden keys
+  const isDirty = Object.values(overrideDraft).some(v => String(v).trim()) || extraCols.some(c => c.trim())
-  const fieldSuggestions = [...new Set([
-    ...displayCols,
-    ...(selectedRecord?.overrides ? Object.keys(selectedRecord.overrides) : []),
-  ])].filter(f => !transformedFields.includes(f)).sort()
-
-  // Value suggestions per field derived from current page of rows
-  const valuesByField = {}
-  rows.forEach(row => {
-    Object.entries(row).forEach(([k, v]) => {
-      if (v != null && !HIDDEN_COLS.has(k)) {
-        if (!valuesByField[k]) valuesByField[k] = new Set()
-        valuesByField[k].add(String(v))
-      }
-    })
-  })
-
-  const isDirty = Object.keys(overrideDraft).length > 0 || extraFields.some(ef => ef.field.trim())
 
   return (
     <div className="flex h-full min-h-0 overflow-hidden">
@@ -295,41 +288,27 @@ export default function Records({ source }) {
                   value={f.pattern}
                   onChange={e => updateFilter(i, 'pattern', e.target.value)}
                 />
-                <button
+                <button onClick={() => removeFilter(i)} className="text-gray-300 hover:text-gray-500 ml-1 leading-none">×</button>
-                  onClick={() => removeFilter(i)}
-                  className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
-                >×</button>
               </div>
             ))}
-            <button
+            <button onClick={addFilter}
-              onClick={addFilter}
+              className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1">
-              className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
-            >
               + filter
             </button>
             {filters.length > 0 && (
-              <button
+              <button onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
-                onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
+                className="text-xs text-gray-400 hover:text-red-500">clear</button>
-                className="text-xs text-gray-400 hover:text-red-500"
-              >
-                clear
-              </button>
             )}
           </div>
         )}
 
         {loading && <p className="text-sm text-gray-400">Loading…</p>}
-
+        {!loading && viewError && <p className="text-sm text-red-500">View error: {viewError} — check field types in Sources.</p>}
-        {!loading && viewError && (
-          <p className="text-sm text-red-500">View error: {viewError} — check field types in Sources.</p>
-        )}
-
         {!loading && exists === false && (
           <p className="text-sm text-gray-400">
             No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
           </p>
         )}
-
         {!loading && exists && rows.length === 0 && (
           <p className="text-sm text-gray-400">
             {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
@@ -345,15 +324,10 @@ export default function Records({ source }) {
                     {displayCols.map(col => {
                       const active = sort.col === col
                       return (
-                        <th
+                        <th key={col} onClick={() => toggleSort(col)}
-                          key={col}
+                          className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600">
-                          onClick={() => toggleSort(col)}
-                          className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"
-                        >
                           {col}
-                          <span className="ml-1 text-gray-300">
+                          <span className="ml-1 text-gray-300">{active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}</span>
-                            {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'}
-                          </span>
                         </th>
                       )
                     })}
@@ -364,12 +338,9 @@ export default function Records({ source }) {
                     const isOverridden = row._overridden
                     const isSelected = selectedRow?.id != null && selectedRow.id === row.id
                     return (
-                      <tr
+                      <tr key={i} onClick={() => openPanel(row)}
-                        key={i}
-                        onClick={() => openPanel(row)}
                         className={`border-t border-gray-50 cursor-pointer transition-colors
-                          ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}
+                          ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}>
-                      >
                         {displayCols.map((col, j) => {
                           const formatted = formatVal(row[col])
                           return (
@@ -387,24 +358,20 @@ export default function Records({ source }) {
 
             <div className="flex items-center gap-3 text-sm text-gray-500">
               <button onClick={prev} disabled={offset === 0}
-                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">← Prev</button>
-                ← Prev
-              </button>
               <span>{offset + 1}–{offset + rows.length}</span>
               <button onClick={next} disabled={rows.length < LIMIT}
-                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
+                className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">Next →</button>
-                Next →
-              </button>
             </div>
           </>
         )}
       </div>
 
-      {/* Override panel */}
+      {/* Panel */}
       {panelOpen && (
         <div className="w-80 border-l border-gray-200 bg-white flex flex-col overflow-hidden flex-shrink-0">
           <div className="flex items-center justify-between px-3 py-2 border-b border-gray-100">
-            <span className="text-xs font-semibold text-gray-600 uppercase tracking-wide">Override</span>
+            <span className="text-xs font-semibold text-gray-600 uppercase tracking-wide">Record</span>
             <button onClick={closePanel} className="text-gray-300 hover:text-gray-500 leading-none text-lg">×</button>
           </div>
 
@@ -418,106 +385,93 @@ export default function Records({ source }) {
                 </div>
               )}
 
-              <table className="w-full text-xs">
+              {/* Read-only transformed fields */}
-                <thead>
+              <div className="border-b border-gray-100">
-                  <tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
+                {Object.entries(selectedRecord.transformed || {}).map(([field, val]) => (
-                    <th className="px-3 py-1.5 font-medium w-32">field</th>
+                  <div key={field} className="flex items-baseline gap-2 px-3 py-1 border-t border-gray-50 first:border-t-0">
-                    <th className="px-3 py-1.5 font-medium">value</th>
+                    <span className="text-xs font-mono text-gray-400 w-28 shrink-0 truncate">{field}</span>
-                    <th className="w-6" />
+                    <span className="text-xs font-mono text-gray-600 truncate">{formatVal(val) ?? <span className="text-gray-300">—</span>}</span>
-                  </tr>
+                  </div>
-                </thead>
+                ))}
-                <tbody>
+              </div>
-                  {transformedFields.map(field => {
-                    const currentVal = selectedRecord.transformed[field]
-                    const isOverridden = field in overrideDraft
-                    const suggestions = [...(valuesByField[field] || [])].sort()
-                    return (
-                      <tr key={field} className={`border-t border-gray-50 ${isOverridden ? 'bg-amber-50' : ''}`}>
-                        <td className={`px-3 py-1 font-mono whitespace-nowrap ${isOverridden ? 'text-amber-700' : 'text-gray-400'}`}>
-                          {field}
-                        </td>
-                        <td className="px-1 py-1">
-                          <AutocompleteInput
-                            className={`w-full text-xs font-mono px-2 py-0.5 rounded border focus:outline-none ${
-                              isOverridden ? 'bg-amber-50 border-amber-300 text-amber-800' : 'bg-transparent border-transparent hover:border-gray-200 focus:border-blue-300 text-gray-700'
-                            }`}
-                            value={isOverridden ? overrideDraft[field] : String(currentVal ?? '')}
-                            onChange={v => setOverrideDraft(d => ({ ...d, [field]: v }))}
-                            onEnter={handleSaveOverrides}
-                            suggestions={suggestions}
-                            onFocus={() => {
-                              if (!(field in overrideDraft))
-                                setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') }))
-                            }}
-                          />
-                        </td>
-                        <td className="pr-2 text-center">
-                          {isOverridden && (
-                            <button
-                              onClick={() => setOverrideDraft(d => { const n = { ...d }; delete n[field]; return n })}
-                              className="text-gray-300 hover:text-red-400 leading-none text-base"
-                              title="Clear override">×</button>
-                          )}
-                        </td>
-                      </tr>
-                    )
-                  })}
 
-                  {/* New field rows */}
+              {/* Override cols — Mappings-style */}
-                  {extraFields.map((ef, i) => (
+              <div className="flex-1">
-                    <tr key={`extra-${i}`} className="border-t border-gray-50 bg-blue-50">
+                <div className="flex items-center justify-between px-3 py-1.5 bg-gray-50 border-b border-gray-100">
-                      <td className="px-1 py-1">
+                  <span className="text-xs font-medium text-gray-500 uppercase tracking-wide">Override</span>
-                        <AutocompleteInput
+                  <button
-                          className="w-full text-xs font-mono px-2 py-0.5 rounded border border-blue-200 focus:outline-none focus:border-blue-400 bg-white text-gray-700"
+                    onClick={() => setExtraCols(ec => [...ec, ''])}
-                          value={ef.field}
+                    className="text-gray-400 hover:text-gray-700 font-medium text-sm leading-none"
-                          placeholder="field name…"
+                    title="Add column">+</button>
-                          onChange={v => setExtraFields(fs => fs.map((f, j) => j === i ? { ...f, field: v } : f))}
+                </div>
-                          suggestions={fieldSuggestions}
-                        />
-                      </td>
-                      <td className="px-1 py-1">
-                        <AutocompleteInput
-                          className="w-full text-xs font-mono px-2 py-0.5 rounded border border-blue-200 focus:outline-none focus:border-blue-400 bg-white text-gray-700"
-                          value={ef.value}
-                          placeholder="value…"
-                          onChange={v => setExtraFields(fs => fs.map((f, j) => j === i ? { ...f, value: v } : f))}
-                          suggestions={ef.field ? [...(valuesByField[ef.field] || [])].sort() : []}
-                          onEnter={handleSaveOverrides}
-                        />
-                      </td>
-                      <td className="pr-2 text-center">
-                        <button
-                          onClick={() => setExtraFields(fs => fs.filter((_, j) => j !== i))}
-                          className="text-gray-300 hover:text-red-400 leading-none text-base">×</button>
-                      </td>
-                    </tr>
-                  ))}
 
-                  <tr className="border-t border-gray-100">
+                <table className="w-full text-xs">
-                    <td colSpan={3} className="px-3 py-1.5">
+                  <tbody>
-                      <button
+                    {allOverrideCols.map((col, idx) => {
-                        onClick={() => setExtraFields(fs => [...fs, { field: '', value: '' }])}
+                      const isExtra = idx >= overrideCols.length
-                        className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-0.5">
+                      const suggestions = [...(globalValues[col] || [])].sort()
-                        + add field
+                      const val = overrideDraft[col] ?? ''
-                      </button>
+                      return (
-                    </td>
+                        <tr key={col || `extra-${idx}`} className="border-t border-gray-50">
-                  </tr>
+                          <td className="px-3 py-1 w-28 shrink-0">
-                </tbody>
+                            {isExtra ? (
-              </table>
+                              <input
+                                className="w-full text-xs font-mono border border-gray-200 rounded px-1 py-0.5 focus:outline-none focus:border-blue-400"
+                                value={col}
+                                placeholder="field name"
+                                onChange={e => {
+                                  const newName = e.target.value
+                                  setExtraCols(ec => { const c = [...ec]; c[idx - overrideCols.length] = newName; return c })
+                                  if (val) setOverrideDraft(d => {
+                                    const n = { ...d }
+                                    delete n[col]
+                                    if (newName) n[newName] = val
+                                    return n
+                                  })
+                                }}
+                              />
+                            ) : (
+                              <span className="font-mono text-gray-500 truncate block">{col}</span>
+                            )}
+                          </td>
+                          <td className="px-1 py-1">
+                            <AutocompleteInput
+                              className={`w-full text-xs font-mono px-2 py-0.5 rounded border focus:outline-none ${
+                                val ? 'border-amber-300 bg-amber-50 text-amber-800' : 'border-gray-200 text-gray-700'
+                              }`}
+                              value={val}
+                              onChange={v => setOverrideDraft(d => ({ ...d, [col]: v }))}
+                              onEnter={handleSaveOverrides}
+                              suggestions={suggestions}
+                            />
+                          </td>
+                          <td className="pr-2 text-center">
+                            {val && (
+                              <button
+                                onClick={() => setOverrideDraft(d => { const n = { ...d }; delete n[col]; return n })}
+                                className="text-gray-300 hover:text-red-400 leading-none text-base">×</button>
+                            )}
+                          </td>
+                        </tr>
+                      )
+                    })}
+                  </tbody>
+                </table>
+              </div>
 
-              <div className="flex gap-2 px-3 py-2 border-t border-gray-100 mt-auto">
+              <div className="flex gap-2 px-3 py-2 border-t border-gray-100">
                 <button
                   onClick={handleSaveOverrides}
                   disabled={panelSaving || !isDirty}
                   className="flex-1 text-xs bg-blue-600 text-white rounded px-3 py-1.5 hover:bg-blue-700 disabled:opacity-40">
-                  {panelSaving ? 'Saving…' : 'Save overrides'}
+                  {panelSaving ? 'Saving…' : 'Save'}
                 </button>
-                {selectedRecord.overrides && (
+                {selectedRecord.overrides && Object.keys(selectedRecord.overrides).length > 0 && (
                   <button
                     onClick={handleClearOverrides}
                     disabled={panelSaving}
                     className="text-xs border border-gray-200 rounded px-3 py-1.5 text-gray-500 hover:border-red-300 hover:text-red-500 disabled:opacity-40">
-                    Clear all
+                    Clear
                   </button>
                 )}
               </div>