From d24537d3db9f02407bba91c8539cca3a64edfb1c Mon Sep 17 00:00:00 2001
From: Paul Trowbridge <paul@hptrow.me>
Date: Sun, 26 Jul 2026 23:10:01 -0400
Subject: [PATCH] List override and transformed keys as source fields; show
 record id
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

get_source_fields only unioned schema fields, raw data keys, rule
output_field and mapping output keys. Keys that live solely in
records.transformed or records.overrides — a manual override such as
dcard's "Note", or a transformed key whose rule was since deleted — never
appeared on the source page, so there was no way to add them to the view.
Read both columns off the records directly.

Records showed no id column, leaving no handle to identify a row. Split
the hidden-column set: HIDDEN_COLS still keeps id out of the override
editor, GRID_HIDDEN_COLS hides only _overridden, and gridCols() pins id
first in the grid and the filter dropdown.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BDFU3ueEbCYGE37oDgeu2j
---
 database/sources.sql     | 10 ++++++++++
 ui/src/pages/Records.jsx | 16 +++++++++++++---
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/database/sources.sql b/database/sources.sql
index cf31cba..49c9083 100644
--- a/database/sources.sql
+++ b/database/sources.sql
@@ -65,6 +65,16 @@ RETURNS TABLE (key TEXT, origins TEXT[]) AS $$
         SELECT jsonb_object_keys(data) AS key, 'raw'     AS origin
         FROM dataflow.records WHERE source_name = p_source_name
         UNION ALL
+        -- transformed/overrides are read straight off the records so that keys with
+        -- no surviving rule or mapping (e.g. a manual override) still get listed
+        SELECT jsonb_object_keys(transformed) AS key, 'transformed' AS origin
+        FROM dataflow.records
+        WHERE source_name = p_source_name AND transformed IS NOT NULL
+        UNION ALL
+        SELECT jsonb_object_keys(overrides) AS key, 'override' AS origin
+        FROM dataflow.records
+        WHERE source_name = p_source_name AND overrides IS NOT NULL
+        UNION ALL
         SELECT output_field AS key, 'rule: ' || name     AS origin
         FROM dataflow.rules WHERE source_name = p_source_name
         UNION ALL
diff --git a/ui/src/pages/Records.jsx b/ui/src/pages/Records.jsx
index 09d0940..c1be1f9 100644
--- a/ui/src/pages/Records.jsx
+++ b/ui/src/pages/Records.jsx
@@ -62,7 +62,16 @@ function AutocompleteInput({ value, onChange, onEnter, suggestions = [], classNa
 }
 
 const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
+// Hidden everywhere a field can be edited as an override — you can't override an id
 const HIDDEN_COLS = new Set(['id', '_overridden'])
+// The grid and its filters keep id: it's the only handle on a specific row
+const GRID_HIDDEN_COLS = new Set(['_overridden'])
+
+// id first, then the rest in view order
+function gridCols(keys) {
+  const visible = keys.filter(c => !GRID_HIDDEN_COLS.has(c))
+  return visible.includes('id') ? ['id', ...visible.filter(c => c !== 'id')] : visible
+}
 
 function formatVal(val) {
   if (val === null || val === undefined) return null
@@ -179,7 +188,8 @@ export default function Records({ source }) {
   }
 
   function addFilter() {
-    const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+    // id is filterable but a poor default — start on the first data column
+    const visCols = gridCols(cols).filter(c => c !== 'id')
     setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }])
   }
 
@@ -275,8 +285,8 @@ export default function Records({ source }) {
 
   if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
 
-  const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
-  const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
+  const displayCols = gridCols(rows.length > 0 ? Object.keys(rows[0]) : cols)
+  const visCols = gridCols(cols)
 
   // For bulk bar: only established override keys
   const allOverrideCols = [...new Set([...overrideCols, ...extraCols])]