pt/dataflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

List override and transformed keys as source fields; show record id

Browse Source 
get_source_fields only unioned schema fields, raw data keys, rule
output_field and mapping output keys. Keys that live solely in
records.transformed or records.overrides — a manual override such as
dcard's "Note", or a transformed key whose rule was since deleted — never
appeared on the source page, so there was no way to add them to the view.
Read both columns off the records directly.

Records showed no id column, leaving no handle to identify a row. Split
the hidden-column set: HIDDEN_COLS still keeps id out of the override
editor, GRID_HIDDEN_COLS hides only _overridden, and gridCols() pins id
first in the grid and the filter dropdown.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BDFU3ueEbCYGE37oDgeu2j
This commit is contained in:
Paul Trowbridge 2026-07-26 23:10:01 -04:00
parent 426f975d9c
commit d24537d3db
2 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
database/sources.sql
View File

@ -65,6 +65,16 @@ RETURNS TABLE (key TEXT, origins TEXT[]) AS $$
SELECT jsonb_object_keys(data) AS key, 'raw' AS origin
FROM dataflow.records WHERE source_name = p_source_name
UNION ALL
-- transformed/overrides are read straight off the records so that keys with
-- no surviving rule or mapping (e.g. a manual override) still get listed
SELECT jsonb_object_keys(transformed) AS key, 'transformed' AS origin
FROM dataflow.records
WHERE source_name = p_source_name AND transformed IS NOT NULL
UNION ALL
SELECT jsonb_object_keys(overrides) AS key, 'override' AS origin
FROM dataflow.records
WHERE source_name = p_source_name AND overrides IS NOT NULL
UNION ALL
SELECT output_field AS key, 'rule: ' || name AS origin
FROM dataflow.rules WHERE source_name = p_source_name
UNION ALL

16
ui/src/pages/Records.jsx
View File

@ -62,7 +62,16 @@ function AutocompleteInput({ value, onChange, onEnter, suggestions = [], classNa
}
const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
// Hidden everywhere a field can be edited as an override — you can't override an id
const HIDDEN_COLS = new Set(['id', '_overridden'])
// The grid and its filters keep id: it's the only handle on a specific row
const GRID_HIDDEN_COLS = new Set(['_overridden'])
// id first, then the rest in view order
function gridCols(keys) {
const visible = keys.filter(c => !GRID_HIDDEN_COLS.has(c))
return visible.includes('id') ? ['id', ...visible.filter(c => c !== 'id')] : visible
}
function formatVal(val) {
if (val === null || val === undefined) return null
@ -179,7 +188,8 @@ export default function Records({ source }) {
}
function addFilter() {
const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
// id is filterable but a poor default — start on the first data column
const visCols = gridCols(cols).filter(c => c !== 'id')
setFilters(f => [...f, { col: visCols[0] || '', pattern: '' }])
}
@ -275,8 +285,8 @@ export default function Records({ source }) {
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c))
const visCols = cols.filter(c => !HIDDEN_COLS.has(c))
const displayCols = gridCols(rows.length > 0 ? Object.keys(rows[0]) : cols)
const visCols = gridCols(cols)
// For bulk bar: only established override keys
const allOverrideCols = [...new Set([...overrideCols, ...extraCols])]