diff --git a/api/routes/sources.js b/api/routes/sources.js index c911396..5b9c46e 100644 --- a/api/routes/sources.js +++ b/api/routes/sources.js @@ -234,6 +234,19 @@ module.exports = (pool) => { } }); + // Override keys — distinct field names used in overrides across all records for this source + router.get('/:name/override-keys', async (req, res, next) => { + try { + const result = await pool.query( + `SELECT DISTINCT jsonb_object_keys(overrides) AS key + FROM dataflow.records + WHERE source_name = ${lit(req.params.name)} AND overrides IS NOT NULL + ORDER BY key` + ); + res.json(result.rows.map(r => r.key)); + } catch (err) { next(err); } + }); + // Pivot layouts router.get('/:name/layouts', async (req, res, next) => { try { diff --git a/ui/src/api.js b/ui/src/api.js index 1a2d0b0..f98bdb6 100644 --- a/ui/src/api.js +++ b/ui/src/api.js @@ -135,6 +135,7 @@ export const api = { getRecords: (source, limit = 100, offset = 0) => request('GET', `/records/source/${source}?limit=${limit}&offset=${offset}`), getRecord: (id) => request('GET', `/records/${id}`), + getOverrideKeys: (source) => request('GET', `/sources/${source}/override-keys`), setRecordOverrides: (id, overrides) => request('PUT', `/records/${id}/overrides`, { overrides }), clearRecordOverrides: (id) => request('DELETE', `/records/${id}/overrides`), } diff --git a/ui/src/pages/Records.jsx b/ui/src/pages/Records.jsx index 3304693..713a364 100644 --- a/ui/src/pages/Records.jsx +++ b/ui/src/pages/Records.jsx @@ -1,7 +1,7 @@ import { useState, useEffect, useRef } from 'react' import { api } from '../api' -function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = [], className, placeholder }) { +function AutocompleteInput({ value, onChange, onEnter, suggestions = [], className, placeholder }) { const [open, setOpen] = useState(false) const [highlighted, setHighlighted] = useState(0) const [dropPos, setDropPos] = useState(null) @@ -25,11 +25,11 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = [] function handleKeyDown(e) { if (e.altKey && e.key === 'ArrowDown') { e.preventDefault(); openList(); return } if (open && filtered.length > 0) { - if (e.key === 'Tab') { e.preventDefault(); setHighlighted(h => (h + 1) % filtered.length); return } - if (e.key === 'ArrowDown') { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return } - if (e.key === 'ArrowUp') { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return } - if (e.key === 'Enter') { e.preventDefault(); select(filtered[highlighted]); return } - if (e.key === 'Escape') { setOpen(false); return } + if (e.key === 'Tab') { e.preventDefault(); setHighlighted(h => (h + 1) % filtered.length); return } + if (e.key === 'ArrowDown') { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return } + if (e.key === 'ArrowUp') { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return } + if (e.key === 'Enter') { e.preventDefault(); select(filtered[highlighted]); return } + if (e.key === 'Escape') { setOpen(false); return } } if (e.key === 'Enter') onEnter?.() } @@ -44,7 +44,6 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = [] { onChange(e.target.value); if (e.target.value) openList() }} onKeyDown={handleKeyDown} - onFocus={onFocus} onBlur={e => { if (!listRef.current?.contains(e.relatedTarget)) setOpen(false) }} /> {open && filtered.length > 0 && dropPos && ( @@ -52,7 +51,8 @@ function AutocompleteInput({ value, onChange, onEnter, onFocus, suggestions = [] style={{ position: 'fixed', top: dropPos.top, left: dropPos.left, minWidth: dropPos.minWidth, zIndex: 9999 }} className="bg-white border border-gray-200 rounded shadow-lg max-h-40 overflow-y-auto"> {filtered.map((s, i) => ( -
{ e.preventDefault(); select(s) }}>{s}
))} @@ -91,12 +91,16 @@ export default function Records({ source }) { const debounceRef = useRef(null) const LIMIT = 100 + // Override cols — loaded from DB once per source, extended by user via + + const [overrideCols, setOverrideCols] = useState([]) // keys seen in overrides across all records + const [extraCols, setExtraCols] = useState([]) // new cols added this session via + + const [globalValues, setGlobalValues] = useState({}) // picklist suggestions + // Override panel const [panelOpen, setPanelOpen] = useState(false) - const [selectedRow, setSelectedRow] = useState(null) // raw view row (has id) - const [selectedRecord, setSelectedRecord] = useState(null) // full record from API - const [overrideDraft, setOverrideDraft] = useState({}) // { field: newValue } - const [extraFields, setExtraFields] = useState([]) // [{field, value}] for new keys + const [selectedRow, setSelectedRow] = useState(null) + const [selectedRecord, setSelectedRecord] = useState(null) + const [overrideDraft, setOverrideDraft] = useState({}) const [panelLoading, setPanelLoading] = useState(false) const [panelSaving, setPanelSaving] = useState(false) const [panelMsg, setPanelMsg] = useState(null) @@ -110,7 +114,11 @@ export default function Records({ source }) { setSelectedRecord(null) setSelectedRow(null) setPanelOpen(false) + setOverrideCols([]) + setExtraCols([]) load(0, null, 'asc', []) + api.getOverrideKeys(source).then(setOverrideCols).catch(() => {}) + api.getGlobalValues().then(setGlobalValues).catch(() => {}) }, [source]) async function load(off, col, dir, filt) { @@ -165,12 +173,10 @@ export default function Records({ source }) { function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) } async function openPanel(row) { - // Open panel immediately, then load full record async setPanelOpen(true) setSelectedRow(row) setSelectedRecord(null) setOverrideDraft({}) - setExtraFields([]) setPanelMsg(null) const id = row.id @@ -196,7 +202,6 @@ export default function Records({ source }) { setSelectedRow(null) setSelectedRecord(null) setOverrideDraft({}) - setExtraFields([]) setPanelMsg(null) } @@ -205,15 +210,19 @@ export default function Records({ source }) { setPanelSaving(true) setPanelMsg(null) try { - const merged = { ...overrideDraft } - extraFields.forEach(({ field, value }) => { if (field.trim()) merged[field.trim()] = value }) - const updated = await api.setRecordOverrides(selectedRecord.id, merged) + // Only send cols that have a non-empty value + const toSave = Object.fromEntries( + Object.entries(overrideDraft).filter(([, v]) => String(v).trim()) + ) + const updated = await api.setRecordOverrides(selectedRecord.id, toSave) setSelectedRecord(updated) setOverrideDraft(updated.overrides || {}) + // Merge any new cols from extraCols into overrideCols + setOverrideCols(prev => [...new Set([...prev, ...extraCols.filter(c => c.trim())])]) + setExtraCols([]) setPanelMsg({ text: 'Saved.', ok: true }) - // Refresh the row in the table setRows(rs => rs.map(r => r.id === updated.id - ? { ...r, _overridden: updated.overrides != null } + ? { ...r, _overridden: updated.overrides != null && Object.keys(updated.overrides).length > 0 } : r )) } catch (err) { @@ -231,7 +240,7 @@ export default function Records({ source }) { const updated = await api.clearRecordOverrides(selectedRecord.id) setSelectedRecord(updated) setOverrideDraft({}) - setPanelMsg({ text: 'Overrides cleared. Transformed values restored.', ok: true }) + setPanelMsg({ text: 'Cleared.', ok: true }) setRows(rs => rs.map(r => r.id === updated.id ? { ...r, _overridden: false } : r)) } catch (err) { setPanelMsg({ text: err.message, ok: false }) @@ -245,26 +254,10 @@ export default function Records({ source }) { const displayCols = (rows.length > 0 ? Object.keys(rows[0]) : cols).filter(c => !HIDDEN_COLS.has(c)) const visCols = cols.filter(c => !HIDDEN_COLS.has(c)) - const transformedFields = selectedRecord?.transformed ? Object.keys(selectedRecord.transformed) : [] + // All override cols: known from DB + new ones added this session + const allOverrideCols = [...new Set([...overrideCols, ...extraCols])] - // Suggestions for new field names: all display cols + any already-overridden keys - const fieldSuggestions = [...new Set([ - ...displayCols, - ...(selectedRecord?.overrides ? Object.keys(selectedRecord.overrides) : []), - ])].filter(f => !transformedFields.includes(f)).sort() - - // Value suggestions per field derived from current page of rows - const valuesByField = {} - rows.forEach(row => { - Object.entries(row).forEach(([k, v]) => { - if (v != null && !HIDDEN_COLS.has(k)) { - if (!valuesByField[k]) valuesByField[k] = new Set() - valuesByField[k].add(String(v)) - } - }) - }) - - const isDirty = Object.keys(overrideDraft).length > 0 || extraFields.some(ef => ef.field.trim()) + const isDirty = Object.values(overrideDraft).some(v => String(v).trim()) || extraCols.some(c => c.trim()) return (
@@ -295,41 +288,27 @@ export default function Records({ source }) { value={f.pattern} onChange={e => updateFilter(i, 'pattern', e.target.value)} /> - +
))} - {filters.length > 0 && ( - + )} )} {loading &&

Loading…

} - - {!loading && viewError && ( -

View error: {viewError} — check field types in Sources.

- )} - + {!loading && viewError &&

View error: {viewError} — check field types in Sources.

} {!loading && exists === false && (

No view generated yet. Go to Sources, check fields as In view, then click Generate view.

)} - {!loading && exists && rows.length === 0 && (

{filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'} @@ -345,15 +324,10 @@ export default function Records({ source }) { {displayCols.map(col => { const active = sort.col === col return ( - toggleSort(col)} - className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600" - > + toggleSort(col)} + className="px-3 py-2 font-medium whitespace-nowrap cursor-pointer select-none hover:text-gray-600"> {col} - - {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'} - + {active ? (sort.dir === 'asc' ? '▲' : '▼') : '⇅'} ) })} @@ -364,12 +338,9 @@ export default function Records({ source }) { const isOverridden = row._overridden const isSelected = selectedRow?.id != null && selectedRow.id === row.id return ( - openPanel(row)} + openPanel(row)} className={`border-t border-gray-50 cursor-pointer transition-colors - ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`} - > + ${isSelected ? 'bg-blue-50' : isOverridden ? 'bg-amber-50 hover:bg-amber-100' : 'hover:bg-gray-50'}`}> {displayCols.map((col, j) => { const formatted = formatVal(row[col]) return ( @@ -387,24 +358,20 @@ export default function Records({ source }) {

+ className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">← Prev {offset + 1}–{offset + rows.length} + className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">Next →
)} - {/* Override panel */} + {/* Panel */} {panelOpen && (
- Override + Record
@@ -418,106 +385,93 @@ export default function Records({ source }) {
)} - - - - - - - - - {transformedFields.map(field => { - const currentVal = selectedRecord.transformed[field] - const isOverridden = field in overrideDraft - const suggestions = [...(valuesByField[field] || [])].sort() - return ( - - - - - - ) - })} + {/* Read-only transformed fields */} +
+ {Object.entries(selectedRecord.transformed || {}).map(([field, val]) => ( +
+ {field} + {formatVal(val) ?? } +
+ ))} +
- {/* New field rows */} - {extraFields.map((ef, i) => ( - - - - - - ))} + {/* Override cols — Mappings-style */} +
+
+ Override + +
-
- - - -
fieldvalue -
- {field} - - setOverrideDraft(d => ({ ...d, [field]: v }))} - onEnter={handleSaveOverrides} - suggestions={suggestions} - onFocus={() => { - if (!(field in overrideDraft)) - setOverrideDraft(d => ({ ...d, [field]: String(currentVal ?? '') })) - }} - /> - - {isOverridden && ( - - )} -
- setExtraFields(fs => fs.map((f, j) => j === i ? { ...f, field: v } : f))} - suggestions={fieldSuggestions} - /> - - setExtraFields(fs => fs.map((f, j) => j === i ? { ...f, value: v } : f))} - suggestions={ef.field ? [...(valuesByField[ef.field] || [])].sort() : []} - onEnter={handleSaveOverrides} - /> - - -
- -
+ + + {allOverrideCols.map((col, idx) => { + const isExtra = idx >= overrideCols.length + const suggestions = [...(globalValues[col] || [])].sort() + const val = overrideDraft[col] ?? '' + return ( + + + + + + ) + })} + +
+ {isExtra ? ( + { + const newName = e.target.value + setExtraCols(ec => { const c = [...ec]; c[idx - overrideCols.length] = newName; return c }) + if (val) setOverrideDraft(d => { + const n = { ...d } + delete n[col] + if (newName) n[newName] = val + return n + }) + }} + /> + ) : ( + {col} + )} + + setOverrideDraft(d => ({ ...d, [col]: v }))} + onEnter={handleSaveOverrides} + suggestions={suggestions} + /> + + {val && ( + + )} +
+ -
+
- {selectedRecord.overrides && ( + {selectedRecord.overrides && Object.keys(selectedRecord.overrides).length > 0 && ( )}