Source setup UX, Pivot page, and import/view fixes

- Fix stale import_records in sources.sql that referenced deleted generate_constraint_key
- Auto-transform after import, auto-generate view after create
- New source form matches existing source layout (In view, Seq, type dropdown)
- Sample data table (50 rows) shown below field config in both new and existing source views
- Import sample CSV on create (checked by default)
- Sortable column headers on field table
- Choose CSV styled as a button showing filename
- + button in sidebar opens new source form
- Records tab shows error message when view cast fails instead of blank
- Pivot page with Perspective viewer, per-source saved layouts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-04-14 21:31:44 -04:00
parent d495ef2fc5
commit ebd88a2df8
7 changed files with 528 additions and 193 deletions

View File

@ -52,19 +52,22 @@ module.exports = (pool) => {
const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true });
if (records.length === 0) return res.status(400).json({ error: 'CSV file is empty' });
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/;
const sample = records[0];
const sampleRows = records.slice(0, 50);
const fields = Object.keys(sample).map(key => {
const val = sample[key];
const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null);
let type = 'text';
if (!isNaN(parseFloat(val)) && isFinite(val) && val.charAt(0) !== '0') {
if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) {
type = 'numeric';
} else if (Date.parse(val) > Date.parse('1950-01-01') && Date.parse(val) < Date.parse('2050-01-01')) {
} else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) {
type = 'date';
}
return { name: key, type };
});
res.json({ name: '', constraint_fields: [], fields });
res.json({ name: '', constraint_fields: [], fields, sampleRows });
} catch (err) {
next(err);
}

View File

@ -144,145 +144,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql STABLE;
-- ── Import (uniqueness constraint) ────────────────────────────────────────────
CREATE OR REPLACE FUNCTION import_records(p_source_name TEXT, p_data JSONB)
RETURNS JSON AS $$
DECLARE
v_constraint_fields TEXT[];
v_record JSONB;
v_constraint_key TEXT;
v_inserted INTEGER := 0;
v_duplicates INTEGER := 0;
v_log_id INTEGER;
BEGIN
SELECT constraint_fields INTO v_constraint_fields
FROM dataflow.sources WHERE name = p_source_name;
IF v_constraint_fields IS NULL THEN
RETURN json_build_object('success', false, 'error', 'Source not found: ' || p_source_name);
END IF;
FOR v_record IN SELECT * FROM jsonb_array_elements(p_data) LOOP
v_constraint_key := dataflow.generate_constraint_key(v_record, v_constraint_fields);
BEGIN
INSERT INTO dataflow.records (source_name, data, constraint_key)
VALUES (p_source_name, v_record, v_constraint_key);
v_inserted := v_inserted + 1;
EXCEPTION WHEN unique_violation THEN
v_duplicates := v_duplicates + 1;
END;
END LOOP;
INSERT INTO dataflow.import_log (source_name, records_imported, records_duplicate)
VALUES (p_source_name, v_inserted, v_duplicates)
RETURNING id INTO v_log_id;
RETURN json_build_object('success', true, 'imported', v_inserted, 'duplicates', v_duplicates, 'log_id', v_log_id);
END;
$$ LANGUAGE plpgsql;
-- ── Transformations ───────────────────────────────────────────────────────────
CREATE OR REPLACE FUNCTION dataflow.jsonb_merge(a JSONB, b JSONB)
RETURNS JSONB AS $$
SELECT COALESCE(a, '{}') || COALESCE(b, '{}')
$$ LANGUAGE sql IMMUTABLE;
DROP AGGREGATE IF EXISTS dataflow.jsonb_concat_obj(JSONB);
CREATE AGGREGATE dataflow.jsonb_concat_obj(JSONB) (
sfunc = dataflow.jsonb_merge,
stype = JSONB,
initcond = '{}'
);
DROP FUNCTION IF EXISTS apply_transformations(TEXT, INTEGER[]);
CREATE OR REPLACE FUNCTION apply_transformations(
p_source_name TEXT,
p_record_ids INTEGER[] DEFAULT NULL,
p_overwrite BOOLEAN DEFAULT FALSE
) RETURNS JSON AS $$
WITH
qualifying AS (
SELECT id, data FROM dataflow.records
WHERE source_name = p_source_name
AND (p_overwrite OR transformed IS NULL)
AND (p_record_ids IS NULL OR id = ANY(p_record_ids))
),
rx AS (
SELECT
q.id,
r.name AS rule_name,
r.sequence,
r.output_field,
r.retain,
r.function_type,
COALESCE(mt.rn, rp.rn, 1) AS result_number,
CASE WHEN array_length(mt.mt, 1) = 1 THEN to_jsonb(mt.mt[1]) ELSE to_jsonb(mt.mt) END AS match_val,
to_jsonb(rp.rp) AS replace_val
FROM dataflow.rules r
INNER JOIN qualifying q ON q.data ? r.field
LEFT JOIN LATERAL regexp_matches(q.data ->> r.field, r.pattern, r.flags)
WITH ORDINALITY AS mt(mt, rn) ON r.function_type = 'extract'
LEFT JOIN LATERAL regexp_replace(q.data ->> r.field, r.pattern, r.replace_value, r.flags)
WITH ORDINALITY AS rp(rp, rn) ON r.function_type = 'replace'
WHERE r.source_name = p_source_name AND r.enabled = true
),
agg_matches AS (
SELECT
id, rule_name, sequence, output_field, retain, function_type,
CASE function_type
WHEN 'replace' THEN jsonb_agg(replace_val) -> 0
ELSE
CASE WHEN max(result_number) = 1
THEN jsonb_agg(match_val ORDER BY result_number) -> 0
ELSE jsonb_agg(match_val ORDER BY result_number)
END
END AS extracted
FROM rx
GROUP BY id, rule_name, sequence, output_field, retain, function_type
),
linked AS (
SELECT
a.id, a.sequence, a.output_field, a.retain, a.extracted, m.output AS mapped
FROM agg_matches a
LEFT JOIN dataflow.mappings m ON
m.source_name = p_source_name
AND m.rule_name = a.rule_name
AND m.input_value = a.extracted
WHERE a.extracted IS NOT NULL
),
rule_output AS (
SELECT
id, sequence,
CASE
WHEN mapped IS NOT NULL THEN
mapped || CASE WHEN retain THEN jsonb_build_object(output_field, extracted) ELSE '{}'::jsonb END
ELSE jsonb_build_object(output_field, extracted)
END AS output
FROM linked
),
record_additions AS (
SELECT id, dataflow.jsonb_concat_obj(output ORDER BY sequence) AS additions
FROM rule_output GROUP BY id
),
updated AS (
UPDATE dataflow.records rec
SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb),
transformed_at = CURRENT_TIMESTAMP
FROM qualifying q
LEFT JOIN record_additions ra ON ra.id = q.id
WHERE rec.id = q.id
RETURNING rec.id
)
SELECT json_build_object('success', true, 'transformed', count(*)) FROM updated
$$ LANGUAGE sql;
CREATE OR REPLACE FUNCTION reprocess_records(p_source_name TEXT)
RETURNS JSON AS $$
SELECT dataflow.apply_transformations(p_source_name, NULL, TRUE)
$$ LANGUAGE sql;
-- ── View generation ───────────────────────────────────────────────────────────
CREATE OR REPLACE FUNCTION generate_source_view(p_source_name TEXT)

View File

@ -8,6 +8,7 @@ import Rules from './pages/Rules'
import Mappings from './pages/Mappings'
import Records from './pages/Records'
import Log from './pages/Log'
import Pivot from './pages/Pivot'
const NAV = [
{ to: '/sources', label: 'Sources' },
@ -15,6 +16,7 @@ const NAV = [
{ to: '/rules', label: 'Rules' },
{ to: '/mappings', label: 'Mappings' },
{ to: '/records', label: 'Records' },
{ to: '/pivot', label: 'Pivot' },
{ to: '/log', label: 'Log' },
]
@ -77,7 +79,7 @@ export default function App() {
<div className="px-3 py-3 border-b border-gray-200">
<div className="flex items-center justify-between mb-1">
<label className="text-xs text-gray-500">Source</label>
<NavLink to="/sources" className="text-xs text-blue-400 hover:text-blue-600 leading-none" title="New source" onClick={() => setSidebarOpen(false)}>+</NavLink>
<NavLink to="/sources?new=1" className="text-xs text-blue-400 hover:text-blue-600 leading-none" title="New source" onClick={() => setSidebarOpen(false)}>+</NavLink>
</div>
<select
className="w-full text-sm border border-gray-200 rounded px-2 py-1 bg-white focus:outline-none focus:border-blue-400"
@ -143,6 +145,7 @@ export default function App() {
<Route path="/rules" element={<Rules source={source} />} />
<Route path="/mappings" element={<Mappings source={source} />} />
<Route path="/records" element={<Records source={source} />} />
<Route path="/pivot" element={<Pivot source={source} />} />
<Route path="/log" element={<Log />} />
</Routes>
</div>

View File

@ -117,6 +117,11 @@ export default function Mappings({ source }) {
const [importing, setImporting] = useState(false)
const [sortBy, setSortBy] = useState(null)
const [globalValues, setGlobalValues] = useState({})
const [selected, setSelected] = useState(new Set())
const [bulkDraft, setBulkDraft] = useState({})
const [cursorKey, setCursorKey] = useState(null)
const [rowFilter, setRowFilter] = useState('')
const rowRefs = useRef({})
useEffect(() => {
if (!source) return
@ -135,11 +140,27 @@ export default function Mappings({ source }) {
setAllValues(a)
setDrafts({})
setExtraCols([])
setSelected(new Set())
setBulkDraft({})
setCursorKey(null)
setRowFilter('')
})
.catch(() => {})
.finally(() => setLoading(false))
}, [source, selectedRule])
// Auto-select all rows matching the regex filter when it changes
useEffect(() => {
if (!rowFilter) return
let re = null
try { re = new RegExp(rowFilter, 'i') } catch { return }
const tabF = filter === 'unmapped' ? allValues.filter(r => !r.is_mapped)
: filter === 'mapped' ? allValues.filter(r => r.is_mapped)
: allValues
const matches = tabF.filter(r => re.test(displayValue(r.extracted_value)))
setSelected(new Set(matches.map(r => valueKey(r.extracted_value))))
}, [rowFilter, filter, allValues])
// Derive output columns and datalist suggestions from mapped rows + global pool
const existingCols = []
const valuesByCol = {}
@ -161,12 +182,21 @@ export default function Mappings({ source }) {
const unmappedCount = allValues.filter(r => !r.is_mapped).length
const mappedCount = allValues.filter(r => r.is_mapped).length
const filteredRows = filter === 'unmapped'
const tabFiltered = filter === 'unmapped'
? allValues.filter(r => !r.is_mapped)
: filter === 'mapped'
? allValues.filter(r => r.is_mapped)
: allValues
let rowFilterRe = null
let rowFilterError = false
if (rowFilter) {
try { rowFilterRe = new RegExp(rowFilter, 'i') } catch { rowFilterError = true }
}
const filteredRows = rowFilterRe
? tabFiltered.filter(r => rowFilterRe.test(displayValue(r.extracted_value)))
: tabFiltered
function toggleSort(col) {
setSortBy(s => {
if (s?.col === col) return { col, dir: s.dir === 'asc' ? 'desc' : 'asc' }
@ -196,7 +226,12 @@ export default function Mappings({ source }) {
function setCellValue(extractedValue, col, value) {
const k = valueKey(extractedValue)
setDrafts(d => ({ ...d, [k]: { ...(d[k] || {}), [col]: value } }))
const targets = selected.has(k) && selected.size > 1 ? [...selected] : [k]
setDrafts(d => {
const next = { ...d }
for (const sk of targets) next[sk] = { ...(next[sk] || {}), [col]: value }
return next
})
}
async function saveRow(row) {
@ -249,6 +284,34 @@ export default function Mappings({ source }) {
await Promise.all(dirty.map(row => saveRow(row)))
}
async function applyBulk() {
const output = Object.fromEntries(
Object.entries(bulkDraft).filter(([, v]) => v.trim())
)
if (Object.keys(output).length === 0) return
const rows = sortedRows(filteredRows).filter(r => selected.has(valueKey(r.extracted_value)))
await Promise.all(rows.map(async row => {
const k = valueKey(row.extracted_value)
const merged = { ...(row.is_mapped ? row.output : {}), ...output }
setSaving(s => ({ ...s, [k]: true }))
try {
if (row.is_mapped && row.mapping_id) {
const updated = await api.updateMapping(row.mapping_id, { output: merged })
setAllValues(av => av.map(x => valueKey(x.extracted_value) === k ? { ...x, output: updated.output } : x))
} else {
const created = await api.createMapping({ source_name: source, rule_name: row.rule_name, input_value: row.extracted_value, output: merged })
setAllValues(av => av.map(x => valueKey(x.extracted_value) === k ? { ...x, is_mapped: true, mapping_id: created.id, output: merged } : x))
}
} catch (err) {
alert(err.message)
} finally {
setSaving(s => ({ ...s, [k]: false }))
}
}))
setSelected(new Set())
setBulkDraft({})
}
async function deleteRow(row) {
if (!row.mapping_id) return
try {
@ -318,6 +381,24 @@ export default function Mappings({ source }) {
</div>
)}
{selectedRule && (
<div className="relative">
<input
className={`text-xs font-mono border rounded px-2 py-1.5 w-44 focus:outline-none focus:border-blue-400 ${
rowFilterError ? 'border-red-400 bg-red-50' : rowFilter ? 'border-blue-300' : 'border-gray-200'
}`}
placeholder="filter regex…"
value={rowFilter}
onChange={e => setRowFilter(e.target.value)}
/>
{rowFilter && !rowFilterError && (
<span className="absolute right-2 top-1/2 -translate-y-1/2 text-xs text-gray-400">
{filteredRows.length}
</span>
)}
</div>
)}
{dirtyCount > 0 && (
<button
onClick={saveAllPending}
@ -370,9 +451,49 @@ export default function Mappings({ source }) {
)}
{selectedRule && !loading && allValues.length > 0 && (
<div className="overflow-x-auto">
{/* Bulk assign bar */}
{selected.size > 0 && (
<div className="flex items-center gap-2 mb-2 p-2 bg-blue-50 border border-blue-200 rounded flex-wrap">
<span className="text-xs text-blue-700 font-medium whitespace-nowrap">{selected.size} selected</span>
{cols.map(col => (
<AutocompleteInput
key={col}
className="border border-blue-300 rounded px-2 py-1 text-xs min-w-24 focus:outline-none focus:border-blue-500 bg-white"
placeholder={col}
value={bulkDraft[col] || ''}
onChange={v => setBulkDraft(d => ({ ...d, [col]: v }))}
suggestions={[...(valuesByCol[col] || [])].sort()}
/>
))}
<button
onClick={applyBulk}
disabled={Object.values(bulkDraft).every(v => !v.trim())}
className="text-xs bg-blue-600 text-white px-3 py-1 rounded hover:bg-blue-700 disabled:opacity-40 whitespace-nowrap"
>
Apply to {selected.size}
</button>
<button
onClick={() => { setSelected(new Set()); setBulkDraft({}) }}
className="text-xs text-blue-400 hover:text-blue-600"
>
cancel
</button>
</div>
)}
<table className="w-full text-xs bg-white border border-gray-200 rounded">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
<th className="px-2 py-2 w-6">
<input
type="checkbox"
className="cursor-pointer"
checked={displayRows.length > 0 && displayRows.every(r => selected.has(valueKey(r.extracted_value)))}
onChange={e => {
if (e.target.checked) setSelected(new Set(displayRows.map(r => valueKey(r.extracted_value))))
else setSelected(new Set())
}}
/>
</th>
<SortHeader col="input_value" label="input_value" sortBy={sortBy} onSort={toggleSort} />
<SortHeader col="count" label="count" sortBy={sortBy} onSort={toggleSort} className="text-right" />
{existingCols.map(col => (
@ -402,9 +523,29 @@ export default function Mappings({ source }) {
<tbody>
{displayRows.map(row => {
const k = valueKey(row.extracted_value)
const rowIdx = displayRows.indexOf(row)
const isSaving = saving[k]
const isSelected = selected.has(k)
const hasDraft = !!(drafts[k] && Object.keys(drafts[k]).length > 0)
const rowBg = hasDraft ? 'bg-blue-50' : row.is_mapped ? '' : 'bg-yellow-50'
const rowBg = isSelected ? 'bg-blue-50' : hasDraft ? 'bg-blue-50' : row.is_mapped ? '' : 'bg-yellow-50'
function handleRowClick(e) {
if (e.target.closest('input,button,a,select')) return
setSelected(s => { const n = new Set(s); n.has(k) ? n.delete(k) : n.add(k); return n })
setCursorKey(k)
}
function handleRowKeyDown(e) {
if (!e.shiftKey || (e.key !== 'ArrowDown' && e.key !== 'ArrowUp')) return
e.preventDefault()
const delta = e.key === 'ArrowDown' ? 1 : -1
const curIdx = cursorKey ? displayRows.findIndex(r => valueKey(r.extracted_value) === cursorKey) : rowIdx
const nextIdx = Math.max(0, Math.min(displayRows.length - 1, curIdx + delta))
const nextKey = valueKey(displayRows[nextIdx].extracted_value)
setSelected(s => new Set([...s, nextKey]))
setCursorKey(nextKey)
rowRefs.current[nextKey]?.focus()
}
const samples = row.sample
? (Array.isArray(row.sample) ? row.sample : [row.sample])
: []
@ -417,7 +558,25 @@ export default function Mappings({ source }) {
return (
<>
<tr key={k} className={`border-t border-gray-50 hover:bg-gray-50 ${rowBg}`}>
<tr
key={k}
ref={el => rowRefs.current[k] = el}
tabIndex={0}
className={`border-t border-gray-50 hover:bg-gray-50 cursor-pointer outline-none ${rowBg}`}
onClick={handleRowClick}
onKeyDown={handleRowKeyDown}
>
<td className="px-2 py-1.5">
<input
type="checkbox"
className="cursor-pointer"
checked={isSelected}
onChange={() => {
setSelected(s => { const n = new Set(s); n.has(k) ? n.delete(k) : n.add(k); return n })
setCursorKey(k)
}}
/>
</td>
<td className="px-3 py-1.5 font-mono text-gray-800 whitespace-nowrap">{displayValue(row.extracted_value)}</td>
<td className="px-3 py-1.5 text-right text-gray-400">{row.record_count}</td>
{cols.map(col => (
@ -467,7 +626,7 @@ export default function Mappings({ source }) {
const sampleCols = [...new Set(samples.flatMap(r => Object.keys(r)))]
return (
<tr key={`${k}-sample`} className="border-t border-gray-50 bg-gray-50">
<td colSpan={2 + cols.length + 4} className="px-3 py-2">
<td colSpan={3 + cols.length + 4} className="px-3 py-2">
<table className="w-full text-xs border border-gray-100 rounded bg-white">
<thead>
<tr className="bg-gray-50 border-b border-gray-100">

155
ui/src/pages/Pivot.jsx Normal file
View File

@ -0,0 +1,155 @@
import { useEffect, useRef, useState } from 'react'
import { api } from '../api'
// Fetch all rows for a source (no pagination limit)
async function fetchAllRows(source) {
const res = await api.getViewData(source, 100000, 0)
return res.rows || []
}
let perspectiveLoaded = false
let perspectivePromise = null
function loadPerspective() {
if (perspectivePromise) return perspectivePromise
perspectivePromise = (async () => {
// Inject theme CSS once
if (!document.getElementById('psp-theme')) {
const link = document.createElement('link')
link.id = 'psp-theme'
link.rel = 'stylesheet'
link.crossOrigin = 'anonymous'
link.href = 'https://cdn.jsdelivr.net/npm/@perspective-dev/viewer/dist/css/themes.css'
document.head.appendChild(link)
}
const [{ default: perspective }] = await Promise.all([
import(/* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/@perspective-dev/client@4.4.0/dist/cdn/perspective.js'),
import(/* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/@perspective-dev/viewer@4.4.0/dist/cdn/perspective-viewer.js'),
import(/* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/@perspective-dev/viewer-datagrid@4.4.0/dist/cdn/perspective-viewer-datagrid.js'),
import(/* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/@perspective-dev/viewer-d3fc@4.4.0/dist/cdn/perspective-viewer-d3fc.js'),
])
perspectiveLoaded = true
return perspective
})()
return perspectivePromise
}
const LAYOUT_KEY = (source) => `psp_layout_${source}`
export default function Pivot({ source }) {
const viewerRef = useRef()
const workerRef = useRef()
const [status, setStatus] = useState('idle')
const [error, setError] = useState('')
const [saved, setSaved] = useState(false)
useEffect(() => {
if (!source) return
let cancelled = false
setSaved(false)
async function init() {
setStatus('loading')
setError('')
try {
const [perspective, rows] = await Promise.all([
loadPerspective(),
fetchAllRows(source),
])
if (cancelled) return
if (!rows.length) {
setStatus('noview')
return
}
if (workerRef.current) {
try { workerRef.current.terminate() } catch {}
}
const worker = await perspective.worker()
if (cancelled) { worker.terminate(); return }
workerRef.current = worker
await worker.table(rows, { name: source })
if (cancelled) return
const viewer = viewerRef.current
await viewer.load(worker)
const saved = localStorage.getItem(LAYOUT_KEY(source))
if (saved) {
await viewer.restore(JSON.parse(saved))
} else {
await viewer.restore({ table: source, settings: true })
}
setStatus('ready')
} catch (err) {
if (!cancelled) { setStatus('error'); setError(err.message) }
}
}
init()
return () => { cancelled = true }
}, [source])
async function saveLayout() {
const viewer = viewerRef.current
if (!viewer) return
const layout = await viewer.save()
localStorage.setItem(LAYOUT_KEY(source), JSON.stringify(layout))
setSaved(true)
setTimeout(() => setSaved(false), 2000)
}
function clearLayout() {
localStorage.removeItem(LAYOUT_KEY(source))
const viewer = viewerRef.current
if (viewer) viewer.restore({ table: source, settings: true })
}
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
return (
<div className="relative w-full h-full">
{status === 'ready' && (
<div className="absolute top-2 right-3 z-20 flex gap-2">
<button
onClick={saveLayout}
className="text-xs bg-white border border-gray-200 rounded px-2 py-1 text-gray-500 hover:border-gray-400 shadow-sm"
>
{saved ? 'Saved!' : 'Save layout'}
</button>
{localStorage.getItem(LAYOUT_KEY(source)) && (
<button
onClick={clearLayout}
className="text-xs bg-white border border-gray-200 rounded px-2 py-1 text-gray-400 hover:text-red-500 shadow-sm"
>
Reset
</button>
)}
</div>
)}
{status === 'loading' && (
<div className="absolute inset-0 flex items-center justify-center z-10 bg-gray-50">
<p className="text-sm text-gray-400">Loading</p>
</div>
)}
{status === 'error' && (
<div className="absolute inset-0 flex items-center justify-center z-10 bg-gray-50">
<p className="text-sm text-red-500">Error: {error}</p>
</div>
)}
{status === 'noview' && (
<div className="absolute inset-0 flex items-center justify-center z-10 bg-gray-50">
<p className="text-sm text-gray-400">No view data generate a view and transform records first.</p>
</div>
)}
<perspective-viewer
ref={viewerRef}
style={{ position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 }}
/>
</div>
)
}

View File

@ -24,6 +24,7 @@ export default function Records({ source }) {
const [exists, setExists] = useState(null)
const [offset, setOffset] = useState(0)
const [loading, setLoading] = useState(false)
const [viewError, setViewError] = useState(null)
const [sort, setSort] = useState({ col: null, dir: 'asc' })
const [filters, setFilters] = useState([])
const debounceRef = useRef(null)
@ -34,6 +35,7 @@ export default function Records({ source }) {
setOffset(0)
setSort({ col: null, dir: 'asc' })
setFilters([])
setViewError(null)
load(0, null, 'asc', [])
}, [source])
@ -47,7 +49,7 @@ export default function Records({ source }) {
if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
} catch (err) {
console.error(err)
setViewError(err.message)
} finally {
setLoading(false)
}
@ -145,6 +147,10 @@ export default function Records({ source }) {
{loading && <p className="text-sm text-gray-400">Loading</p>}
{!loading && viewError && (
<p className="text-sm text-red-500">View error: {viewError} check field types in Sources.</p>
)}
{!loading && exists === false && (
<p className="text-sm text-gray-400">
No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.

View File

@ -1,13 +1,42 @@
import { useState, useEffect, useRef } from 'react'
import { useSearchParams } from 'react-router-dom'
import { api } from '../api'
const FIELD_TYPES = ['text', 'numeric', 'date']
function SampleTable({ rows }) {
if (!rows || rows.length === 0) return null
const cols = Object.keys(rows[0])
return (
<div className="overflow-auto border border-gray-100 rounded bg-gray-50 max-h-36">
<table className="text-xs w-full">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50 sticky top-0">
{cols.map(c => <th key={c} className="px-2 py-1 font-medium whitespace-nowrap">{c}</th>)}
</tr>
</thead>
<tbody>
{rows.map((row, i) => (
<tr key={i} className="border-t border-gray-100">
{cols.map(c => (
<td key={c} className="px-2 py-1 whitespace-nowrap text-gray-600 max-w-32 truncate font-mono">
{row[c] == null ? <span className="text-gray-300"></span> : String(row[c])}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
)
}
export default function Sources({ source, sources, setSources, setSource }) {
const [constraintFields, setConstraintFields] = useState('')
const [globalPicklist, setGlobalPicklist] = useState(true)
const [schemaFields, setSchemaFields] = useState([])
const [stats, setStats] = useState(null)
const [sampleRows, setSampleRows] = useState([])
const [saving, setSaving] = useState(false)
const [reprocessing, setReprocessing] = useState(false)
const [generating, setGenerating] = useState(false)
@ -15,14 +44,24 @@ export default function Sources({ source, sources, setSources, setSource }) {
const [error, setError] = useState('')
const [viewName, setViewName] = useState('')
const [availableFields, setAvailableFields] = useState([])
const [fieldSort, setFieldSort] = useState({ col: 'key', dir: 'asc' })
const [creating, setCreating] = useState(false)
const [form, setForm] = useState({ name: '', constraint_fields: '', fields: [], schema: [] })
const [form, setForm] = useState({ name: '', constraint_fields: '', fields: [], schema: [], importSample: true })
const [createError, setCreateError] = useState('')
const [createLoading, setCreateLoading] = useState(false)
const [csvFileName, setCsvFileName] = useState('')
const fileRef = useRef()
const [searchParams, setSearchParams] = useSearchParams()
const sourceObj = sources.find(s => s.name === source)
useEffect(() => {
if (searchParams.get('new') === '1') {
setCreating(true)
setSearchParams({})
}
}, [searchParams])
useEffect(() => {
if (!sourceObj) return
setConstraintFields(sourceObj.constraint_fields?.join(', ') || '')
@ -33,8 +72,10 @@ export default function Sources({ source, sources, setSources, setSource }) {
setError('')
setStats(null)
setAvailableFields([])
setSampleRows([])
api.getStats(sourceObj.name).then(setStats).catch(() => {})
api.getFields(sourceObj.name).then(setAvailableFields).catch(() => {})
api.getRecords(sourceObj.name, 50).then(rows => setSampleRows(rows.map(r => r.data).filter(Boolean))).catch(() => {})
}, [source, sourceObj?.name])
async function handleSave(e) {
@ -46,6 +87,10 @@ export default function Sources({ source, sources, setSources, setSource }) {
const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
const config = { ...(sourceObj.config || {}), fields }
await api.updateSource(sourceObj.name, { constraint_fields, config, global_picklist: globalPicklist })
if (fields.length > 0) {
const res = await api.generateView(sourceObj.name)
if (res.success) setViewName(res.view)
}
const updated = await api.getSources()
setSources(updated)
setResult('Saved.')
@ -111,13 +156,15 @@ export default function Sources({ source, sources, setSources, setSource }) {
async function handleSuggest(e) {
const file = e.target.files[0]
if (!file) return
setCsvFileName(file.name)
try {
const suggestion = await api.suggestSource(file)
setForm(f => ({
...f,
fields: suggestion.fields,
constraint_fields: '',
schema: suggestion.fields.map(f => ({ name: f.name, type: f.type }))
schema: suggestion.fields.map(f => ({ name: f.name, type: f.type, seq: suggestion.fields.indexOf(f) + 1 })),
sampleRows: suggestion.sampleRows || []
}))
} catch (err) {
setCreateError(err.message)
@ -136,10 +183,16 @@ export default function Sources({ source, sources, setSources, setSource }) {
try {
const config = form.schema.length > 0 ? { fields: form.schema } : {}
await api.createSource({ name: form.name, constraint_fields: constraintArr, config, global_picklist: form.global_picklist !== false })
if (form.schema.length > 0) {
await api.generateView(form.name)
}
if (form.importSample && fileRef.current?.files[0]) {
await api.importCSV(form.name, fileRef.current.files[0])
}
const updated = await api.getSources()
setSources(updated)
setSource(form.name)
setForm({ name: '', constraint_fields: '', fields: [], schema: [], global_picklist: true })
setForm({ name: '', constraint_fields: '', fields: [], schema: [], importSample: true })
setCreating(false)
} catch (err) {
setCreateError(err.message)
@ -149,7 +202,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
}
return (
<div className="p-6 max-w-2xl">
<div className="p-6 max-w-5xl">
<div className="flex items-center justify-between mb-6">
<h1 className="text-xl font-semibold text-gray-800">
{sourceObj ? sourceObj.name : 'Sources'}
@ -185,16 +238,43 @@ export default function Sources({ source, sources, setSources, setSource }) {
<table className="w-full text-xs">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="pb-1 font-medium">Key</th>
<th className="pb-1 font-medium">Origin</th>
<th className="pb-1 font-medium">Type</th>
<th className="pb-1 font-medium text-center">Constraint</th>
<th className="pb-1 font-medium text-center">In view</th>
<th className="pb-1 font-medium text-center">Seq</th>
{[
{ col: 'key', label: 'Key' },
{ col: 'origin', label: 'Origin' },
{ col: 'type', label: 'Type' },
{ col: 'constraint', label: 'Constraint', center: true },
{ col: 'inview', label: 'In view', center: true },
{ col: 'seq', label: 'Seq', center: true },
].map(({ col, label, center }) => (
<th
key={col}
onClick={() => setFieldSort(s => ({ col, dir: s.col === col && s.dir === 'asc' ? 'desc' : 'asc' }))}
className={`pb-1 font-medium cursor-pointer select-none hover:text-gray-600 ${center ? 'text-center' : ''}`}
>
{label}
<span className="ml-1 text-gray-300">
{fieldSort.col === col ? (fieldSort.dir === 'asc' ? '▲' : '▼') : '⇅'}
</span>
</th>
))}
</tr>
</thead>
<tbody>
{availableFields.map(f => {
{[...availableFields].sort((a, b) => {
const constraintList = constraintFields.split(',').map(s => s.trim())
const aSchema = schemaFields.find(sf => sf.name === a.key)
const bSchema = schemaFields.find(sf => sf.name === b.key)
let av, bv
if (fieldSort.col === 'key') { av = a.key; bv = b.key }
else if (fieldSort.col === 'origin') { av = a.origins.join(','); bv = b.origins.join(',') }
else if (fieldSort.col === 'type') { av = aSchema?.type || ''; bv = bSchema?.type || '' }
else if (fieldSort.col === 'constraint') { av = constraintList.includes(a.key) ? 0 : 1; bv = constraintList.includes(b.key) ? 0 : 1 }
else if (fieldSort.col === 'inview') { av = aSchema ? 0 : 1; bv = bSchema ? 0 : 1 }
else if (fieldSort.col === 'seq') { av = aSchema?.seq ?? 999; bv = bSchema?.seq ?? 999 }
if (av < bv) return fieldSort.dir === 'asc' ? -1 : 1
if (av > bv) return fieldSort.dir === 'asc' ? 1 : -1
return 0
}).map(f => {
const isRaw = f.origins.includes('raw')
const constraintChecked = constraintFields.split(',').map(s => s.trim()).includes(f.key)
const schemaEntry = schemaFields.find(sf => sf.name === f.key)
@ -301,6 +381,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
</>
)}
</div>
<SampleTable rows={sampleRows} />
</div>
)}
@ -349,8 +430,14 @@ export default function Sources({ source, sources, setSources, setSource }) {
<h2 className="text-sm font-semibold text-gray-700 mb-3">New source</h2>
<div className="mb-4">
<label className="text-xs text-gray-500 block mb-1">Upload a CSV to auto-detect fields</label>
<input type="file" accept=".csv" ref={fileRef} onChange={handleSuggest} className="text-sm text-gray-600" />
<input type="file" accept=".csv" ref={fileRef} onChange={handleSuggest} className="hidden" />
<button
type="button"
onClick={() => fileRef.current?.click()}
className="text-sm border border-gray-300 rounded px-3 py-1.5 text-gray-600 hover:bg-gray-50 hover:border-gray-400"
>
{csvFileName || 'Choose CSV…'}
</button>
</div>
<form onSubmit={handleCreate} className="space-y-3">
@ -365,38 +452,87 @@ export default function Sources({ source, sources, setSources, setSource }) {
</div>
{form.fields.length > 0 && (
<div>
<label className="text-xs text-gray-500 block mb-1">Detected fields check to use as constraint fields</label>
<div className="pt-2 border-t border-gray-100 space-y-2">
<table className="w-full text-xs">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="pb-1 font-medium">Field</th>
<th className="pb-1 font-medium">Key</th>
<th className="pb-1 font-medium">Type</th>
<th className="pb-1 font-medium text-center">Constraint</th>
<th className="pb-1 font-medium text-center">In view</th>
<th className="pb-1 font-medium text-center">Seq</th>
</tr>
</thead>
<tbody>
{form.fields.map(f => (
<tr key={f.name} className="border-t border-gray-50">
<td className="py-1 font-mono text-gray-700">{f.name}</td>
<td className="py-1 text-gray-400">{f.type}</td>
<td className="py-1 text-center">
<input
type="checkbox"
checked={form.constraint_fields.split(',').map(s => s.trim()).includes(f.name)}
onChange={e => {
const current = form.constraint_fields.split(',').map(s => s.trim()).filter(Boolean)
const next = e.target.checked
? [...current, f.name]
: current.filter(n => n !== f.name)
setForm(ff => ({ ...ff, constraint_fields: next.join(', ') }))
}}
/>
</td>
</tr>
))}
{form.fields.map(f => {
const schemaEntry = form.schema.find(s => s.name === f.name)
const inView = !!schemaEntry
const currentType = schemaEntry?.type || f.type
return (
<tr key={f.name} className="border-t border-gray-50">
<td className="py-1 font-mono text-gray-700">{f.name}</td>
<td className="py-1">
{inView && (
<select
className="border border-gray-200 rounded px-1 py-0.5 text-xs focus:outline-none focus:border-blue-400"
value={currentType}
onChange={e => setForm(ff => ({
...ff,
schema: ff.schema.map(s => s.name === f.name ? { ...s, type: e.target.value } : s)
}))}
>
{FIELD_TYPES.map(t => <option key={t} value={t}>{t}</option>)}
</select>
)}
</td>
<td className="py-1 text-center">
<input
type="checkbox"
checked={form.constraint_fields.split(',').map(s => s.trim()).includes(f.name)}
onChange={e => {
const current = form.constraint_fields.split(',').map(s => s.trim()).filter(Boolean)
const next = e.target.checked
? [...current, f.name]
: current.filter(n => n !== f.name)
setForm(ff => ({ ...ff, constraint_fields: next.join(', ') }))
}}
/>
</td>
<td className="py-1 text-center">
<input
type="checkbox"
checked={inView}
onChange={e => {
if (e.target.checked) {
const nextSeq = form.schema.length > 0
? Math.max(...form.schema.map(s => s.seq ?? 0)) + 1
: 1
setForm(ff => ({ ...ff, schema: [...ff.schema, { name: f.name, type: f.type, seq: nextSeq }] }))
} else {
setForm(ff => ({ ...ff, schema: ff.schema.filter(s => s.name !== f.name) }))
}
}}
/>
</td>
<td className="py-1 text-center">
{inView && (
<input
type="number"
className="w-12 border border-gray-200 rounded px-1 py-0.5 text-xs text-center focus:outline-none focus:border-blue-400"
value={schemaEntry.seq ?? ''}
onChange={e => setForm(ff => ({
...ff,
schema: ff.schema.map(s => s.name === f.name ? { ...s, seq: parseInt(e.target.value) || 0 } : s)
}))}
/>
)}
</td>
</tr>
)
})}
</tbody>
</table>
<SampleTable rows={form.sampleRows || []} />
</div>
)}
@ -412,14 +548,26 @@ export default function Sources({ source, sources, setSources, setSource }) {
</div>
)}
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
<input
type="checkbox"
checked={form.global_picklist !== false}
onChange={e => setForm(f => ({ ...f, global_picklist: e.target.checked }))}
/>
Global picklist
</label>
<div className="flex gap-4">
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
<input
type="checkbox"
checked={form.global_picklist !== false}
onChange={e => setForm(f => ({ ...f, global_picklist: e.target.checked }))}
/>
Global picklist
</label>
{form.fields.length > 0 && (
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
<input
type="checkbox"
checked={form.importSample !== false}
onChange={e => setForm(f => ({ ...f, importSample: e.target.checked }))}
/>
Import sample data
</label>
)}
</div>
{createError && <p className="text-xs text-red-500">{createError}</p>}