pt/dataflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add multi-capture regex, computed view fields, collapsible rules, and live preview

Browse Source 
- Support multi-capture-group regex: mappings.input_value changed to JSONB,
  regexp_match() result stored as scalar or array JSONB in transformed column
- Computed expression fields in generated views: {fieldname} refs substituted
  with (transformed->>'fieldname')::numeric for arithmetic in view columns
- Fix generate_source_view to DROP VIEW before CREATE (avoids column drop error)
- Collapsible rule cards that open directly to inline edit form
- Debounced live regex preview (extract + replace) with popout modal for 50 rows
- Records page now shows dfv.<source> view output instead of raw records
- Unified field table in Sources: single table with In view, Seq, expression columns
- Fix "Rule already exists" error when editing by passing rule.id directly to submit
- Fix Sources page clearing on F5 by watching sourceObj?.name in useEffect dep

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-03-29 16:37:15 -04:00
parent eb50704ca0
commit 928a54932d
12 changed files with 658 additions and 501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/routes/mappings.js
View File

@ -79,7 +79,7 @@ module.exports = (pool) => {
`INSERT INTO mappings (source_name, rule_name, input_value, output) `INSERT INTO mappings (source_name, rule_name, input_value, output)
VALUES ($1, $2, $3, $4) VALUES ($1, $2, $3, $4)
RETURNING *`, RETURNING *`,
[source_name, rule_name, input_value, JSON.stringify(output)] [source_name, rule_name, JSON.stringify(input_value), JSON.stringify(output)]
); );
res.status(201).json(result.rows[0]); res.status(201).json(result.rows[0]);
@ -116,7 +116,7 @@ module.exports = (pool) => {
ON CONFLICT (source_name, rule_name, input_value) ON CONFLICT (source_name, rule_name, input_value)
DO UPDATE SET output = EXCLUDED.output DO UPDATE SET output = EXCLUDED.output
RETURNING *`, RETURNING *`,
[source_name, rule_name, input_value, JSON.stringify(output)] [source_name, rule_name, JSON.stringify(input_value), JSON.stringify(output)]
); );
results.push(result.rows[0]); results.push(result.rows[0]);

79
api/routes/rules.js
View File

@ -21,6 +21,49 @@ module.exports = (pool) => {
} }
}); });
// Preview an ad-hoc pattern against real records (no saved rule needed)
router.get('/preview', async (req, res, next) => {
try {
const { source, field, pattern, flags, function_type = 'extract', replace_value = '', limit = 20 } = req.query;
if (!source || !field || !pattern) {
return res.status(400).json({ error: 'source, field, and pattern are required' });
}
const fullPattern = (flags ? `(?${flags})` : '') + pattern;
const query = function_type === 'replace'
? `SELECT
id,
data->>$1 AS raw_value,
to_jsonb(regexp_replace(data->>$1, $2, $3)) AS extracted_value
FROM records
WHERE source_name = $4 AND data ? $1
ORDER BY id DESC LIMIT $5`
: `SELECT
r.id,
r.data->>$1 AS raw_value,
CASE
WHEN m.match IS NULL THEN NULL
WHEN cardinality(m.match) = 1 THEN to_jsonb(m.match[1])
ELSE to_jsonb(m.match)
END AS extracted_value
FROM records r
CROSS JOIN LATERAL (SELECT regexp_match(r.data->>$1, $2) AS match) m
WHERE r.source_name = $3 AND r.data ? $1
ORDER BY r.id DESC LIMIT $4`;
const params = function_type === 'replace'
? [field, fullPattern, replace_value, source, parseInt(limit)]
: [field, fullPattern, source, parseInt(limit)];
const result = await pool.query(query, params);
res.json(result.rows);
} catch (err) {
next(err);
}
});
// Test a rule against real records // Test a rule against real records
router.get('/:id/test', async (req, res, next) => { router.get('/:id/test', async (req, res, next) => {
try { try {
@ -40,13 +83,18 @@ module.exports = (pool) => {
const pattern = (rule.flags ? `(?${rule.flags})` : '') + rule.pattern; const pattern = (rule.flags ? `(?${rule.flags})` : '') + rule.pattern;
const result = await pool.query( const result = await pool.query(
`SELECT `SELECT
id, r.id,
data->>$1 AS raw_value, r.data->>$1 AS raw_value,
substring(data->>$1 FROM $2) AS extracted_value CASE
FROM records WHEN m.match IS NULL THEN NULL
WHERE source_name = $3 WHEN cardinality(m.match) = 1 THEN to_jsonb(m.match[1])
AND data ? $1 ELSE to_jsonb(m.match)
ORDER BY id DESC END AS extracted_value
FROM records r
CROSS JOIN LATERAL (SELECT regexp_match(r.data->>$1, $2) AS match) m
WHERE r.source_name = $3
AND r.data ? $1
ORDER BY r.id DESC
LIMIT $4`, LIMIT $4`,
[rule.field, pattern, rule.source_name, parseInt(limit)] [rule.field, pattern, rule.source_name, parseInt(limit)]
); );
@ -81,7 +129,7 @@ module.exports = (pool) => {
// Create rule // Create rule
router.post('/', async (req, res, next) => { router.post('/', async (req, res, next) => {
try { try {
const { source_name, name, field, pattern, output_field, function_type, flags, enabled, sequence } = req.body; const { source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
if (!source_name || !name || !field || !pattern || !output_field) { if (!source_name || !name || !field || !pattern || !output_field) {
return res.status(400).json({ return res.status(400).json({
@ -94,10 +142,10 @@ module.exports = (pool) => {
} }
const result = await pool.query( const result = await pool.query(
`INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, enabled, sequence) `INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING *`, RETURNING *`,
[source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', enabled !== false, sequence || 0] [source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', replace_value || '', enabled !== false, sequence || 0]
); );
res.status(201).json(result.rows[0]); res.status(201).json(result.rows[0]);
@ -115,7 +163,7 @@ module.exports = (pool) => {
// Update rule // Update rule
router.put('/:id', async (req, res, next) => { router.put('/:id', async (req, res, next) => {
try { try {
const { name, field, pattern, output_field, function_type, flags, enabled, sequence } = req.body; const { name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
if (function_type && !['extract', 'replace'].includes(function_type)) { if (function_type && !['extract', 'replace'].includes(function_type)) {
return res.status(400).json({ error: 'function_type must be "extract" or "replace"' }); return res.status(400).json({ error: 'function_type must be "extract" or "replace"' });
@ -129,11 +177,12 @@ module.exports = (pool) => {
output_field = COALESCE($5, output_field), output_field = COALESCE($5, output_field),
function_type = COALESCE($6, function_type), function_type = COALESCE($6, function_type),
flags = COALESCE($7, flags), flags = COALESCE($7, flags),
enabled = COALESCE($8, enabled), replace_value = COALESCE($8, replace_value),
sequence = COALESCE($9, sequence) enabled = COALESCE($9, enabled),
sequence = COALESCE($10, sequence)
WHERE id = $1 WHERE id = $1
RETURNING *`, RETURNING *`,
[req.params.id, name, field, pattern, output_field, function_type, flags, enabled, sequence] [req.params.id, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence]
); );
if (result.rows.length === 0) { if (result.rows.length === 0) {

27
api/routes/sources.js
View File

@ -282,5 +282,32 @@ module.exports = (pool) => {
} }
}); });
router.get('/:name/view-data', async (req, res, next) => {
try {
const { limit = 100, offset = 0 } = req.query;
const viewName = `dfv.${req.params.name}`;
// Check view exists
const check = await pool.query(
`SELECT 1 FROM information_schema.views
WHERE table_schema = 'dfv' AND table_name = $1`,
[req.params.name]
);
if (check.rows.length === 0) {
return res.json({ exists: false, rows: [] });
}
const result = await pool.query(
`SELECT * FROM ${viewName} LIMIT $1 OFFSET $2`,
[parseInt(limit), parseInt(offset)]
);
res.json({ exists: true, rows: result.rows });
} catch (err) {
next(err);
}
});
return router; return router;
}; };

90
database/functions.sql
View File

@ -78,7 +78,8 @@ DECLARE
v_record RECORD; v_record RECORD;
v_rule RECORD; v_rule RECORD;
v_transformed JSONB; v_transformed JSONB;
v_extracted TEXT; v_match TEXT[];
v_extracted JSONB;
v_mapping JSONB; v_mapping JSONB;
v_count INTEGER := 0; v_count INTEGER := 0;
BEGIN BEGIN
@ -102,24 +103,30 @@ BEGIN
LOOP LOOP
-- Apply rule based on function type -- Apply rule based on function type
IF v_rule.function_type = 'replace' THEN IF v_rule.function_type = 'replace' THEN
v_extracted := regexp_replace(
v_record.data->>v_rule.field,
CASE WHEN v_rule.flags != '' THEN '(?' || v_rule.flags || ')' ELSE '' END || v_rule.pattern,
v_rule.output_field
);
v_transformed := jsonb_set( v_transformed := jsonb_set(
v_transformed, v_transformed,
ARRAY[v_rule.field], ARRAY[v_rule.output_field],
to_jsonb(v_extracted) to_jsonb(regexp_replace(
v_record.data->>v_rule.field,
CASE WHEN v_rule.flags != '' THEN '(?' || v_rule.flags || ')' ELSE '' END || v_rule.pattern,
v_rule.replace_value
))
); );
ELSE ELSE
-- extract (default) -- extract (default): regexp_match returns all capture groups as text[]
v_extracted := substring( v_match := regexp_match(
v_record.data->>v_rule.field v_record.data->>v_rule.field,
FROM CASE WHEN v_rule.flags != '' THEN '(?' || v_rule.flags || ')' ELSE '' END || v_rule.pattern CASE WHEN v_rule.flags != '' THEN '(?' || v_rule.flags || ')' ELSE '' END || v_rule.pattern
); );
IF v_extracted IS NOT NULL THEN IF v_match IS NOT NULL THEN
-- Single capture group → scalar string; multiple groups → JSON array
IF cardinality(v_match) = 1 THEN
v_extracted := to_jsonb(v_match[1]);
ELSE
v_extracted := to_jsonb(v_match);
END IF;
-- Check if there's a mapping for this value -- Check if there's a mapping for this value
SELECT output INTO v_mapping SELECT output INTO v_mapping
FROM dataflow.mappings FROM dataflow.mappings
@ -131,11 +138,11 @@ BEGIN
-- Apply mapping (merge mapped fields into result) -- Apply mapping (merge mapped fields into result)
v_transformed := v_transformed || v_mapping; v_transformed := v_transformed || v_mapping;
ELSE ELSE
-- No mapping, just add extracted value -- No mapping, store extracted value (scalar or array)
v_transformed := jsonb_set( v_transformed := jsonb_set(
v_transformed, v_transformed,
ARRAY[v_rule.output_field], ARRAY[v_rule.output_field],
to_jsonb(v_extracted) v_extracted
); );
END IF; END IF;
END IF; END IF;
@ -164,13 +171,14 @@ COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and map
-- Function: get_unmapped_values -- Function: get_unmapped_values
-- Find extracted values that need mappings -- Find extracted values that need mappings
------------------------------------------------------ ------------------------------------------------------
CREATE OR REPLACE FUNCTION get_unmapped_values( DROP FUNCTION IF EXISTS get_unmapped_values(TEXT, TEXT);
CREATE FUNCTION get_unmapped_values(
p_source_name TEXT, p_source_name TEXT,
p_rule_name TEXT DEFAULT NULL p_rule_name TEXT DEFAULT NULL
) RETURNS TABLE ( ) RETURNS TABLE (
rule_name TEXT, rule_name TEXT,
output_field TEXT, output_field TEXT,
extracted_value TEXT, extracted_value JSONB,
record_count BIGINT, record_count BIGINT,
sample_records JSONB sample_records JSONB
) AS $$ ) AS $$
@ -180,7 +188,7 @@ BEGIN
SELECT SELECT
r.name AS rule_name, r.name AS rule_name,
r.output_field, r.output_field,
rec.transformed->>r.output_field AS extracted_value, rec.transformed->r.output_field AS extracted_value,
rec.data AS raw_record rec.data AS raw_record
FROM FROM
dataflow.records rec dataflow.records rec
@ -265,24 +273,44 @@ BEGIN
LOOP LOOP
IF v_cols != '' THEN v_cols := v_cols || ', '; END IF; IF v_cols != '' THEN v_cols := v_cols || ', '; END IF;
CASE v_field->>'type' IF v_field->>'expression' IS NOT NULL THEN
WHEN 'date' THEN -- Computed expression: substitute {fieldname} refs with (transformed->>'fieldname')::type
v_cols := v_cols || format('(transformed->>%L)::date AS %I', -- e.g. "{Amount} * {sign}" → "(transformed->>'Amount')::numeric * (transformed->>'sign')::numeric"
v_field->>'name', v_field->>'name'); DECLARE
WHEN 'numeric' THEN v_expr TEXT := v_field->>'expression';
v_cols := v_cols || format('(transformed->>%L)::numeric AS %I', v_ref TEXT;
v_field->>'name', v_field->>'name'); v_cast TEXT := COALESCE(NULLIF(v_field->>'type', ''), 'numeric');
ELSE BEGIN
v_cols := v_cols || format('transformed->>%L AS %I', WHILE v_expr ~ '\{[^}]+\}' LOOP
v_field->>'name', v_field->>'name'); v_ref := substring(v_expr FROM '\{([^}]+)\}');
END CASE; v_expr := replace(v_expr, '{' || v_ref || '}',
format('(transformed->>%L)::numeric', v_ref));
END LOOP;
v_cols := v_cols || format('%s AS %I', v_expr, v_field->>'name');
END;
ELSE
CASE v_field->>'type'
WHEN 'date' THEN
v_cols := v_cols || format('(transformed->>%L)::date AS %I',
v_field->>'name', v_field->>'name');
WHEN 'numeric' THEN
v_cols := v_cols || format('(transformed->>%L)::numeric AS %I',
v_field->>'name', v_field->>'name');
ELSE
v_cols := v_cols || format('transformed->>%L AS %I',
v_field->>'name', v_field->>'name');
END CASE;
END IF;
END LOOP; END LOOP;
CREATE SCHEMA IF NOT EXISTS dfv; CREATE SCHEMA IF NOT EXISTS dfv;
v_view := 'dfv.' || quote_ident(p_source_name); v_view := 'dfv.' || quote_ident(p_source_name);
v_sql := format(
'CREATE OR REPLACE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL', EXECUTE format('DROP VIEW IF EXISTS %s', v_view);
v_sql := format(
'CREATE VIEW %s AS SELECT %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
v_view, v_cols, p_source_name v_view, v_cols, p_source_name
); );

22
database/migrate_input_value_jsonb.sql Normal file
View File

@ -0,0 +1,22 @@
--
-- Migration: Change mappings.input_value from TEXT to JSONB
-- Allows multi-capture-group regex results to be used as mapping keys
--
SET search_path TO dataflow, public;
-- Drop dependent constraint and index first
ALTER TABLE dataflow.mappings DROP CONSTRAINT mappings_source_name_rule_name_input_value_key;
DROP INDEX IF EXISTS dataflow.idx_mappings_input;
-- Convert column: existing TEXT values become JSONB strings e.g. "MEIJER"
ALTER TABLE dataflow.mappings
ALTER COLUMN input_value TYPE JSONB
USING to_jsonb(input_value);
-- Recreate constraint and index
ALTER TABLE dataflow.mappings
ADD CONSTRAINT mappings_source_name_rule_name_input_value_key
UNIQUE (source_name, rule_name, input_value);
CREATE INDEX idx_mappings_input ON dataflow.mappings(source_name, rule_name, input_value);

3
database/schema.sql
View File

@ -72,6 +72,7 @@ CREATE TABLE rules (
output_field TEXT NOT NULL, -- Name of extracted field (e.g., 'merchant') output_field TEXT NOT NULL, -- Name of extracted field (e.g., 'merchant')
function_type TEXT NOT NULL DEFAULT 'extract', -- 'extract' or 'replace' function_type TEXT NOT NULL DEFAULT 'extract', -- 'extract' or 'replace'
flags TEXT NOT NULL DEFAULT '', -- Regex flags (e.g., 'i' for case-insensitive) flags TEXT NOT NULL DEFAULT '', -- Regex flags (e.g., 'i' for case-insensitive)
replace_value TEXT NOT NULL DEFAULT '', -- Replacement string (replace mode only)
-- Options -- Options
enabled BOOLEAN DEFAULT true, enabled BOOLEAN DEFAULT true,
@ -100,7 +101,7 @@ CREATE TABLE mappings (
rule_name TEXT NOT NULL, rule_name TEXT NOT NULL,
-- Mapping -- Mapping
input_value TEXT NOT NULL, -- Extracted value to match input_value JSONB NOT NULL, -- Extracted value to match (string or array of capture groups)
output JSONB NOT NULL, -- Standardized output output JSONB NOT NULL, -- Standardized output
-- Metadata -- Metadata

11
ui/src/App.jsx
View File

@ -41,7 +41,14 @@ export default function App() {
{/* Source selector */} {/* Source selector */}
<div className="px-3 py-3 border-b border-gray-200"> <div className="px-3 py-3 border-b border-gray-200">
<label className="text-xs text-gray-500 block mb-1">Source</label> <div className="flex items-center justify-between mb-1">
<label className="text-xs text-gray-500">Source</label>
<NavLink to="/sources"
className="text-xs text-blue-400 hover:text-blue-600 leading-none"
title="New source"
onClick={() => {/* nav handled by link */}}
>+</NavLink>
</div>
<select <select
className="w-full text-sm border border-gray-200 rounded px-2 py-1 bg-white focus:outline-none focus:border-blue-400" className="w-full text-sm border border-gray-200 rounded px-2 py-1 bg-white focus:outline-none focus:border-blue-400"
value={source} value={source}
@ -74,7 +81,7 @@ export default function App() {
<div className="flex-1 overflow-auto"> <div className="flex-1 overflow-auto">
<Routes> <Routes>
<Route path="/" element={<Navigate to="/sources" replace />} /> <Route path="/" element={<Navigate to="/sources" replace />} />
<Route path="/sources" element={<Sources sources={sources} setSources={setSources} setSource={setSource} />} /> <Route path="/sources" element={<Sources source={source} sources={sources} setSources={setSources} setSource={setSource} />} />
<Route path="/import" element={<Import source={source} />} /> <Route path="/import" element={<Import source={source} />} />
<Route path="/rules" element={<Rules source={source} />} /> <Route path="/rules" element={<Rules source={source} />} />
<Route path="/mappings" element={<Mappings source={source} />} /> <Route path="/mappings" element={<Mappings source={source} />} />

3
ui/src/api.js
View File

@ -39,6 +39,7 @@ export const api = {
reprocess: (name) => request('POST', `/sources/${name}/reprocess`), reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
generateView: (name) => request('POST', `/sources/${name}/view`), generateView: (name) => request('POST', `/sources/${name}/view`),
getFields: (name) => request('GET', `/sources/${name}/fields`), getFields: (name) => request('GET', `/sources/${name}/fields`),
getViewData: (name, limit = 100, offset = 0) => request('GET', `/sources/${name}/view-data?limit=${limit}&offset=${offset}`),
// Rules // Rules
getRules: (source) => request('GET', `/rules/source/${source}`), getRules: (source) => request('GET', `/rules/source/${source}`),
@ -46,6 +47,8 @@ export const api = {
updateRule: (id, body) => request('PUT', `/rules/${id}`, body), updateRule: (id, body) => request('PUT', `/rules/${id}`, body),
deleteRule: (id) => request('DELETE', `/rules/${id}`), deleteRule: (id) => request('DELETE', `/rules/${id}`),
testRule: (id, limit = 20) => request('GET', `/rules/${id}/test?limit=${limit}`), testRule: (id, limit = 20) => request('GET', `/rules/${id}/test?limit=${limit}`),
previewRule: (source, field, pattern, flags, function_type = 'extract', replace_value = '', limit = 20) =>
request('GET', `/rules/preview?source=${encodeURIComponent(source)}&field=${encodeURIComponent(field)}&pattern=${encodeURIComponent(pattern)}&flags=${encodeURIComponent(flags || '')}&function_type=${function_type}&replace_value=${encodeURIComponent(replace_value)}&limit=${limit}`),
// Mappings // Mappings
getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`), getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`),

50
ui/src/pages/Mappings.jsx
View File

@ -1,6 +1,17 @@
import { useState, useEffect } from 'react' import { useState, useEffect } from 'react'
import { api } from '../api' import { api } from '../api'
// Stable string key for a value that may be a string or array
function valueKey(v) {
return Array.isArray(v) ? JSON.stringify(v) : String(v)
}
// Human-readable display of a string or array extracted value
function displayValue(v) {
if (Array.isArray(v)) return v.join(' · ')
return v
}
export default function Mappings({ source }) { export default function Mappings({ source }) {
const [tab, setTab] = useState('unmapped') const [tab, setTab] = useState('unmapped')
const [rules, setRules] = useState([]) const [rules, setRules] = useState([])
@ -37,40 +48,44 @@ export default function Mappings({ source }) {
}, [source, selectedRule, tab]) }, [source, selectedRule, tab])
function getDraft(extractedValue, outputField) { function getDraft(extractedValue, outputField) {
return drafts[extractedValue] || [{ key: outputField, value: '' }] return drafts[valueKey(extractedValue)] || [{ key: outputField, value: '' }]
} }
function updateDraftKey(extractedValue, index, newKey) { function updateDraftKey(extractedValue, index, newKey) {
const k = valueKey(extractedValue)
setDrafts(d => { setDrafts(d => {
const current = d[extractedValue] || [{ key: '', value: '' }] const current = d[k] || [{ key: '', value: '' }]
const updated = current.map((pair, i) => i === index ? { ...pair, key: newKey } : pair) const updated = current.map((pair, i) => i === index ? { ...pair, key: newKey } : pair)
return { ...d, [extractedValue]: updated } return { ...d, [k]: updated }
}) })
} }
function updateDraftValue(extractedValue, index, newValue) { function updateDraftValue(extractedValue, index, newValue) {
const k = valueKey(extractedValue)
setDrafts(d => { setDrafts(d => {
const current = d[extractedValue] || [{ key: '', value: '' }] const current = d[k] || [{ key: '', value: '' }]
const updated = current.map((pair, i) => i === index ? { ...pair, value: newValue } : pair) const updated = current.map((pair, i) => i === index ? { ...pair, value: newValue } : pair)
return { ...d, [extractedValue]: updated } return { ...d, [k]: updated }
}) })
} }
function addDraftPair(extractedValue, outputField) { function addDraftPair(extractedValue, outputField) {
const k = valueKey(extractedValue)
setDrafts(d => { setDrafts(d => {
const current = d[extractedValue] || [{ key: outputField, value: '' }] const current = d[k] || [{ key: outputField, value: '' }]
return { ...d, [extractedValue]: [...current, { key: '', value: '' }] } return { ...d, [k]: [...current, { key: '', value: '' }] }
}) })
} }
async function saveMapping(row) { async function saveMapping(row) {
const k = valueKey(row.extracted_value)
const pairs = getDraft(row.extracted_value, row.output_field) const pairs = getDraft(row.extracted_value, row.output_field)
const output = Object.fromEntries( const output = Object.fromEntries(
pairs.filter(p => p.key && p.value).map(p => [p.key, p.value]) pairs.filter(p => p.key && p.value).map(p => [p.key, p.value])
) )
if (Object.keys(output).length === 0) return if (Object.keys(output).length === 0) return
setSaving(s => ({ ...s, [row.extracted_value]: true })) setSaving(s => ({ ...s, [k]: true }))
try { try {
await api.createMapping({ await api.createMapping({
source_name: source, source_name: source,
@ -78,12 +93,12 @@ export default function Mappings({ source }) {
input_value: row.extracted_value, input_value: row.extracted_value,
output output
}) })
setUnmapped(u => u.filter(x => x.extracted_value !== row.extracted_value)) setUnmapped(u => u.filter(x => valueKey(x.extracted_value) !== k))
setDrafts(d => { const n = { ...d }; delete n[row.extracted_value]; return n }) setDrafts(d => { const n = { ...d }; delete n[k]; return n })
} catch (err) { } catch (err) {
alert(err.message) alert(err.message)
} finally { } finally {
setSaving(s => ({ ...s, [row.extracted_value]: false })) setSaving(s => ({ ...s, [k]: false }))
} }
} }
@ -177,19 +192,20 @@ export default function Mappings({ source }) {
: ( : (
<div className="space-y-2"> <div className="space-y-2">
{unmapped.map(row => { {unmapped.map(row => {
const k = valueKey(row.extracted_value)
const pairs = getDraft(row.extracted_value, row.output_field) const pairs = getDraft(row.extracted_value, row.output_field)
const isSaving = saving[row.extracted_value] const isSaving = saving[k]
const sampleKey = `${row.rule_name}:${row.extracted_value}` const sampleKey = `${row.rule_name}:${k}`
const samples = row.sample_records || [] const samples = row.sample_records || []
return ( return (
<div key={`${row.rule_name}:${row.extracted_value}`} <div key={`${row.rule_name}:${k}`}
className="bg-white border border-gray-200 rounded px-4 py-3"> className="bg-white border border-gray-200 rounded px-4 py-3">
<div className="flex items-start gap-3"> <div className="flex items-start gap-3">
{/* Left: value info */} {/* Left: value info */}
<div className="flex-1 min-w-0"> <div className="flex-1 min-w-0">
<div className="flex items-baseline gap-2"> <div className="flex items-baseline gap-2">
<span className="font-mono text-sm text-gray-800">{row.extracted_value}</span> <span className="font-mono text-sm text-gray-800">{displayValue(row.extracted_value)}</span>
<span className="text-xs text-gray-400">{row.record_count} records</span> <span className="text-xs text-gray-400">{row.record_count} records</span>
<span className="text-xs text-gray-300">· {row.rule_name}</span> <span className="text-xs text-gray-300">· {row.rule_name}</span>
</div> </div>
@ -277,7 +293,7 @@ export default function Mappings({ source }) {
editingId === m.id ? ( editingId === m.id ? (
<tr key={m.id} className="border-t border-gray-50 bg-blue-50"> <tr key={m.id} className="border-t border-gray-50 bg-blue-50">
<td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td> <td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td>
<td className="px-4 py-2 font-mono text-gray-700">{m.input_value}</td> <td className="px-4 py-2 font-mono text-gray-700">{displayValue(m.input_value)}</td>
<td className="px-4 py-2"> <td className="px-4 py-2">
<div className="space-y-1"> <div className="space-y-1">
{(editDrafts[m.id] || []).map((pair, i) => ( {(editDrafts[m.id] || []).map((pair, i) => (
@ -320,7 +336,7 @@ export default function Mappings({ source }) {
) : ( ) : (
<tr key={m.id} className="border-t border-gray-50 hover:bg-gray-50"> <tr key={m.id} className="border-t border-gray-50 hover:bg-gray-50">
<td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td> <td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td>
<td className="px-4 py-2 font-mono text-gray-700">{m.input_value}</td> <td className="px-4 py-2 font-mono text-gray-700">{displayValue(m.input_value)}</td>
<td className="px-4 py-2 font-mono text-xs text-gray-500"> <td className="px-4 py-2 font-mono text-xs text-gray-500">
{JSON.stringify(m.output)} {JSON.stringify(m.output)}
</td> </td>

181
ui/src/pages/Records.jsx
View File

@ -2,28 +2,24 @@ import { useState, useEffect } from 'react'
import { api } from '../api' import { api } from '../api'
export default function Records({ source }) { export default function Records({ source }) {
const [records, setRecords] = useState([]) const [rows, setRows] = useState([])
const [rules, setRules] = useState([]) const [exists, setExists] = useState(null)
const [mappings, setMappings] = useState([])
const [offset, setOffset] = useState(0) const [offset, setOffset] = useState(0)
const [view, setView] = useState('transformed') // 'raw' | 'transformed'
const [expanded, setExpanded] = useState(null)
const [loading, setLoading] = useState(false) const [loading, setLoading] = useState(false)
const LIMIT = 50 const LIMIT = 100
useEffect(() => { useEffect(() => {
if (!source) return if (!source) return
setOffset(0) setOffset(0)
load(0) load(0)
api.getRules(source).then(setRules).catch(() => {})
api.getMappings(source).then(setMappings).catch(() => {})
}, [source]) }, [source])
async function load(off) { async function load(off) {
setLoading(true) setLoading(true)
try { try {
const res = await api.getRecords(source, LIMIT, off) const res = await api.getViewData(source, LIMIT, off)
setRecords(res) setExists(res.exists)
setRows(res.rows)
} catch (err) { } catch (err) {
console.error(err) console.error(err)
} finally { } finally {
@ -36,145 +32,50 @@ export default function Records({ source }) {
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div> if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
const displayData = (record) => view === 'raw' ? record.data : (record.transformed || record.data)
// Build a lookup: rule_name + input_value → mapping output
const mappingLookup = {}
for (const m of mappings) {
mappingLookup[`${m.rule_name}::${m.input_value}`] = m.output
}
return ( return (
<div className="p-6"> <div className="p-6">
<div className="flex items-center justify-between mb-4"> <div className="flex items-center justify-between mb-4">
<h1 className="text-xl font-semibold text-gray-800">Records — {source}</h1> <h1 className="text-xl font-semibold text-gray-800">Records — {source}</h1>
<div className="flex bg-gray-100 rounded p-0.5"> {exists && rows.length > 0 && (
{['transformed', 'raw'].map(v => ( <span className="text-xs text-gray-400 font-mono">dfv.{source}</span>
<button key={v} onClick={() => setView(v)} )}
className={`text-sm px-3 py-1 rounded transition-colors ${
view === v ? 'bg-white text-gray-800 shadow-sm' : 'text-gray-500'
}`}>
{v}
</button>
))}
</div>
</div> </div>
{loading && <p className="text-sm text-gray-400">Loading…</p>} {loading && <p className="text-sm text-gray-400">Loading…</p>}
{!loading && records.length === 0 && ( {!loading && exists === false && (
<p className="text-sm text-gray-400">No records yet. Import a CSV file first.</p> <p className="text-sm text-gray-400">
No view generated yet. Go to <span className="font-medium text-gray-600">Sources</span>, check fields as <span className="font-medium text-gray-600">In view</span>, then click <span className="font-medium text-gray-600">Generate view</span>.
</p>
)} )}
{!loading && records.length > 0 && ( {!loading && exists && rows.length === 0 && (
<p className="text-sm text-gray-400">View exists but no transformed records yet. Import data and run a transform first.</p>
)}
{!loading && exists && rows.length > 0 && (
<> <>
<div className="bg-white border border-gray-200 rounded overflow-hidden mb-4"> <div className="bg-white border border-gray-200 rounded overflow-auto mb-4">
{(() => { <table className="w-full text-sm">
const sample = displayData(records[0]) || {} <thead>
const cols = Object.keys(sample).slice(0, 8) <tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
return ( {Object.keys(rows[0]).map(col => (
<table className="w-full text-sm"> <th key={col} className="px-3 py-2 font-medium whitespace-nowrap">{col}</th>
<thead> ))}
<tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50"> </tr>
{cols.map(c => ( </thead>
<th key={c} className="px-3 py-2 font-medium truncate max-w-32">{c}</th> <tbody>
))} {rows.map((row, i) => (
<th className="px-3 py-2 font-medium w-8"></th> <tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
</tr> {Object.values(row).map((val, j) => (
</thead> <td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
<tbody> {val === null ? <span className="text-gray-300">—</span> : String(val)}
{records.map(record => { </td>
const data = displayData(record) || {} ))}
const isExpanded = expanded === record.id </tr>
return ( ))}
<> </tbody>
<tr key={record.id} </table>
className="border-t border-gray-50 hover:bg-gray-50 cursor-pointer"
onClick={() => setExpanded(isExpanded ? null : record.id)}>
{cols.map(c => (
<td key={c} className="px-3 py-2 text-xs text-gray-600 truncate max-w-32">
{String(data[c] ?? '')}
</td>
))}
<td className="px-3 py-2 text-xs text-gray-300">
{isExpanded ? '▲' : '▼'}
</td>
</tr>
{isExpanded && (
<tr key={`${record.id}-expanded`} className="bg-gray-50 border-t border-gray-100">
<td colSpan={cols.length + 1} className="px-4 py-3 space-y-4">
{/* Transformations breakdown */}
{record.transformed && rules.length > 0 && (
<div>
<p className="text-xs font-medium text-gray-500 mb-1">Transformations</p>
<table className="w-full text-xs">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="pb-1 font-medium pr-4">Rule</th>
<th className="pb-1 font-medium pr-4">Input value</th>
<th className="pb-1 font-medium pr-4">Extracted</th>
<th className="pb-1 font-medium">Mapped output</th>
</tr>
</thead>
<tbody>
{rules.map(rule => {
const inputVal = record.data?.[rule.field]
const extractedVal = record.transformed?.[rule.output_field]
const mappedOutput = extractedVal != null
? mappingLookup[`${rule.name}::${extractedVal}`]
: undefined
return (
<tr key={rule.id} className="border-t border-gray-50">
<td className="py-1 pr-4 font-medium text-gray-700">{rule.name}</td>
<td className="py-1 pr-4 font-mono text-gray-500 max-w-48 truncate">
{inputVal ?? <span className="text-gray-300">—</span>}
</td>
<td className="py-1 pr-4 font-mono text-gray-700">
{extractedVal != null
? extractedVal
: <span className="text-gray-300">—</span>}
</td>
<td className="py-1 font-mono text-gray-500">
{mappedOutput
? Object.entries(mappedOutput).map(([k, v]) => (
<span key={k} className="inline-block mr-2">
<span className="text-gray-400">{k}:</span> {v}
</span>
))
: <span className="text-gray-300">—</span>}
</td>
</tr>
)
})}
</tbody>
</table>
</div>
)}
{/* Full data dump */}
<div>
<p className="text-xs font-medium text-gray-500 mb-1">
{view === 'transformed' ? 'Transformed data' : 'Raw data'}
</p>
<pre className="text-xs text-gray-600 whitespace-pre-wrap font-mono bg-white border border-gray-100 rounded p-2">
{JSON.stringify(displayData(record), null, 2)}
</pre>
{view === 'transformed' && !record.transformed && (
<p className="text-xs text-orange-400 mt-1">Not yet transformed</p>
)}
</div>
</td>
</tr>
)}
</>
)
})}
</tbody>
</table>
)
})()}
</div> </div>
<div className="flex items-center gap-3 text-sm text-gray-500"> <div className="flex items-center gap-3 text-sm text-gray-500">
@ -182,8 +83,8 @@ export default function Records({ source }) {
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40"> className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
← Prev ← Prev
</button> </button>
<span>{offset + 1}–{offset + records.length}</span> <span>{offset + 1}–{offset + rows.length}</span>
<button onClick={next} disabled={records.length < LIMIT} <button onClick={next} disabled={rows.length < LIMIT}
className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40"> className="px-3 py-1 border border-gray-200 rounded hover:bg-gray-50 disabled:opacity-40">
Next → Next →
</button> </button>

241
ui/src/pages/Rules.jsx
View File

@ -1,9 +1,71 @@
import { useState, useEffect } from 'react' import { useState, useEffect, useRef } from 'react'
import { api } from '../api' import { api } from '../api'
const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', sequence: 0 } const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', replace_value: '', sequence: 0 }
function PreviewModal({ rows, onClose }) {
const matched = rows.filter(r => r.extracted_value != null).length
return (
<div className="fixed inset-0 bg-black/40 flex items-center justify-center z-50" onClick={onClose}>
<div className="bg-white rounded-lg shadow-xl w-3/4 max-w-3xl max-h-[80vh] flex flex-col"
onClick={e => e.stopPropagation()}>
<div className="flex items-center justify-between px-5 py-3 border-b border-gray-100">
<span className="text-sm font-medium text-gray-700">
Pattern results — <span className="text-gray-500 font-normal">{matched}/{rows.length} matched</span>
</span>
<button onClick={onClose} className="text-gray-400 hover:text-gray-600 text-lg leading-none">✕</button>
</div>
<div className="overflow-auto flex-1 px-5 py-3">
<table className="w-full text-xs">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="pb-2 font-medium w-1/2 pr-4">Raw value</th>
<th className="pb-2 font-medium">Result</th>
</tr>
</thead>
<tbody>
{rows.map((r, i) => (
<tr key={i} className="border-t border-gray-50">
<td className="py-1 font-mono text-gray-400 pr-4 break-all">{r.raw_value}</td>
<td className={`py-1 font-mono break-all ${r.extracted_value != null ? 'text-gray-800' : 'text-gray-300'}`}>
{r.extracted_value != null
? (Array.isArray(r.extracted_value) ? r.extracted_value.join(' · ') : String(r.extracted_value))
: '—'}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
)
}
function FormPanel({ form, setForm, editing, error, loading, fields, source, onSubmit, onCancel }) {
const [preview, setPreview] = useState([])
const [previewing, setPreviewing] = useState(false)
const [modalOpen, setModalOpen] = useState(false)
const debounceRef = useRef(null)
// Auto-preview when field/pattern/flags change, debounced
useEffect(() => {
if (!form.field || !form.pattern) { setPreview([]); return }
clearTimeout(debounceRef.current)
debounceRef.current = setTimeout(async () => {
setPreviewing(true)
try {
const results = await api.previewRule(source, form.field, form.pattern, form.flags, form.function_type, form.replace_value, 50)
setPreview(results)
} catch {
setPreview([])
} finally {
setPreviewing(false)
}
}, 500)
return () => clearTimeout(debounceRef.current)
}, [form.field, form.pattern, form.flags, form.function_type, form.replace_value, source])
function FormPanel({ form, setForm, editing, error, loading, fields, onSubmit, onCancel }) {
return ( return (
<div className="bg-white border border-gray-200 rounded p-4 mb-4"> <div className="bg-white border border-gray-200 rounded p-4 mb-4">
<h2 className="text-sm font-semibold text-gray-700 mb-3">{editing ? 'Edit rule' : 'New rule'}</h2> <h2 className="text-sm font-semibold text-gray-700 mb-3">{editing ? 'Edit rule' : 'New rule'}</h2>
@ -82,6 +144,49 @@ function FormPanel({ form, setForm, editing, error, loading, fields, onSubmit, o
/> />
</div> </div>
</div> </div>
{form.function_type === 'replace' && (
<div>
<label className="text-xs text-gray-500 block mb-1">Replacement string</label>
<input
className="w-full border border-gray-200 rounded px-3 py-1.5 text-sm font-mono focus:outline-none focus:border-blue-400"
value={form.replace_value} onChange={e => setForm(f => ({ ...f, replace_value: e.target.value }))}
placeholder="e.g. leave blank to delete the match"
/>
</div>
)}
{/* Live preview */}
{(preview.length > 0 || previewing) && (
<div className="border border-gray-100 rounded p-2 bg-gray-50">
<div className="flex items-center justify-between mb-1">
<p className="text-xs text-gray-400">
{previewing ? 'Testing…' : `${preview.filter(r => r.extracted_value != null).length}/${preview.length} matched`}
</p>
{!previewing && preview.length > 0 && (
<button type="button" onClick={() => setModalOpen(true)}
className="text-xs text-blue-400 hover:text-blue-600">expand</button>
)}
</div>
{!previewing && (
<table className="w-full text-xs">
<tbody>
{preview.slice(0, 5).map((r, i) => (
<tr key={i} className="border-t border-gray-100 first:border-0">
<td className="py-0.5 font-mono text-gray-400 truncate max-w-0 w-1/2 pr-3">{r.raw_value}</td>
<td className={`py-0.5 font-mono truncate ${r.extracted_value != null ? 'text-gray-800' : 'text-gray-300'}`}>
{r.extracted_value != null
? (Array.isArray(r.extracted_value) ? r.extracted_value.join(' · ') : String(r.extracted_value))
: '—'}
</td>
</tr>
))}
</tbody>
</table>
)}
</div>
)}
{modalOpen && <PreviewModal rows={preview} onClose={() => setModalOpen(false)} />}
{error && <p className="text-xs text-red-500">{error}</p>} {error && <p className="text-xs text-red-500">{error}</p>}
<div className="flex gap-2"> <div className="flex gap-2">
<button type="submit" disabled={loading} <button type="submit" disabled={loading}
@ -102,6 +207,7 @@ export default function Rules({ source }) {
const [rules, setRules] = useState([]) const [rules, setRules] = useState([])
const [creating, setCreating] = useState(false) const [creating, setCreating] = useState(false)
const [editing, setEditing] = useState(null) const [editing, setEditing] = useState(null)
const [expanded, setExpanded] = useState(null)
const [form, setForm] = useState(EMPTY_FORM) const [form, setForm] = useState(EMPTY_FORM)
const [testResults, setTestResults] = useState({}) const [testResults, setTestResults] = useState({})
const [fields, setFields] = useState([]) const [fields, setFields] = useState([])
@ -130,6 +236,7 @@ export default function Rules({ source }) {
output_field: rule.output_field, output_field: rule.output_field,
function_type: rule.function_type || 'extract', function_type: rule.function_type || 'extract',
flags: rule.flags || '', flags: rule.flags || '',
replace_value: rule.replace_value || '',
sequence: rule.sequence, sequence: rule.sequence,
}) })
setEditing(rule.id) setEditing(rule.id)
@ -137,13 +244,14 @@ export default function Rules({ source }) {
setError('') setError('')
} }
async function handleSubmit(e) { async function handleSubmit(e, ruleId = null) {
e.preventDefault() e.preventDefault()
setError('') setError('')
setLoading(true) setLoading(true)
const id = ruleId ?? editing
try { try {
if (editing) { if (id) {
await api.updateRule(editing, { ...form, source_name: source }) await api.updateRule(id, { ...form, source_name: source })
} else { } else {
await api.createRule({ ...form, source_name: source }) await api.createRule({ ...form, source_name: source })
} }
@ -151,6 +259,7 @@ export default function Rules({ source }) {
setRules(updated) setRules(updated)
setCreating(false) setCreating(false)
setEditing(null) setEditing(null)
setExpanded(null)
} catch (err) { } catch (err) {
setError(err.message) setError(err.message)
} finally { } finally {
@ -202,7 +311,7 @@ export default function Rules({ source }) {
{creating && ( {creating && (
<FormPanel <FormPanel
form={form} setForm={setForm} editing={false} form={form} setForm={setForm} editing={false}
error={error} loading={loading} fields={fields} error={error} loading={loading} fields={fields} source={source}
onSubmit={handleSubmit} onSubmit={handleSubmit}
onCancel={() => { setCreating(false); setError('') }} onCancel={() => { setCreating(false); setError('') }}
/> />
@ -212,74 +321,62 @@ export default function Rules({ source }) {
<p className="text-sm text-gray-400">No rules yet. Add a regex rule to start extracting values.</p> <p className="text-sm text-gray-400">No rules yet. Add a regex rule to start extracting values.</p>
)} )}
<div className="space-y-3"> <div className="space-y-2">
{rules.map(rule => ( {rules.map(rule => {
<div key={rule.id} className="bg-white border border-gray-200 rounded"> const isExpanded = expanded === rule.id
<div className="flex items-center gap-3 px-4 py-3"> return (
<button <div key={rule.id} className="bg-white border border-gray-200 rounded">
onClick={() => handleToggle(rule)} {/* Header — always visible, click to expand/collapse */}
className={`w-8 h-4 rounded-full transition-colors ${rule.enabled ? 'bg-blue-500' : 'bg-gray-200'}`} <div
title={rule.enabled ? 'Disable' : 'Enable'} className="flex items-center gap-3 px-4 py-3 cursor-pointer hover:bg-gray-50 select-none"
/> onClick={() => {
<div className="flex-1 min-w-0"> if (isExpanded) { setExpanded(null); setEditing(null) }
<span className="font-medium text-gray-800 text-sm">{rule.name}</span> else { setExpanded(rule.id); startEdit(rule) }
<span className="text-gray-400 text-xs ml-2">seq {rule.sequence}</span> }}
<div className="text-xs text-gray-400 mt-0.5"> >
<span className="font-mono">{rule.field}</span> <button
<span className="mx-1">→</span> onClick={e => { e.stopPropagation(); handleToggle(rule) }}
<span className="font-mono bg-gray-50 px-1 rounded">{rule.pattern}</span> className={`w-8 h-4 rounded-full flex-shrink-0 transition-colors ${rule.enabled ? 'bg-blue-500' : 'bg-gray-200'}`}
{rule.flags && <span className="text-blue-400 ml-1">/{rule.flags}</span>} title={rule.enabled ? 'Disable' : 'Enable'}
<span className="mx-1">→</span>
<span className="font-mono">{rule.output_field}</span>
{rule.function_type === 'replace' && <span className="ml-1 text-orange-400">(replace)</span>}
</div>
</div>
<div className="flex gap-2">
<button onClick={() => handleTest(rule.id)}
className="text-xs text-blue-500 hover:text-blue-700">Test</button>
<button onClick={() => editing === rule.id ? setEditing(null) : startEdit(rule)}
className="text-xs text-gray-400 hover:text-gray-600">Edit</button>
<button onClick={() => handleDelete(rule.id)}
className="text-xs text-red-400 hover:text-red-600">Delete</button>
</div>
</div>
{editing === rule.id && (
<div className="px-4 pb-4">
<FormPanel
form={form} setForm={setForm} editing={true}
error={error} loading={loading} fields={fields}
onSubmit={handleSubmit}
onCancel={() => setEditing(null)}
/> />
<div className="flex-1 min-w-0">
<span className="font-medium text-gray-800 text-sm">{rule.name}</span>
<span className="text-gray-400 text-xs ml-2">seq {rule.sequence}</span>
{!isExpanded && (
<div className="text-xs text-gray-400 mt-0.5 truncate">
<span className="font-mono">{rule.field}</span>
<span className="mx-1">→</span>
<span className="font-mono bg-gray-50 px-1 rounded">{rule.pattern}</span>
{rule.flags && <span className="text-blue-400 ml-1">/{rule.flags}</span>}
<span className="mx-1">→</span>
<span className="font-mono">{rule.output_field}</span>
{rule.function_type === 'replace' && <span className="ml-1 text-orange-400">(replace)</span>}
</div>
)}
</div>
<span className="text-xs text-gray-300 flex-shrink-0">{isExpanded ? '▲' : '▼'}</span>
</div> </div>
)}
{testResults[rule.id] && ( {/* Expanded content */}
<div className="border-t border-gray-100 px-4 py-3"> {isExpanded && (
<p className="text-xs text-gray-500 mb-2">Test results (last 20 records)</p> <div className="border-t border-gray-100">
<table className="w-full text-xs"> <div className="px-4 pt-3 pb-1 flex justify-end">
<thead> <button onClick={e => { e.stopPropagation(); handleDelete(rule.id) }}
<tr className="text-left text-gray-400"> className="text-xs text-red-400 hover:text-red-600">Delete</button>
<th className="pb-1 font-medium w-1/2">Raw value</th> </div>
<th className="pb-1 font-medium">Extracted</th> <div className="px-4 pb-4">
</tr> <FormPanel
</thead> form={form} setForm={setForm} editing={true}
<tbody> error={error} loading={loading} fields={fields} source={source}
{testResults[rule.id].slice(0, 10).map((r, i) => ( onSubmit={e => handleSubmit(e, rule.id)}
<tr key={i} className="border-t border-gray-50"> onCancel={() => { setEditing(null); setExpanded(null) }}
<td className="py-1 font-mono text-gray-500 truncate max-w-0 w-1/2 pr-2">{r.raw_value}</td> />
<td className={`py-1 font-mono ${r.extracted_value ? 'text-gray-800' : 'text-gray-300'}`}> </div>
{r.extracted_value ?? '—'} </div>
</td> )}
</tr> </div>
))} )
</tbody> })}
</table>
</div>
)}
</div>
))}
</div> </div>
</div> </div>
) )

448
ui/src/pages/Sources.jsx
View File

@ -3,22 +3,37 @@ import { api } from '../api'
const FIELD_TYPES = ['text', 'numeric', 'date'] const FIELD_TYPES = ['text', 'numeric', 'date']
function SourceDetail({ source, onClose, onDeleted, setSources, setSource }) { export default function Sources({ source, sources, setSources, setSource }) {
const [dedup, setDedup] = useState(source.dedup_fields?.join(', ') || '') const [dedup, setDedup] = useState('')
const [schemaFields, setSchemaFields] = useState(source.config?.fields || []) const [schemaFields, setSchemaFields] = useState([])
const [stats, setStats] = useState(null) const [stats, setStats] = useState(null)
const [saving, setSaving] = useState(false) const [saving, setSaving] = useState(false)
const [reprocessing, setReprocessing] = useState(false) const [reprocessing, setReprocessing] = useState(false)
const [generating, setGenerating] = useState(false) const [generating, setGenerating] = useState(false)
const [result, setResult] = useState('') const [result, setResult] = useState('')
const [error, setError] = useState('') const [error, setError] = useState('')
const [viewName, setViewName] = useState(source.config?.fields?.length ? `dfv.${source.name}` : '') const [viewName, setViewName] = useState('')
const [availableFields, setAvailableFields] = useState([]) const [availableFields, setAvailableFields] = useState([])
const [creating, setCreating] = useState(false)
const [form, setForm] = useState({ name: '', dedup_fields: '', fields: [], schema: [] })
const [createError, setCreateError] = useState('')
const [createLoading, setCreateLoading] = useState(false)
const fileRef = useRef()
const sourceObj = sources.find(s => s.name === source)
useEffect(() => { useEffect(() => {
api.getStats(source.name).then(setStats).catch(() => {}) if (!sourceObj) return
api.getFields(source.name).then(setAvailableFields).catch(() => {}) setDedup(sourceObj.dedup_fields?.join(', ') || '')
}, [source.name]) setSchemaFields((sourceObj.config?.fields || []).map((f, i) => ({ seq: i + 1, ...f })))
setViewName(sourceObj.config?.fields?.length ? `dfv.${sourceObj.name}` : '')
setResult('')
setError('')
setStats(null)
setAvailableFields([])
api.getStats(sourceObj.name).then(setStats).catch(() => {})
api.getFields(sourceObj.name).then(setAvailableFields).catch(() => {})
}, [source, sourceObj?.name])
async function handleSave(e) { async function handleSave(e) {
e.preventDefault() e.preventDefault()
@ -26,8 +41,9 @@ function SourceDetail({ source, onClose, onDeleted, setSources, setSource }) {
setError('') setError('')
try { try {
const dedup_fields = dedup.split(',').map(s => s.trim()).filter(Boolean) const dedup_fields = dedup.split(',').map(s => s.trim()).filter(Boolean)
const config = { ...(source.config || {}), fields: schemaFields.filter(f => f.name) } const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
await api.updateSource(source.name, { dedup_fields, config }) const config = { ...(sourceObj.config || {}), fields }
await api.updateSource(sourceObj.name, { dedup_fields, config })
const updated = await api.getSources() const updated = await api.getSources()
setSources(updated) setSources(updated)
setResult('Saved.') setResult('Saved.')
@ -43,11 +59,11 @@ function SourceDetail({ source, onClose, onDeleted, setSources, setSource }) {
setResult('') setResult('')
setError('') setError('')
try { try {
// Save schema first, then generate view from the saved config
const dedup_fields = dedup.split(',').map(s => s.trim()).filter(Boolean) const dedup_fields = dedup.split(',').map(s => s.trim()).filter(Boolean)
const config = { ...(source.config || {}), fields: schemaFields.filter(f => f.name) } const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
await api.updateSource(source.name, { dedup_fields, config }) const config = { ...(sourceObj.config || {}), fields }
const res = await api.generateView(source.name) await api.updateSource(sourceObj.name, { dedup_fields, config })
const res = await api.generateView(sourceObj.name)
if (res.success) { if (res.success) {
setViewName(res.view) setViewName(res.view)
setResult(`View created: ${res.view}`) setResult(`View created: ${res.view}`)
@ -62,14 +78,14 @@ function SourceDetail({ source, onClose, onDeleted, setSources, setSource }) {
} }
async function handleReprocess() { async function handleReprocess() {
if (!confirm(`Reprocess all records for "${source.name}"? This will clear and reapply all transformations.`)) return if (!confirm(`Reprocess all records for "${sourceObj.name}"? This will clear and reapply all transformations.`)) return
setReprocessing(true) setReprocessing(true)
setResult('') setResult('')
setError('') setError('')
try { try {
const res = await api.reprocess(source.name) const res = await api.reprocess(sourceObj.name)
setResult(`Reprocessed ${res.transformed} records.`) setResult(`Reprocessed ${res.transformed} records.`)
api.getStats(source.name).then(setStats).catch(() => {}) api.getStats(sourceObj.name).then(setStats).catch(() => {})
} catch (err) { } catch (err) {
setError(err.message) setError(err.message)
} finally { } finally {
@ -77,157 +93,18 @@ function SourceDetail({ source, onClose, onDeleted, setSources, setSource }) {
} }
} }
return ( async function handleDelete() {
<div className="mt-1 bg-white border border-gray-200 rounded p-4 space-y-4"> if (!confirm(`Delete source "${sourceObj.name}" and all its data?`)) return
{/* Stats */} try {
{stats && ( await api.deleteSource(sourceObj.name)
<div className="flex gap-4 text-xs"> const updated = await api.getSources()
<span className="text-gray-500"><span className="font-medium text-gray-800">{stats.total_records}</span> total</span> setSources(updated)
<span className="text-gray-500"><span className="font-medium text-gray-800">{stats.transformed_records}</span> transformed</span> if (updated.length > 0) setSource(updated[0].name)
<span className="text-gray-500"><span className="font-medium text-gray-800">{stats.pending_records}</span> pending</span> else setSource('')
</div> } catch (err) {
)} alert(err.message)
}
{/* Unified field table */} }
{availableFields.length > 0 && (
<div className="pt-2 border-t border-gray-100 space-y-2">
<table className="w-full text-xs">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="pb-1 font-medium">Key</th>
<th className="pb-1 font-medium">Origin</th>
<th className="pb-1 font-medium">Type</th>
<th className="pb-1 font-medium text-center">Dedup</th>
<th className="pb-1 font-medium text-center">In view</th>
</tr>
</thead>
<tbody>
{availableFields.map(f => {
const isRaw = f.origins.includes('raw')
const dedupChecked = dedup.split(',').map(s => s.trim()).includes(f.key)
const schemaEntry = schemaFields.find(sf => sf.name === f.key)
const inView = !!schemaEntry
return (
<tr key={f.key} className="border-t border-gray-50">
<td className="py-1 font-mono text-gray-700">{f.key}</td>
<td className="py-1 text-gray-400">{f.origins.join(', ')}</td>
<td className="py-1">
{inView && (
<select
className="border border-gray-200 rounded px-1 py-0.5 text-xs focus:outline-none focus:border-blue-400"
value={schemaEntry.type}
onChange={e => setSchemaFields(sf =>
sf.map(s => s.name === f.key ? { ...s, type: e.target.value } : s)
)}
>
{FIELD_TYPES.map(t => <option key={t} value={t}>{t}</option>)}
</select>
)}
</td>
<td className="py-1 text-center">
{isRaw && (
<input
type="checkbox"
checked={dedupChecked}
onChange={e => {
const current = dedup.split(',').map(s => s.trim()).filter(Boolean)
const next = e.target.checked
? [...current, f.key]
: current.filter(k => k !== f.key)
setDedup(next.join(', '))
}}
/>
)}
</td>
<td className="py-1 text-center">
<input
type="checkbox"
checked={inView}
onChange={e => {
if (e.target.checked) {
setSchemaFields(sf => [...sf, { name: f.key, type: 'text' }])
} else {
setSchemaFields(sf => sf.filter(s => s.name !== f.key))
}
}}
/>
</td>
</tr>
)
})}
</tbody>
</table>
<div className="flex items-center gap-3 pt-1">
<form onSubmit={handleSave}>
<button type="submit" disabled={saving}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
{saving ? 'Saving…' : 'Save'}
</button>
</form>
{schemaFields.length > 0 && (
<>
<button
onClick={handleGenerateView}
disabled={generating}
className="text-xs bg-green-600 text-white px-2 py-1.5 rounded hover:bg-green-700 disabled:opacity-50"
>
{generating ? 'Generating…' : 'Generate view'}
</button>
{viewName && (
<code className="text-xs bg-gray-100 px-2 py-1 rounded text-gray-600">{viewName}</code>
)}
</>
)}
</div>
</div>
)}
{/* Save button when no fields loaded yet */}
{availableFields.length === 0 && (
<form onSubmit={handleSave}>
<button type="submit" disabled={saving}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
{saving ? 'Saving…' : 'Save'}
</button>
</form>
)}
{/* Reprocess */}
<div className="flex items-center gap-3 pt-2 border-t border-gray-100">
<button
onClick={handleReprocess}
disabled={reprocessing}
className="text-sm bg-orange-500 text-white px-3 py-1.5 rounded hover:bg-orange-600 disabled:opacity-50"
>
{reprocessing ? 'Reprocessing…' : 'Reprocess all records'}
</button>
<span className="text-xs text-gray-400">Clears and reruns all transformation rules</span>
</div>
{result && <p className="text-xs text-green-600">{result}</p>}
{error && <p className="text-xs text-red-500">{error}</p>}
<div className="pt-1 border-t border-gray-100 flex justify-between">
<button onClick={onClose} className="text-xs text-gray-400 hover:text-gray-600">Close</button>
<button
onClick={() => onDeleted(source.name)}
className="text-xs text-red-400 hover:text-red-600"
>
Delete source
</button>
</div>
</div>
)
}
export default function Sources({ sources, setSources, setSource }) {
const [creating, setCreating] = useState(false)
const [expanded, setExpanded] = useState(null)
const [form, setForm] = useState({ name: '', dedup_fields: '', fields: [], schema: [] })
const [error, setError] = useState('')
const [loading, setLoading] = useState(false)
const fileRef = useRef()
async function handleSuggest(e) { async function handleSuggest(e) {
const file = e.target.files[0] const file = e.target.files[0]
@ -241,93 +118,222 @@ export default function Sources({ sources, setSources, setSource }) {
schema: suggestion.fields.map(f => ({ name: f.name, type: f.type })) schema: suggestion.fields.map(f => ({ name: f.name, type: f.type }))
})) }))
} catch (err) { } catch (err) {
setError(err.message) setCreateError(err.message)
} }
} }
async function handleCreate(e) { async function handleCreate(e) {
e.preventDefault() e.preventDefault()
setError('') setCreateError('')
const dedup = form.dedup_fields.split(',').map(s => s.trim()).filter(Boolean) const dedupArr = form.dedup_fields.split(',').map(s => s.trim()).filter(Boolean)
if (!form.name || dedup.length === 0) { if (!form.name || dedupArr.length === 0) {
setError('Name and at least one dedup field required') setCreateError('Name and at least one dedup field required')
return return
} }
setLoading(true) setCreateLoading(true)
try { try {
const config = form.schema.length > 0 ? { fields: form.schema } : {} const config = form.schema.length > 0 ? { fields: form.schema } : {}
await api.createSource({ name: form.name, dedup_fields: dedup, config }) await api.createSource({ name: form.name, dedup_fields: dedupArr, config })
const updated = await api.getSources() const updated = await api.getSources()
setSources(updated) setSources(updated)
setSource(form.name) setSource(form.name)
setForm({ name: '', dedup_fields: '', fields: [], schema: [] }) setForm({ name: '', dedup_fields: '', fields: [], schema: [] })
setCreating(false) setCreating(false)
} catch (err) { } catch (err) {
setError(err.message) setCreateError(err.message)
} finally { } finally {
setLoading(false) setCreateLoading(false)
}
}
async function handleDeleted(name) {
if (!confirm(`Delete source "${name}" and all its data?`)) return
try {
await api.deleteSource(name)
const updated = await api.getSources()
setSources(updated)
setExpanded(null)
if (updated.length > 0) setSource(updated[0].name)
} catch (err) {
alert(err.message)
} }
} }
return ( return (
<div className="p-6 max-w-2xl"> <div className="p-6 max-w-2xl">
<div className="flex items-center justify-between mb-6"> <div className="flex items-center justify-between mb-6">
<h1 className="text-xl font-semibold text-gray-800">Sources</h1> <h1 className="text-xl font-semibold text-gray-800">
{sourceObj ? sourceObj.name : 'Sources'}
</h1>
<button <button
onClick={() => { setCreating(true); setError(''); setExpanded(null) }} onClick={() => { setCreating(true); setCreateError('') }}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700" className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700"
> >
New source New source
</button> </button>
</div> </div>
{sources.length === 0 && !creating && ( {/* No source selected */}
<p className="text-gray-500 text-sm">No sources yet. Create one to get started.</p> {!sourceObj && !creating && (
<p className="text-sm text-gray-400">No sources yet. Create one to get started.</p>
)} )}
<div className="space-y-2"> {/* Source detail */}
{sources.map(s => ( {sourceObj && !creating && (
<div key={s.name}> <div className="space-y-4">
<div {/* Stats */}
className="flex items-center justify-between bg-white border border-gray-200 rounded px-4 py-3 cursor-pointer hover:bg-gray-50" {stats && (
onClick={() => setExpanded(expanded === s.name ? null : s.name)} <div className="flex gap-4 text-xs">
> <span className="text-gray-500"><span className="font-medium text-gray-800">{stats.total_records}</span> total</span>
<div> <span className="text-gray-500"><span className="font-medium text-gray-800">{stats.transformed_records}</span> transformed</span>
<span className="font-medium text-gray-800">{s.name}</span> <span className="text-gray-500"><span className="font-medium text-gray-800">{stats.pending_records}</span> pending</span>
<span className="ml-3 text-xs text-gray-400">dedup: {s.dedup_fields?.join(', ')}</span>
</div>
<span className="text-xs text-gray-300">{expanded === s.name ? '▲' : '▼'}</span>
</div> </div>
)}
{expanded === s.name && ( {/* Unified field table */}
<SourceDetail {availableFields.length > 0 && (
source={s} <div className="pt-2 border-t border-gray-100 space-y-2">
onClose={() => setExpanded(null)} <table className="w-full text-xs">
onDeleted={handleDeleted} <thead>
setSources={setSources} <tr className="text-left text-gray-400 border-b border-gray-100">
setSource={setSource} <th className="pb-1 font-medium">Key</th>
/> <th className="pb-1 font-medium">Origin</th>
)} <th className="pb-1 font-medium">Type</th>
<th className="pb-1 font-medium text-center">Dedup</th>
<th className="pb-1 font-medium text-center">In view</th>
<th className="pb-1 font-medium text-center">Seq</th>
</tr>
</thead>
<tbody>
{availableFields.map(f => {
const isRaw = f.origins.includes('raw')
const dedupChecked = dedup.split(',').map(s => s.trim()).includes(f.key)
const schemaEntry = schemaFields.find(sf => sf.name === f.key)
const inView = !!schemaEntry
return (
<tr key={f.key} className="border-t border-gray-50">
<td className="py-1 font-mono text-gray-700">{f.key}</td>
<td className="py-1 text-gray-400">{f.origins.join(', ')}</td>
<td className="py-1">
{inView && (
<div className="flex gap-1 items-center">
<select
className="border border-gray-200 rounded px-1 py-0.5 text-xs focus:outline-none focus:border-blue-400"
value={schemaEntry.type}
onChange={e => setSchemaFields(sf =>
sf.map(s => s.name === f.key ? { ...s, type: e.target.value } : s)
)}
>
{FIELD_TYPES.map(t => <option key={t} value={t}>{t}</option>)}
</select>
<input
className="border border-gray-200 rounded px-1 py-0.5 text-xs font-mono w-32 focus:outline-none focus:border-blue-400"
value={schemaEntry.expression || ''}
placeholder="{field} * {sign}"
onChange={e => setSchemaFields(sf =>
sf.map(s => s.name === f.key ? { ...s, expression: e.target.value || undefined } : s)
)}
/>
</div>
)}
</td>
<td className="py-1 text-center">
{isRaw && (
<input
type="checkbox"
checked={dedupChecked}
onChange={e => {
const current = dedup.split(',').map(s => s.trim()).filter(Boolean)
const next = e.target.checked
? [...current, f.key]
: current.filter(k => k !== f.key)
setDedup(next.join(', '))
}}
/>
)}
</td>
<td className="py-1 text-center">
<input
type="checkbox"
checked={inView}
onChange={e => {
if (e.target.checked) {
const nextSeq = schemaFields.length > 0
? Math.max(...schemaFields.map(s => s.seq ?? 0)) + 1
: 1
setSchemaFields(sf => [...sf, { name: f.key, type: 'text', seq: nextSeq }])
} else {
setSchemaFields(sf => sf.filter(s => s.name !== f.key))
}
}}
/>
</td>
<td className="py-1 text-center">
{inView && (
<input
type="number"
className="w-12 border border-gray-200 rounded px-1 py-0.5 text-xs text-center focus:outline-none focus:border-blue-400"
value={schemaEntry.seq ?? ''}
onChange={e => setSchemaFields(sf =>
sf.map(s => s.name === f.key ? { ...s, seq: parseInt(e.target.value) || 0 } : s)
)}
/>
)}
</td>
</tr>
)
})}
</tbody>
</table>
<div className="flex items-center gap-3 pt-1">
<form onSubmit={handleSave}>
<button type="submit" disabled={saving}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
{saving ? 'Saving…' : 'Save'}
</button>
</form>
{schemaFields.length > 0 && (
<>
<button
onClick={handleGenerateView}
disabled={generating}
className="text-xs bg-green-600 text-white px-2 py-1.5 rounded hover:bg-green-700 disabled:opacity-50"
>
{generating ? 'Generating…' : 'Generate view'}
</button>
{viewName && (
<code className="text-xs bg-gray-100 px-2 py-1 rounded text-gray-600">{viewName}</code>
)}
</>
)}
</div>
</div>
)}
{/* Save button when no fields loaded yet */}
{availableFields.length === 0 && (
<form onSubmit={handleSave}>
<button type="submit" disabled={saving}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
{saving ? 'Saving…' : 'Save'}
</button>
</form>
)}
{/* Reprocess */}
<div className="flex items-center gap-3 pt-2 border-t border-gray-100">
<button
onClick={handleReprocess}
disabled={reprocessing}
className="text-sm bg-orange-500 text-white px-3 py-1.5 rounded hover:bg-orange-600 disabled:opacity-50"
>
{reprocessing ? 'Reprocessing…' : 'Reprocess all records'}
</button>
<span className="text-xs text-gray-400">Clears and reruns all transformation rules</span>
</div> </div>
))}
</div> {result && <p className="text-xs text-green-600">{result}</p>}
{error && <p className="text-xs text-red-500">{error}</p>}
<div className="pt-2 border-t border-gray-100">
<button onClick={handleDelete} className="text-xs text-red-400 hover:text-red-600">
Delete source
</button>
</div>
</div>
)}
{/* Create form */} {/* Create form */}
{creating && ( {creating && (
<div className="mt-6 bg-white border border-gray-200 rounded p-4"> <div className="bg-white border border-gray-200 rounded p-4">
<h2 className="text-sm font-semibold text-gray-700 mb-3">New source</h2> <h2 className="text-sm font-semibold text-gray-700 mb-3">New source</h2>
<div className="mb-4"> <div className="mb-4">
@ -394,15 +400,15 @@ export default function Sources({ sources, setSources, setSource }) {
</div> </div>
)} )}
{error && <p className="text-xs text-red-500">{error}</p>} {createError && <p className="text-xs text-red-500">{createError}</p>}
<div className="flex gap-2"> <div className="flex gap-2">
<button type="submit" disabled={loading} <button type="submit" disabled={createLoading}
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50"> className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
{loading ? 'Creating…' : 'Create'} {createLoading ? 'Creating…' : 'Create'}
</button> </button>
<button type="button" <button type="button"
onClick={() => { setCreating(false); setError(''); setForm({ name: '', dedup_fields: '', fields: [], schema: [] }) }} onClick={() => { setCreating(false); setCreateError(''); setForm({ name: '', dedup_fields: '', fields: [], schema: [] }) }}
className="text-sm text-gray-500 px-3 py-1.5 rounded hover:bg-gray-100"> className="text-sm text-gray-500 px-3 py-1.5 rounded hover:bg-gray-100">
Cancel Cancel
</button> </button>