Records filters, global picklist, autocomplete, and rule reprocess
- Records tab: regex filter bar (postgres ~*), add/remove filters, debounced, ANDed together; get_view_data gains p_filters JSONB param - Global picklist: sources.global_picklist flag (default true) controls whether a source's mapped output values feed the cross-source autocomplete suggestion pool; toggle on Sources page; get_global_output_values() SQL function - Mappings: replace native datalist with custom AutocompleteInput component — Alt+Down opens, Tab cycles, Enter selects, arrow keys navigate, Escape closes - Rules: auto-reprocess source records when a rule is created or updated - preview_rule: fix BIGINT/INT return type mismatch - Stale get_import_log removed from sources.sql - TSV export: fetch with auth headers instead of plain <a href> (fixes 401) - + column button: more visible styling Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d63d70cd52
commit
d495ef2fc5
@ -39,6 +39,21 @@ module.exports = (pool) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Get global output values (for autocomplete across all global_picklist=true sources)
|
||||
router.get('/global-values', async (req, res, next) => {
|
||||
try {
|
||||
const result = await pool.query(`SELECT * FROM get_global_output_values()`);
|
||||
const map = {};
|
||||
for (const { col, val } of result.rows) {
|
||||
if (!map[col]) map[col] = [];
|
||||
map[col].push(val);
|
||||
}
|
||||
res.json(map);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Get unmapped values
|
||||
router.get('/source/:source_name/unmapped', async (req, res, next) => {
|
||||
try {
|
||||
|
||||
@ -73,7 +73,9 @@ module.exports = (pool) => {
|
||||
const result = await pool.query(
|
||||
`SELECT * FROM create_rule(${lit(source_name)}, ${lit(name)}, ${lit(field)}, ${lit(pattern)}, ${lit(output_field)}, ${lit(function_type || 'extract')}, ${lit(flags || '')}, ${lit(replace_value || '')}, ${lit(enabled !== false)}, ${lit(retain === true)}, ${lit(sequence || 0)})`
|
||||
);
|
||||
res.status(201).json(result.rows[0]);
|
||||
const rule = result.rows[0];
|
||||
await pool.query(`SELECT reprocess_records(${lit(source_name)})`);
|
||||
res.status(201).json(rule);
|
||||
} catch (err) {
|
||||
if (err.code === '23505') return res.status(409).json({ error: 'Rule already exists for this source' });
|
||||
if (err.code === '23503') return res.status(404).json({ error: 'Source not found' });
|
||||
@ -93,7 +95,9 @@ module.exports = (pool) => {
|
||||
`SELECT * FROM update_rule(${lit(parseInt(req.params.id))}, ${n(name)}, ${n(field)}, ${n(pattern)}, ${n(output_field)}, ${n(function_type)}, ${n(flags)}, ${n(replace_value)}, ${n(enabled)}, ${n(retain)}, ${n(sequence)})`
|
||||
);
|
||||
if (result.rows.length === 0) return res.status(404).json({ error: 'Rule not found' });
|
||||
res.json(result.rows[0]);
|
||||
const rule = result.rows[0];
|
||||
await pool.query(`SELECT reprocess_records(${lit(rule.source_name)})`);
|
||||
res.json(rule);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
|
||||
@ -73,12 +73,12 @@ module.exports = (pool) => {
|
||||
// Create source
|
||||
router.post('/', async (req, res, next) => {
|
||||
try {
|
||||
const { name, constraint_fields, config } = req.body;
|
||||
const { name, constraint_fields, config, global_picklist } = req.body;
|
||||
if (!name || !constraint_fields || !Array.isArray(constraint_fields)) {
|
||||
return res.status(400).json({ error: 'Missing required fields: name, constraint_fields (array)' });
|
||||
}
|
||||
const result = await pool.query(
|
||||
`SELECT * FROM create_source(${lit(name)}, ${arr(constraint_fields)}, ${lit(config || {})})`
|
||||
`SELECT * FROM create_source(${lit(name)}, ${arr(constraint_fields)}, ${lit(config || {})}, ${lit(global_picklist !== false)})`
|
||||
);
|
||||
res.status(201).json(result.rows[0]);
|
||||
} catch (err) {
|
||||
@ -90,9 +90,10 @@ module.exports = (pool) => {
|
||||
// Update source
|
||||
router.put('/:name', async (req, res, next) => {
|
||||
try {
|
||||
const { constraint_fields, config } = req.body;
|
||||
const { constraint_fields, config, global_picklist } = req.body;
|
||||
const gpVal = global_picklist !== undefined ? lit(global_picklist) : 'NULL';
|
||||
const result = await pool.query(
|
||||
`SELECT * FROM update_source(${lit(req.params.name)}, ${constraint_fields ? arr(constraint_fields) : 'NULL'}, ${config ? lit(config) : 'NULL'})`
|
||||
`SELECT * FROM update_source(${lit(req.params.name)}, ${constraint_fields ? arr(constraint_fields) : 'NULL'}, ${config ? lit(config) : 'NULL'}, ${gpVal})`
|
||||
);
|
||||
if (result.rows.length === 0) return res.status(404).json({ error: 'Source not found' });
|
||||
res.json(result.rows[0]);
|
||||
@ -212,9 +213,13 @@ module.exports = (pool) => {
|
||||
// Get view data (paginated, sortable)
|
||||
router.get('/:name/view-data', async (req, res, next) => {
|
||||
try {
|
||||
const { limit = 100, offset = 0, sort_col, sort_dir } = req.query;
|
||||
const { limit = 100, offset = 0, sort_col, sort_dir, filters } = req.query;
|
||||
let parsedFilters = null;
|
||||
if (filters) {
|
||||
try { parsedFilters = JSON.parse(filters); } catch { /* ignore bad JSON */ }
|
||||
}
|
||||
const result = await pool.query(
|
||||
`SELECT get_view_data(${lit(req.params.name)}, ${lit(parseInt(limit))}, ${lit(parseInt(offset))}, ${lit(sort_col || null)}, ${lit(sort_dir || 'asc')}) as result`
|
||||
`SELECT get_view_data(${lit(req.params.name)}, ${lit(parseInt(limit))}, ${lit(parseInt(offset))}, ${lit(sort_col || null)}, ${lit(sort_dir || 'asc')}, ${parsedFilters ? lit(parsedFilters) : 'NULL'}) as result`
|
||||
);
|
||||
res.json(result.rows[0].result);
|
||||
} catch (err) {
|
||||
|
||||
@ -206,3 +206,18 @@ BEGIN
|
||||
ORDER BY count(*) DESC;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
-- ── Global picklist ───────────────────────────────────────────────────────────
|
||||
|
||||
CREATE OR REPLACE FUNCTION get_global_output_values()
|
||||
RETURNS TABLE (col TEXT, val TEXT) AS $$
|
||||
SELECT DISTINCT e.key AS col, e.value AS val
|
||||
FROM dataflow.mappings m
|
||||
JOIN dataflow.sources s ON s.name = m.source_name
|
||||
CROSS JOIN LATERAL jsonb_each_text(m.output) AS e(key, value)
|
||||
WHERE s.global_picklist = true
|
||||
AND e.value IS NOT NULL
|
||||
AND e.value <> ''
|
||||
ORDER BY e.key, e.value;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
|
||||
@ -85,7 +85,7 @@ CREATE OR REPLACE FUNCTION preview_rule(
|
||||
p_replace_value TEXT DEFAULT '',
|
||||
p_limit INT DEFAULT 20
|
||||
)
|
||||
RETURNS TABLE (id BIGINT, raw_value TEXT, extracted_value JSONB) AS $$
|
||||
RETURNS TABLE (id INT, raw_value TEXT, extracted_value JSONB) AS $$
|
||||
BEGIN
|
||||
IF p_function_type = 'replace' THEN
|
||||
RETURN QUERY
|
||||
|
||||
@ -17,19 +17,20 @@ RETURNS dataflow.sources AS $$
|
||||
SELECT * FROM dataflow.sources WHERE name = p_name;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION create_source(p_name TEXT, p_constraint_fields TEXT[], p_config JSONB DEFAULT '{}')
|
||||
CREATE OR REPLACE FUNCTION create_source(p_name TEXT, p_constraint_fields TEXT[], p_config JSONB DEFAULT '{}', p_global_picklist BOOLEAN DEFAULT true)
|
||||
RETURNS dataflow.sources AS $$
|
||||
INSERT INTO dataflow.sources (name, constraint_fields, config)
|
||||
VALUES (p_name, p_constraint_fields, p_config)
|
||||
INSERT INTO dataflow.sources (name, constraint_fields, config, global_picklist)
|
||||
VALUES (p_name, p_constraint_fields, p_config, p_global_picklist)
|
||||
RETURNING *;
|
||||
$$ LANGUAGE sql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION update_source(p_name TEXT, p_constraint_fields TEXT[] DEFAULT NULL, p_config JSONB DEFAULT NULL)
|
||||
CREATE OR REPLACE FUNCTION update_source(p_name TEXT, p_constraint_fields TEXT[] DEFAULT NULL, p_config JSONB DEFAULT NULL, p_global_picklist BOOLEAN DEFAULT NULL)
|
||||
RETURNS dataflow.sources AS $$
|
||||
UPDATE dataflow.sources
|
||||
SET constraint_fields = COALESCE(p_constraint_fields, constraint_fields),
|
||||
config = COALESCE(p_config, config),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
config = COALESCE(p_config, config),
|
||||
global_picklist = COALESCE(p_global_picklist, global_picklist),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE name = p_name
|
||||
RETURNING *;
|
||||
$$ LANGUAGE sql;
|
||||
@ -41,13 +42,6 @@ $$ LANGUAGE sql;
|
||||
|
||||
-- ── Import log ────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE OR REPLACE FUNCTION get_import_log(p_source_name TEXT)
|
||||
RETURNS SETOF dataflow.import_log AS $$
|
||||
SELECT * FROM dataflow.import_log
|
||||
WHERE source_name = p_source_name
|
||||
ORDER BY imported_at DESC;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
|
||||
-- ── Stats ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE OR REPLACE FUNCTION get_source_stats(p_source_name TEXT)
|
||||
@ -87,16 +81,21 @@ $$ LANGUAGE sql STABLE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION get_view_data(
|
||||
p_source_name TEXT,
|
||||
p_limit INT DEFAULT 100,
|
||||
p_offset INT DEFAULT 0,
|
||||
p_sort_col TEXT DEFAULT NULL,
|
||||
p_sort_dir TEXT DEFAULT 'asc'
|
||||
p_limit INT DEFAULT 100,
|
||||
p_offset INT DEFAULT 0,
|
||||
p_sort_col TEXT DEFAULT NULL,
|
||||
p_sort_dir TEXT DEFAULT 'asc',
|
||||
p_filters JSONB DEFAULT NULL -- [{col, pattern}, ...] — postgres regex (~*)
|
||||
)
|
||||
RETURNS JSON AS $$
|
||||
DECLARE
|
||||
v_exists BOOLEAN;
|
||||
v_order TEXT := '';
|
||||
v_rows JSON;
|
||||
v_exists BOOLEAN;
|
||||
v_where TEXT := '';
|
||||
v_order TEXT := '';
|
||||
v_rows JSON;
|
||||
v_filter JSONB;
|
||||
v_col TEXT;
|
||||
v_pattern TEXT;
|
||||
BEGIN
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.views
|
||||
@ -107,6 +106,24 @@ BEGIN
|
||||
RETURN json_build_object('exists', FALSE, 'rows', '[]'::json);
|
||||
END IF;
|
||||
|
||||
-- Build WHERE from filters (validate each column exists in the view)
|
||||
IF p_filters IS NOT NULL THEN
|
||||
FOR v_filter IN SELECT value FROM jsonb_array_elements(p_filters) LOOP
|
||||
v_col := v_filter->>'col';
|
||||
v_pattern := v_filter->>'pattern';
|
||||
IF v_pattern IS NOT NULL AND v_pattern <> '' AND EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'dfv'
|
||||
AND table_name = p_source_name
|
||||
AND column_name = v_col
|
||||
) THEN
|
||||
v_where := v_where ||
|
||||
CASE WHEN v_where = '' THEN ' WHERE ' ELSE ' AND ' END ||
|
||||
quote_ident(v_col) || '::text ~* ' || quote_literal(v_pattern);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
IF p_sort_col IS NOT NULL AND EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'dfv'
|
||||
@ -118,11 +135,9 @@ BEGIN
|
||||
|| ' NULLS LAST';
|
||||
END IF;
|
||||
|
||||
-- Subquery applies ORDER BY + LIMIT first, then json_agg collects in that order.
|
||||
-- json_agg on the outer query preserves column order (json not jsonb).
|
||||
EXECUTE format(
|
||||
'SELECT COALESCE(json_agg(row_to_json(t)), ''[]''::json) FROM (SELECT * FROM dfv.%I%s LIMIT %s OFFSET %s) t',
|
||||
p_source_name, v_order, p_limit, p_offset
|
||||
'SELECT COALESCE(json_agg(row_to_json(t)), ''[]''::json) FROM (SELECT * FROM dfv.%I%s%s LIMIT %s OFFSET %s) t',
|
||||
p_source_name, v_where, v_order, p_limit, p_offset
|
||||
) INTO v_rows;
|
||||
|
||||
RETURN json_build_object('exists', TRUE, 'rows', v_rows);
|
||||
|
||||
@ -17,6 +17,7 @@ CREATE TABLE sources (
|
||||
name TEXT PRIMARY KEY,
|
||||
constraint_fields TEXT[] NOT NULL, -- Fields that uniquely identify a record (e.g., ['date', 'amount', 'description'])
|
||||
config JSONB DEFAULT '{}'::jsonb,
|
||||
global_picklist BOOLEAN NOT NULL DEFAULT true, -- Contribute output values to global autocomplete suggestions
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
@ -10,6 +10,11 @@ export function clearCredentials() {
|
||||
_credentials = null
|
||||
}
|
||||
|
||||
export function authHeaders() {
|
||||
if (!_credentials) return {}
|
||||
return { 'Authorization': `Basic ${btoa(`${_credentials.user}:${_credentials.pass}`)}` }
|
||||
}
|
||||
|
||||
async function request(method, path, body, isFormData = false) {
|
||||
const opts = { method, headers: {} }
|
||||
|
||||
@ -65,9 +70,10 @@ export const api = {
|
||||
reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
|
||||
generateView: (name) => request('POST', `/sources/${name}/view`),
|
||||
getFields: (name) => request('GET', `/sources/${name}/fields`),
|
||||
getViewData: (name, limit = 100, offset = 0, sortCol = null, sortDir = 'asc') => {
|
||||
getViewData: (name, limit = 100, offset = 0, sortCol = null, sortDir = 'asc', filters = null) => {
|
||||
const params = new URLSearchParams({ limit, offset })
|
||||
if (sortCol) { params.set('sort_col', sortCol); params.set('sort_dir', sortDir) }
|
||||
if (filters && filters.length > 0) params.set('filters', JSON.stringify(filters))
|
||||
return request('GET', `/sources/${name}/view-data?${params}`)
|
||||
},
|
||||
|
||||
@ -81,6 +87,7 @@ export const api = {
|
||||
request('GET', `/rules/preview?source=${encodeURIComponent(source)}&field=${encodeURIComponent(field)}&pattern=${encodeURIComponent(pattern)}&flags=${encodeURIComponent(flags || '')}&function_type=${function_type}&replace_value=${encodeURIComponent(replace_value)}&limit=${limit}`),
|
||||
|
||||
// Mappings
|
||||
getGlobalValues: () => request('GET', '/mappings/global-values'),
|
||||
getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`),
|
||||
getMappingCounts: (source, rule) => request('GET', `/mappings/source/${source}/counts${rule ? `?rule_name=${rule}` : ''}`),
|
||||
getUnmapped: (source, rule) => request('GET', `/mappings/source/${source}/unmapped${rule ? `?rule_name=${rule}` : ''}`),
|
||||
|
||||
@ -1,5 +1,86 @@
|
||||
import { useState, useEffect } from 'react'
|
||||
import { api } from '../api'
|
||||
import { useState, useEffect, useRef } from 'react'
|
||||
import { api, authHeaders } from '../api'
|
||||
|
||||
function AutocompleteInput({ value, onChange, onEnter, suggestions = [], className, placeholder }) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const [highlighted, setHighlighted] = useState(0)
|
||||
const inputRef = useRef()
|
||||
const listRef = useRef()
|
||||
|
||||
const filtered = value
|
||||
? suggestions.filter(s => s.toLowerCase().includes(value.toLowerCase()))
|
||||
: suggestions
|
||||
|
||||
function openList() {
|
||||
setOpen(true)
|
||||
setHighlighted(0)
|
||||
}
|
||||
|
||||
function select(val) {
|
||||
onChange(val)
|
||||
setOpen(false)
|
||||
inputRef.current?.focus()
|
||||
}
|
||||
|
||||
function handleKeyDown(e) {
|
||||
if (e.altKey && e.key === 'ArrowDown') {
|
||||
e.preventDefault()
|
||||
openList()
|
||||
return
|
||||
}
|
||||
if (open && filtered.length > 0) {
|
||||
if (e.key === 'Tab') {
|
||||
e.preventDefault()
|
||||
setHighlighted(h => (h + 1) % filtered.length)
|
||||
return
|
||||
}
|
||||
if (e.key === 'ArrowDown') { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return }
|
||||
if (e.key === 'ArrowUp') { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return }
|
||||
if (e.key === 'Enter') { e.preventDefault(); select(filtered[highlighted]); return }
|
||||
if (e.key === 'Escape') { setOpen(false); return }
|
||||
}
|
||||
if (e.key === 'Enter') onEnter?.()
|
||||
}
|
||||
|
||||
// Scroll highlighted item into view
|
||||
useEffect(() => {
|
||||
if (!open || !listRef.current) return
|
||||
const item = listRef.current.children[highlighted]
|
||||
item?.scrollIntoView({ block: 'nearest' })
|
||||
}, [highlighted, open])
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
<input
|
||||
ref={inputRef}
|
||||
className={className}
|
||||
value={value}
|
||||
placeholder={placeholder}
|
||||
onChange={e => { onChange(e.target.value); if (!open && e.target.value) openList() }}
|
||||
onKeyDown={handleKeyDown}
|
||||
onBlur={e => { if (!listRef.current?.contains(e.relatedTarget)) setOpen(false) }}
|
||||
/>
|
||||
{open && filtered.length > 0 && (
|
||||
<div
|
||||
ref={listRef}
|
||||
className="absolute z-50 left-0 top-full mt-0.5 bg-white border border-gray-200 rounded shadow-lg max-h-48 overflow-y-auto min-w-full"
|
||||
>
|
||||
{filtered.map((s, i) => (
|
||||
<div
|
||||
key={s}
|
||||
className={`px-2 py-1 text-xs cursor-pointer whitespace-nowrap ${
|
||||
i === highlighted ? 'bg-blue-50 text-blue-700' : 'text-gray-700 hover:bg-gray-50'
|
||||
}`}
|
||||
onMouseDown={e => { e.preventDefault(); select(s) }}
|
||||
>
|
||||
{s}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function valueKey(v) {
|
||||
return Array.isArray(v) ? JSON.stringify(v) : String(v)
|
||||
@ -35,9 +116,11 @@ export default function Mappings({ source }) {
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [importing, setImporting] = useState(false)
|
||||
const [sortBy, setSortBy] = useState(null)
|
||||
const [globalValues, setGlobalValues] = useState({})
|
||||
|
||||
useEffect(() => {
|
||||
if (!source) return
|
||||
api.getGlobalValues().then(setGlobalValues).catch(() => {})
|
||||
api.getRules(source).then(r => setRules(r)).catch(() => {})
|
||||
}, [source])
|
||||
|
||||
@ -57,7 +140,7 @@ export default function Mappings({ source }) {
|
||||
.finally(() => setLoading(false))
|
||||
}, [source, selectedRule])
|
||||
|
||||
// Derive output columns and datalist suggestions from mapped rows
|
||||
// Derive output columns and datalist suggestions from mapped rows + global pool
|
||||
const existingCols = []
|
||||
const valuesByCol = {}
|
||||
allValues.forEach(row => {
|
||||
@ -68,6 +151,11 @@ export default function Mappings({ source }) {
|
||||
valuesByCol[k].add(String(v))
|
||||
})
|
||||
})
|
||||
// Merge global picklist values into suggestions
|
||||
Object.entries(globalValues).forEach(([k, vals]) => {
|
||||
if (!valuesByCol[k]) valuesByCol[k] = new Set()
|
||||
vals.forEach(v => valuesByCol[k].add(v))
|
||||
})
|
||||
const cols = [...existingCols, ...extraCols]
|
||||
|
||||
const unmappedCount = allValues.filter(r => !r.is_mapped).length
|
||||
@ -241,13 +329,26 @@ export default function Mappings({ source }) {
|
||||
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{selectedRule && (
|
||||
<a
|
||||
href={api.exportMappingsUrl(source, selectedRule)}
|
||||
download
|
||||
<button
|
||||
onClick={async () => {
|
||||
try {
|
||||
const url = api.exportMappingsUrl(source, selectedRule)
|
||||
const res = await fetch(url, { headers: authHeaders() })
|
||||
if (!res.ok) throw new Error('Export failed')
|
||||
const blob = await res.blob()
|
||||
const a = document.createElement('a')
|
||||
a.href = URL.createObjectURL(blob)
|
||||
a.download = `mappings_${source}.tsv`
|
||||
a.click()
|
||||
URL.revokeObjectURL(a.href)
|
||||
} catch (err) {
|
||||
alert(err.message)
|
||||
}
|
||||
}}
|
||||
className="text-sm px-3 py-1.5 border border-gray-200 rounded hover:bg-gray-50 text-gray-600"
|
||||
>
|
||||
Export TSV
|
||||
</a>
|
||||
</button>
|
||||
)}
|
||||
<label className={`text-sm px-3 py-1.5 border border-gray-200 rounded cursor-pointer hover:bg-gray-50 text-gray-600 ${importing ? 'opacity-50 pointer-events-none' : ''}`}>
|
||||
{importing ? 'Importing…' : 'Import TSV'}
|
||||
@ -269,13 +370,6 @@ export default function Mappings({ source }) {
|
||||
)}
|
||||
{selectedRule && !loading && allValues.length > 0 && (
|
||||
<div className="overflow-x-auto">
|
||||
{cols.map(col => (
|
||||
<datalist key={col} id={`dl-${col}`}>
|
||||
{[...(valuesByCol[col] || [])].sort().map(v => (
|
||||
<option key={v} value={v} />
|
||||
))}
|
||||
</datalist>
|
||||
))}
|
||||
<table className="w-full text-xs bg-white border border-gray-200 rounded">
|
||||
<thead>
|
||||
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
|
||||
@ -297,7 +391,7 @@ export default function Mappings({ source }) {
|
||||
<th className="px-2 py-2">
|
||||
<button
|
||||
onClick={() => setExtraCols(ec => [...ec, ''])}
|
||||
className="text-gray-300 hover:text-gray-500"
|
||||
className="text-gray-400 hover:text-gray-700 font-medium"
|
||||
title="Add column"
|
||||
>+</button>
|
||||
</th>
|
||||
@ -328,14 +422,14 @@ export default function Mappings({ source }) {
|
||||
<td className="px-3 py-1.5 text-right text-gray-400">{row.record_count}</td>
|
||||
{cols.map(col => (
|
||||
<td key={col} className="px-3 py-1.5">
|
||||
<input
|
||||
list={`dl-${col}`}
|
||||
<AutocompleteInput
|
||||
className={`border rounded px-2 py-1 w-full min-w-24 focus:outline-none focus:border-blue-400 ${
|
||||
hasDraft ? 'border-blue-300' : row.is_mapped ? 'border-gray-200' : 'border-yellow-300'
|
||||
}`}
|
||||
value={cellVal(col)}
|
||||
onChange={e => setCellValue(row.extracted_value, col, e.target.value)}
|
||||
onKeyDown={e => e.key === 'Enter' && saveRow(row)}
|
||||
onChange={v => setCellValue(row.extracted_value, col, v)}
|
||||
onEnter={() => saveRow(row)}
|
||||
suggestions={[...(valuesByCol[col] || [])].sort()}
|
||||
/>
|
||||
</td>
|
||||
))}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { useState, useEffect } from 'react'
|
||||
import { useState, useEffect, useRef } from 'react'
|
||||
import { api } from '../api'
|
||||
|
||||
const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
|
||||
@ -20,25 +20,32 @@ function formatVal(val) {
|
||||
|
||||
export default function Records({ source }) {
|
||||
const [rows, setRows] = useState([])
|
||||
const [cols, setCols] = useState([])
|
||||
const [exists, setExists] = useState(null)
|
||||
const [offset, setOffset] = useState(0)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [sort, setSort] = useState({ col: null, dir: 'asc' })
|
||||
const [filters, setFilters] = useState([])
|
||||
const debounceRef = useRef(null)
|
||||
const LIMIT = 100
|
||||
|
||||
useEffect(() => {
|
||||
if (!source) return
|
||||
setOffset(0)
|
||||
setSort({ col: null, dir: 'asc' })
|
||||
load(0, null, 'asc')
|
||||
setFilters([])
|
||||
load(0, null, 'asc', [])
|
||||
}, [source])
|
||||
|
||||
async function load(off, col, dir) {
|
||||
async function load(off, col, dir, filt) {
|
||||
setLoading(true)
|
||||
try {
|
||||
const res = await api.getViewData(source, LIMIT, off, col, dir)
|
||||
const active = (filt || []).filter(f => f.col && f.pattern)
|
||||
const res = await api.getViewData(source, LIMIT, off, col, dir, active)
|
||||
setExists(res.exists)
|
||||
setRows(res.rows)
|
||||
if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
|
||||
else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
} finally {
|
||||
@ -46,21 +53,44 @@ export default function Records({ source }) {
|
||||
}
|
||||
}
|
||||
|
||||
function triggerLoad(off, col, dir, filt) {
|
||||
clearTimeout(debounceRef.current)
|
||||
debounceRef.current = setTimeout(() => load(off, col, dir, filt), 350)
|
||||
}
|
||||
|
||||
function toggleSort(col) {
|
||||
const next = sort.col === col
|
||||
? { col, dir: sort.dir === 'asc' ? 'desc' : 'asc' }
|
||||
: { col, dir: 'asc' }
|
||||
setSort(next)
|
||||
setOffset(0)
|
||||
load(0, next.col, next.dir)
|
||||
load(0, next.col, next.dir, filters)
|
||||
}
|
||||
|
||||
function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir) }
|
||||
function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir) }
|
||||
function addFilter() {
|
||||
setFilters(f => [...f, { col: cols[0] || '', pattern: '' }])
|
||||
}
|
||||
|
||||
function removeFilter(i) {
|
||||
const next = filters.filter((_, idx) => idx !== i)
|
||||
setFilters(next)
|
||||
setOffset(0)
|
||||
load(0, sort.col, sort.dir, next)
|
||||
}
|
||||
|
||||
function updateFilter(i, key, val) {
|
||||
const next = filters.map((f, idx) => idx === i ? { ...f, [key]: val } : f)
|
||||
setFilters(next)
|
||||
setOffset(0)
|
||||
triggerLoad(0, sort.col, sort.dir, next)
|
||||
}
|
||||
|
||||
function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) }
|
||||
function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
|
||||
|
||||
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
|
||||
|
||||
const cols = rows.length > 0 ? Object.keys(rows[0]) : []
|
||||
const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols
|
||||
|
||||
return (
|
||||
<div className="p-6">
|
||||
@ -71,6 +101,48 @@ export default function Records({ source }) {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Filter bar */}
|
||||
{exists !== false && displayCols.length > 0 && (
|
||||
<div className="mb-4 flex flex-wrap gap-2 items-center">
|
||||
{filters.map((f, i) => (
|
||||
<div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
|
||||
<select
|
||||
className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
|
||||
value={f.col}
|
||||
onChange={e => updateFilter(i, 'col', e.target.value)}
|
||||
>
|
||||
{displayCols.map(c => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
<span className="text-xs text-gray-300 mx-0.5">~*</span>
|
||||
<input
|
||||
className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
|
||||
placeholder="regex…"
|
||||
value={f.pattern}
|
||||
onChange={e => updateFilter(i, 'pattern', e.target.value)}
|
||||
/>
|
||||
<button
|
||||
onClick={() => removeFilter(i)}
|
||||
className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
|
||||
>×</button>
|
||||
</div>
|
||||
))}
|
||||
<button
|
||||
onClick={addFilter}
|
||||
className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
|
||||
>
|
||||
+ filter
|
||||
</button>
|
||||
{filters.length > 0 && (
|
||||
<button
|
||||
onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
|
||||
className="text-xs text-gray-400 hover:text-red-500"
|
||||
>
|
||||
clear
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading && <p className="text-sm text-gray-400">Loading…</p>}
|
||||
|
||||
{!loading && exists === false && (
|
||||
@ -80,7 +152,9 @@ export default function Records({ source }) {
|
||||
)}
|
||||
|
||||
{!loading && exists && rows.length === 0 && (
|
||||
<p className="text-sm text-gray-400">View exists but no transformed records yet. Import data and run a transform first.</p>
|
||||
<p className="text-sm text-gray-400">
|
||||
{filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{!loading && exists && rows.length > 0 && (
|
||||
@ -89,7 +163,7 @@ export default function Records({ source }) {
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
<tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
|
||||
{cols.map(col => {
|
||||
{displayCols.map(col => {
|
||||
const active = sort.col === col
|
||||
return (
|
||||
<th
|
||||
@ -109,7 +183,7 @@ export default function Records({ source }) {
|
||||
<tbody>
|
||||
{rows.map((row, i) => (
|
||||
<tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
|
||||
{cols.map((col, j) => {
|
||||
{displayCols.map((col, j) => {
|
||||
const formatted = formatVal(row[col])
|
||||
return (
|
||||
<td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">
|
||||
|
||||
@ -5,6 +5,7 @@ const FIELD_TYPES = ['text', 'numeric', 'date']
|
||||
|
||||
export default function Sources({ source, sources, setSources, setSource }) {
|
||||
const [constraintFields, setConstraintFields] = useState('')
|
||||
const [globalPicklist, setGlobalPicklist] = useState(true)
|
||||
const [schemaFields, setSchemaFields] = useState([])
|
||||
const [stats, setStats] = useState(null)
|
||||
const [saving, setSaving] = useState(false)
|
||||
@ -25,6 +26,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
useEffect(() => {
|
||||
if (!sourceObj) return
|
||||
setConstraintFields(sourceObj.constraint_fields?.join(', ') || '')
|
||||
setGlobalPicklist(sourceObj.global_picklist !== false)
|
||||
setSchemaFields((sourceObj.config?.fields || []).map((f, i) => ({ seq: i + 1, ...f })))
|
||||
setViewName(sourceObj.config?.fields?.length ? `dfv.${sourceObj.name}` : '')
|
||||
setResult('')
|
||||
@ -43,7 +45,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
const constraint_fields = constraintFields.split(',').map(s => s.trim()).filter(Boolean)
|
||||
const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
|
||||
const config = { ...(sourceObj.config || {}), fields }
|
||||
await api.updateSource(sourceObj.name, { constraint_fields, config })
|
||||
await api.updateSource(sourceObj.name, { constraint_fields, config, global_picklist: globalPicklist })
|
||||
const updated = await api.getSources()
|
||||
setSources(updated)
|
||||
setResult('Saved.')
|
||||
@ -62,7 +64,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
const constraint_fields = constraintFields.split(',').map(s => s.trim()).filter(Boolean)
|
||||
const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
|
||||
const config = { ...(sourceObj.config || {}), fields }
|
||||
await api.updateSource(sourceObj.name, { constraint_fields, config })
|
||||
await api.updateSource(sourceObj.name, { constraint_fields, config, global_picklist: globalPicklist })
|
||||
const res = await api.generateView(sourceObj.name)
|
||||
if (res.success) {
|
||||
setViewName(res.view)
|
||||
@ -133,11 +135,11 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
setCreateLoading(true)
|
||||
try {
|
||||
const config = form.schema.length > 0 ? { fields: form.schema } : {}
|
||||
await api.createSource({ name: form.name, constraint_fields: constraintArr, config })
|
||||
await api.createSource({ name: form.name, constraint_fields: constraintArr, config, global_picklist: form.global_picklist !== false })
|
||||
const updated = await api.getSources()
|
||||
setSources(updated)
|
||||
setSource(form.name)
|
||||
setForm({ name: '', constraint_fields: '', fields: [], schema: [] })
|
||||
setForm({ name: '', constraint_fields: '', fields: [], schema: [], global_picklist: true })
|
||||
setCreating(false)
|
||||
} catch (err) {
|
||||
setCreateError(err.message)
|
||||
@ -273,7 +275,11 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div className="flex items-center gap-3 pt-1">
|
||||
<div className="flex items-center gap-3 pt-1 flex-wrap">
|
||||
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
|
||||
<input type="checkbox" checked={globalPicklist} onChange={e => setGlobalPicklist(e.target.checked)} />
|
||||
Global picklist
|
||||
</label>
|
||||
<form onSubmit={handleSave}>
|
||||
<button type="submit" disabled={saving}
|
||||
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
||||
@ -300,12 +306,18 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
|
||||
{/* Save button when no fields loaded yet */}
|
||||
{availableFields.length === 0 && (
|
||||
<form onSubmit={handleSave}>
|
||||
<button type="submit" disabled={saving}
|
||||
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</form>
|
||||
<div className="flex items-center gap-3">
|
||||
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
|
||||
<input type="checkbox" checked={globalPicklist} onChange={e => setGlobalPicklist(e.target.checked)} />
|
||||
Global picklist
|
||||
</label>
|
||||
<form onSubmit={handleSave}>
|
||||
<button type="submit" disabled={saving}
|
||||
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Reprocess */}
|
||||
@ -400,6 +412,15 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
<label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={form.global_picklist !== false}
|
||||
onChange={e => setForm(f => ({ ...f, global_picklist: e.target.checked }))}
|
||||
/>
|
||||
Global picklist
|
||||
</label>
|
||||
|
||||
{createError && <p className="text-xs text-red-500">{createError}</p>}
|
||||
|
||||
<div className="flex gap-2">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user