Unify mappings UI around single SQL query with full UX improvements
- Add get_all_values() SQL function returning all extracted values (mapped + unmapped) with real record counts and mapping output in one query - Add /mappings/source/:source/all-values API endpoint - Rewrite All tab to use get_all_values directly instead of merging two separate API calls; counts now populated for all rows - Rewrite export.tsv to use get_all_values (real counts for mapped rows) - Fix save bug where editing one output field blanked unedited fields by merging drafts over existing mapping output instead of replacing - Add dirty row highlighting (blue tint) and per-rule Save All button - Fix sort instability during editing by sorting on committed values only Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
4cf5be52e8
commit
dcac6def87
@ -36,6 +36,38 @@ module.exports = (pool) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Get record counts for existing mappings
|
||||||
|
router.get('/source/:source_name/counts', async (req, res, next) => {
|
||||||
|
try {
|
||||||
|
const { rule_name } = req.query;
|
||||||
|
const params = [req.params.source_name];
|
||||||
|
let ruleFilter = '';
|
||||||
|
if (rule_name) {
|
||||||
|
ruleFilter = 'AND m.rule_name = $2';
|
||||||
|
params.push(rule_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await pool.query(`
|
||||||
|
SELECT
|
||||||
|
m.rule_name,
|
||||||
|
m.input_value,
|
||||||
|
COUNT(rec.id) AS record_count
|
||||||
|
FROM mappings m
|
||||||
|
JOIN rules r ON r.source_name = m.source_name AND r.name = m.rule_name
|
||||||
|
LEFT JOIN records rec ON
|
||||||
|
rec.source_name = m.source_name
|
||||||
|
AND rec.transformed ? r.output_field
|
||||||
|
AND rec.transformed -> r.output_field = m.input_value
|
||||||
|
WHERE m.source_name = $1 ${ruleFilter}
|
||||||
|
GROUP BY m.rule_name, m.input_value
|
||||||
|
`, params);
|
||||||
|
|
||||||
|
res.json(result.rows);
|
||||||
|
} catch (err) {
|
||||||
|
next(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Get unmapped values
|
// Get unmapped values
|
||||||
router.get('/source/:source_name/unmapped', async (req, res, next) => {
|
router.get('/source/:source_name/unmapped', async (req, res, next) => {
|
||||||
try {
|
try {
|
||||||
@ -52,7 +84,21 @@ module.exports = (pool) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Export unmapped values + existing mappings as TSV
|
// Get all extracted values (mapped + unmapped) with counts — single SQL run
|
||||||
|
router.get('/source/:source_name/all-values', async (req, res, next) => {
|
||||||
|
try {
|
||||||
|
const { rule_name } = req.query;
|
||||||
|
const result = await pool.query(
|
||||||
|
'SELECT * FROM get_all_values($1, $2)',
|
||||||
|
[req.params.source_name, rule_name || null]
|
||||||
|
);
|
||||||
|
res.json(result.rows);
|
||||||
|
} catch (err) {
|
||||||
|
next(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Export all extracted values (mapped + unmapped) as TSV via get_all_values
|
||||||
// Columns: source_name, rule_name, input_value, record_count, <output keys...>, sample
|
// Columns: source_name, rule_name, input_value, record_count, <output keys...>, sample
|
||||||
// sample is always last and is discarded on import
|
// sample is always last and is discarded on import
|
||||||
router.get('/source/:source_name/export.tsv', async (req, res, next) => {
|
router.get('/source/:source_name/export.tsv', async (req, res, next) => {
|
||||||
@ -60,17 +106,14 @@ module.exports = (pool) => {
|
|||||||
const { rule_name } = req.query;
|
const { rule_name } = req.query;
|
||||||
const source_name = req.params.source_name;
|
const source_name = req.params.source_name;
|
||||||
|
|
||||||
const [unmappedResult, mappedResult] = await Promise.all([
|
const result = await pool.query(
|
||||||
pool.query('SELECT * FROM get_unmapped_values($1, $2)', [source_name, rule_name || null]),
|
'SELECT * FROM get_all_values($1, $2)',
|
||||||
pool.query(
|
[source_name, rule_name || null]
|
||||||
'SELECT * FROM mappings WHERE source_name = $1' + (rule_name ? ' AND rule_name = $2' : '') + ' ORDER BY rule_name, input_value',
|
);
|
||||||
rule_name ? [source_name, rule_name] : [source_name]
|
|
||||||
)
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Collect output keys from existing mappings
|
// Collect output keys from mapped rows
|
||||||
const outputKeys = [];
|
const outputKeys = [];
|
||||||
for (const row of mappedResult.rows) {
|
for (const row of result.rows) {
|
||||||
for (const key of Object.keys(row.output || {})) {
|
for (const key of Object.keys(row.output || {})) {
|
||||||
if (!outputKeys.includes(key)) outputKeys.push(key);
|
if (!outputKeys.includes(key)) outputKeys.push(key);
|
||||||
}
|
}
|
||||||
@ -81,31 +124,17 @@ module.exports = (pool) => {
|
|||||||
// sample is always last
|
// sample is always last
|
||||||
const allCols = ['source_name', 'rule_name', 'input_value', 'record_count', ...outputKeys, 'sample'];
|
const allCols = ['source_name', 'rule_name', 'input_value', 'record_count', ...outputKeys, 'sample'];
|
||||||
|
|
||||||
const dataRows = [];
|
const dataRows = result.rows.map(row => {
|
||||||
|
const input_value = Array.isArray(row.extracted_value)
|
||||||
for (const row of unmappedResult.rows) {
|
? JSON.stringify(row.extracted_value)
|
||||||
const r = {
|
: String(row.extracted_value ?? '');
|
||||||
source_name,
|
const sample = Array.isArray(row.sample)
|
||||||
rule_name: row.rule_name,
|
? row.sample.map(r => r[row.source_field] ?? '').filter(Boolean).slice(0, 3).join(' | ')
|
||||||
input_value: Array.isArray(row.extracted_value) ? JSON.stringify(row.extracted_value) : String(row.extracted_value ?? ''),
|
: String(row.sample ?? '');
|
||||||
record_count: row.record_count,
|
const r = { source_name, rule_name: row.rule_name, input_value, record_count: row.record_count, sample };
|
||||||
sample: Array.isArray(row.sample) ? row.sample.join(' | ') : String(row.sample ?? '')
|
|
||||||
};
|
|
||||||
for (const key of outputKeys) r[key] = '';
|
|
||||||
dataRows.push(r);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const row of mappedResult.rows) {
|
|
||||||
const r = {
|
|
||||||
source_name: row.source_name,
|
|
||||||
rule_name: row.rule_name,
|
|
||||||
input_value: Array.isArray(row.input_value) ? JSON.stringify(row.input_value) : String(row.input_value ?? ''),
|
|
||||||
record_count: '',
|
|
||||||
sample: ''
|
|
||||||
};
|
|
||||||
for (const key of outputKeys) r[key] = row.output?.[key] ?? '';
|
for (const key of outputKeys) r[key] = row.output?.[key] ?? '';
|
||||||
dataRows.push(r);
|
return r;
|
||||||
}
|
});
|
||||||
|
|
||||||
const tsv = [
|
const tsv = [
|
||||||
allCols.map(escape).join('\t'),
|
allCols.map(escape).join('\t'),
|
||||||
|
|||||||
@ -87,9 +87,11 @@ CREATE AGGREGATE dataflow.jsonb_concat_obj(JSONB) (
|
|||||||
-- Function: apply_transformations
|
-- Function: apply_transformations
|
||||||
-- Apply all transformation rules to records (set-based)
|
-- Apply all transformation rules to records (set-based)
|
||||||
------------------------------------------------------
|
------------------------------------------------------
|
||||||
|
DROP FUNCTION IF EXISTS apply_transformations(TEXT, INTEGER[]);
|
||||||
CREATE OR REPLACE FUNCTION apply_transformations(
|
CREATE OR REPLACE FUNCTION apply_transformations(
|
||||||
p_source_name TEXT,
|
p_source_name TEXT,
|
||||||
p_record_ids INTEGER[] DEFAULT NULL -- NULL = all untransformed
|
p_record_ids INTEGER[] DEFAULT NULL, -- NULL = all eligible records
|
||||||
|
p_overwrite BOOLEAN DEFAULT FALSE -- FALSE = skip already-transformed, TRUE = overwrite all
|
||||||
) RETURNS JSON AS $$
|
) RETURNS JSON AS $$
|
||||||
WITH
|
WITH
|
||||||
-- All records to process
|
-- All records to process
|
||||||
@ -97,65 +99,66 @@ qualifying AS (
|
|||||||
SELECT id, data
|
SELECT id, data
|
||||||
FROM dataflow.records
|
FROM dataflow.records
|
||||||
WHERE source_name = p_source_name
|
WHERE source_name = p_source_name
|
||||||
AND transformed IS NULL
|
AND (p_overwrite OR transformed IS NULL)
|
||||||
AND (p_record_ids IS NULL OR id = ANY(p_record_ids))
|
AND (p_record_ids IS NULL OR id = ANY(p_record_ids))
|
||||||
),
|
),
|
||||||
-- Apply each enabled rule to each qualifying record that has the required field
|
-- Mirror TPS rx: fan out one row per regex match, drive from rules → records
|
||||||
rx AS (
|
rx AS (
|
||||||
SELECT
|
SELECT
|
||||||
q.id,
|
q.id,
|
||||||
r.name AS rule_name,
|
r.name AS rule_name,
|
||||||
r.sequence,
|
r.sequence,
|
||||||
r.output_field,
|
r.output_field,
|
||||||
r.retain,
|
r.retain,
|
||||||
CASE r.function_type
|
r.function_type,
|
||||||
WHEN 'replace' THEN
|
COALESCE(mt.rn, rp.rn, 1) AS result_number,
|
||||||
to_jsonb(regexp_replace(
|
-- extract: build map_val and retain_val per match (mirrors TPS)
|
||||||
q.data ->> r.field, r.pattern, r.replace_value, r.flags
|
CASE WHEN array_length(mt.mt, 1) = 1 THEN to_jsonb(mt.mt[1]) ELSE to_jsonb(mt.mt) END AS match_val,
|
||||||
))
|
to_jsonb(rp.rp) AS replace_val
|
||||||
ELSE
|
FROM dataflow.rules r
|
||||||
-- extract: aggregate all matches; single match → scalar, multiple → array
|
INNER JOIN qualifying q ON q.data ? r.field
|
||||||
-- Aggregate first so we can inspect count and first element cleanly
|
LEFT JOIN LATERAL regexp_matches(q.data ->> r.field, r.pattern, r.flags)
|
||||||
(SELECT
|
WITH ORDINALITY AS mt(mt, rn) ON r.function_type = 'extract'
|
||||||
CASE WHEN cnt = 0 THEN NULL
|
LEFT JOIN LATERAL regexp_replace(q.data ->> r.field, r.pattern, r.replace_value, r.flags)
|
||||||
WHEN cnt = 1 THEN agg->0
|
WITH ORDINALITY AS rp(rp, rn) ON r.function_type = 'replace'
|
||||||
ELSE agg
|
|
||||||
END
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
count(*) AS cnt,
|
|
||||||
jsonb_agg(
|
|
||||||
CASE WHEN array_length(mt, 1) = 1
|
|
||||||
THEN to_jsonb(mt[1])
|
|
||||||
ELSE to_jsonb(mt)
|
|
||||||
END
|
|
||||||
ORDER BY rn
|
|
||||||
) AS agg
|
|
||||||
FROM regexp_matches(q.data ->> r.field, r.pattern, r.flags)
|
|
||||||
WITH ORDINALITY AS m(mt, rn)
|
|
||||||
) _agg)
|
|
||||||
END AS extracted
|
|
||||||
FROM qualifying q
|
|
||||||
CROSS JOIN dataflow.rules r
|
|
||||||
WHERE r.source_name = p_source_name
|
WHERE r.source_name = p_source_name
|
||||||
AND r.enabled = true
|
AND r.enabled = true
|
||||||
AND q.data ? r.field
|
|
||||||
),
|
),
|
||||||
-- Join with mappings to find mapped output for each extracted value
|
-- Aggregate match rows back into one value per (record, rule) — mirrors TPS agg_to_target_items
|
||||||
|
agg_matches AS (
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
rule_name,
|
||||||
|
sequence,
|
||||||
|
output_field,
|
||||||
|
retain,
|
||||||
|
function_type,
|
||||||
|
CASE function_type
|
||||||
|
WHEN 'replace' THEN jsonb_agg(replace_val) -> 0
|
||||||
|
ELSE
|
||||||
|
CASE WHEN max(result_number) = 1
|
||||||
|
THEN jsonb_agg(match_val ORDER BY result_number) -> 0
|
||||||
|
ELSE jsonb_agg(match_val ORDER BY result_number)
|
||||||
|
END
|
||||||
|
END AS extracted
|
||||||
|
FROM rx
|
||||||
|
GROUP BY id, rule_name, sequence, output_field, retain, function_type
|
||||||
|
),
|
||||||
|
-- Join with mappings to find mapped output — mirrors TPS link_map
|
||||||
linked AS (
|
linked AS (
|
||||||
SELECT
|
SELECT
|
||||||
rx.id,
|
a.id,
|
||||||
rx.sequence,
|
a.sequence,
|
||||||
rx.output_field,
|
a.output_field,
|
||||||
rx.retain,
|
a.retain,
|
||||||
rx.extracted,
|
a.extracted,
|
||||||
m.output AS mapped
|
m.output AS mapped
|
||||||
FROM rx
|
FROM agg_matches a
|
||||||
LEFT JOIN dataflow.mappings m ON
|
LEFT JOIN dataflow.mappings m ON
|
||||||
m.source_name = p_source_name
|
m.source_name = p_source_name
|
||||||
AND m.rule_name = rx.rule_name
|
AND m.rule_name = a.rule_name
|
||||||
AND m.input_value = rx.extracted
|
AND m.input_value = a.extracted
|
||||||
WHERE rx.extracted IS NOT NULL
|
WHERE a.extracted IS NOT NULL
|
||||||
),
|
),
|
||||||
-- Build per-rule output JSONB:
|
-- Build per-rule output JSONB:
|
||||||
-- mapped → use mapping output; also write output_field if retain = true
|
-- mapped → use mapping output; also write output_field if retain = true
|
||||||
@ -176,7 +179,7 @@ rule_output AS (
|
|||||||
END AS output
|
END AS output
|
||||||
FROM linked
|
FROM linked
|
||||||
),
|
),
|
||||||
-- Merge all rule outputs per record in sequence order (higher sequence wins on conflict)
|
-- Merge all rule outputs per record in sequence order — mirrors TPS agg_to_id
|
||||||
record_additions AS (
|
record_additions AS (
|
||||||
SELECT
|
SELECT
|
||||||
id,
|
id,
|
||||||
@ -200,6 +203,80 @@ $$ LANGUAGE sql;
|
|||||||
|
|
||||||
COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and mappings to records (set-based CTE)';
|
COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and mappings to records (set-based CTE)';
|
||||||
|
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Function: get_all_values
|
||||||
|
-- All extracted values (mapped + unmapped) with counts and mapping output
|
||||||
|
------------------------------------------------------
|
||||||
|
DROP FUNCTION IF EXISTS get_all_values(TEXT, TEXT);
|
||||||
|
CREATE FUNCTION get_all_values(
|
||||||
|
p_source_name TEXT,
|
||||||
|
p_rule_name TEXT DEFAULT NULL
|
||||||
|
) RETURNS TABLE (
|
||||||
|
rule_name TEXT,
|
||||||
|
output_field TEXT,
|
||||||
|
source_field TEXT,
|
||||||
|
extracted_value JSONB,
|
||||||
|
record_count BIGINT,
|
||||||
|
sample JSONB,
|
||||||
|
mapping_id INTEGER,
|
||||||
|
output JSONB,
|
||||||
|
is_mapped BOOLEAN
|
||||||
|
) AS $$
|
||||||
|
BEGIN
|
||||||
|
RETURN QUERY
|
||||||
|
WITH extracted AS (
|
||||||
|
SELECT
|
||||||
|
r.name AS rule_name,
|
||||||
|
r.output_field,
|
||||||
|
r.field AS source_field,
|
||||||
|
rec.transformed->r.output_field AS extracted_value,
|
||||||
|
rec.data AS record_data,
|
||||||
|
row_number() OVER (
|
||||||
|
PARTITION BY r.name, rec.transformed->r.output_field
|
||||||
|
ORDER BY rec.id
|
||||||
|
) AS rn
|
||||||
|
FROM dataflow.records rec
|
||||||
|
CROSS JOIN dataflow.rules r
|
||||||
|
WHERE
|
||||||
|
rec.source_name = p_source_name
|
||||||
|
AND r.source_name = p_source_name
|
||||||
|
AND rec.transformed IS NOT NULL
|
||||||
|
AND rec.transformed ? r.output_field
|
||||||
|
AND (p_rule_name IS NULL OR r.name = p_rule_name)
|
||||||
|
AND rec.data ? r.field
|
||||||
|
),
|
||||||
|
aggregated AS (
|
||||||
|
SELECT
|
||||||
|
e.rule_name,
|
||||||
|
e.output_field,
|
||||||
|
e.source_field,
|
||||||
|
e.extracted_value,
|
||||||
|
count(*) AS record_count,
|
||||||
|
jsonb_agg(e.record_data ORDER BY e.rn) FILTER (WHERE e.rn <= 5) AS sample
|
||||||
|
FROM extracted e
|
||||||
|
GROUP BY e.rule_name, e.output_field, e.source_field, e.extracted_value
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
a.rule_name,
|
||||||
|
a.output_field,
|
||||||
|
a.source_field,
|
||||||
|
a.extracted_value,
|
||||||
|
a.record_count,
|
||||||
|
a.sample,
|
||||||
|
m.id AS mapping_id,
|
||||||
|
m.output,
|
||||||
|
(m.id IS NOT NULL) AS is_mapped
|
||||||
|
FROM aggregated a
|
||||||
|
LEFT JOIN dataflow.mappings m ON
|
||||||
|
m.source_name = p_source_name
|
||||||
|
AND m.rule_name = a.rule_name
|
||||||
|
AND m.input_value = a.extracted_value
|
||||||
|
ORDER BY a.record_count DESC;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
COMMENT ON FUNCTION get_all_values IS 'All extracted values with record counts and mapping output (single query for All tab)';
|
||||||
|
|
||||||
------------------------------------------------------
|
------------------------------------------------------
|
||||||
-- Function: get_unmapped_values
|
-- Function: get_unmapped_values
|
||||||
-- Find extracted values that need mappings
|
-- Find extracted values that need mappings
|
||||||
@ -224,7 +301,11 @@ BEGIN
|
|||||||
r.output_field,
|
r.output_field,
|
||||||
r.field AS source_field,
|
r.field AS source_field,
|
||||||
rec.transformed->r.output_field AS extracted_value,
|
rec.transformed->r.output_field AS extracted_value,
|
||||||
rec.data->>r.field AS source_value
|
rec.data AS record_data,
|
||||||
|
row_number() OVER (
|
||||||
|
PARTITION BY r.name, rec.transformed->r.output_field
|
||||||
|
ORDER BY rec.id
|
||||||
|
) AS rn
|
||||||
FROM
|
FROM
|
||||||
dataflow.records rec
|
dataflow.records rec
|
||||||
CROSS JOIN dataflow.rules r
|
CROSS JOIN dataflow.rules r
|
||||||
@ -242,7 +323,7 @@ BEGIN
|
|||||||
e.source_field,
|
e.source_field,
|
||||||
e.extracted_value,
|
e.extracted_value,
|
||||||
count(*) AS record_count,
|
count(*) AS record_count,
|
||||||
jsonb_agg(DISTINCT e.source_value) FILTER (WHERE e.source_value IS NOT NULL) AS sample
|
jsonb_agg(e.record_data ORDER BY e.rn) FILTER (WHERE e.rn <= 5) AS sample
|
||||||
FROM extracted e
|
FROM extracted e
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
SELECT 1 FROM dataflow.mappings m
|
SELECT 1 FROM dataflow.mappings m
|
||||||
@ -263,17 +344,9 @@ COMMENT ON FUNCTION get_unmapped_values IS 'Find extracted values that need mapp
|
|||||||
------------------------------------------------------
|
------------------------------------------------------
|
||||||
CREATE OR REPLACE FUNCTION reprocess_records(p_source_name TEXT)
|
CREATE OR REPLACE FUNCTION reprocess_records(p_source_name TEXT)
|
||||||
RETURNS JSON AS $$
|
RETURNS JSON AS $$
|
||||||
BEGIN
|
-- Overwrite all records directly — no clear step, mirrors TPS srce_map_overwrite
|
||||||
-- Clear existing transformations
|
SELECT dataflow.apply_transformations(p_source_name, NULL, TRUE)
|
||||||
UPDATE dataflow.records
|
$$ LANGUAGE sql;
|
||||||
SET transformed = NULL,
|
|
||||||
transformed_at = NULL
|
|
||||||
WHERE source_name = p_source_name;
|
|
||||||
|
|
||||||
-- Reapply transformations
|
|
||||||
RETURN dataflow.apply_transformations(p_source_name);
|
|
||||||
END;
|
|
||||||
$$ LANGUAGE plpgsql;
|
|
||||||
|
|
||||||
COMMENT ON FUNCTION reprocess_records IS 'Clear and reapply all transformations for a source';
|
COMMENT ON FUNCTION reprocess_records IS 'Clear and reapply all transformations for a source';
|
||||||
|
|
||||||
|
|||||||
@ -52,7 +52,9 @@ export const api = {
|
|||||||
|
|
||||||
// Mappings
|
// Mappings
|
||||||
getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`),
|
getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`),
|
||||||
|
getMappingCounts: (source, rule) => request('GET', `/mappings/source/${source}/counts${rule ? `?rule_name=${rule}` : ''}`),
|
||||||
getUnmapped: (source, rule) => request('GET', `/mappings/source/${source}/unmapped${rule ? `?rule_name=${rule}` : ''}`),
|
getUnmapped: (source, rule) => request('GET', `/mappings/source/${source}/unmapped${rule ? `?rule_name=${rule}` : ''}`),
|
||||||
|
getAllValues: (source, rule) => request('GET', `/mappings/source/${source}/all-values${rule ? `?rule_name=${rule}` : ''}`),
|
||||||
exportMappingsUrl: (source, rule) => `${BASE}/mappings/source/${source}/export.tsv${rule ? `?rule_name=${rule}` : ''}`,
|
exportMappingsUrl: (source, rule) => `${BASE}/mappings/source/${source}/export.tsv${rule ? `?rule_name=${rule}` : ''}`,
|
||||||
importMappingsCSV: (source, file) => {
|
importMappingsCSV: (source, file) => {
|
||||||
const fd = new FormData()
|
const fd = new FormData()
|
||||||
|
|||||||
@ -1,15 +1,27 @@
|
|||||||
import { useState, useEffect } from 'react'
|
import { useState, useEffect } from 'react'
|
||||||
import { api } from '../api'
|
import { api } from '../api'
|
||||||
|
|
||||||
// Stable string key for a value that may be a string or array
|
|
||||||
function valueKey(v) {
|
function valueKey(v) {
|
||||||
return Array.isArray(v) ? JSON.stringify(v) : String(v)
|
return Array.isArray(v) ? JSON.stringify(v) : String(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Human-readable display of a string or array extracted value
|
|
||||||
function displayValue(v) {
|
function displayValue(v) {
|
||||||
if (Array.isArray(v)) return v.join(' · ')
|
if (Array.isArray(v)) return v.join(' · ')
|
||||||
return v
|
return String(v ?? '')
|
||||||
|
}
|
||||||
|
|
||||||
|
function SortHeader({ ruleName, col, label, sortBy, onSort, className = '' }) {
|
||||||
|
const s = sortBy[ruleName]
|
||||||
|
const active = s?.col === col
|
||||||
|
return (
|
||||||
|
<th
|
||||||
|
className={`px-3 py-2 font-medium cursor-pointer select-none hover:text-gray-600 ${className}`}
|
||||||
|
onClick={() => onSort(ruleName, col)}
|
||||||
|
>
|
||||||
|
{label}
|
||||||
|
<span className="ml-1 text-gray-300">{active ? (s.dir === 'asc' ? '↑' : '↓') : '↕'}</span>
|
||||||
|
</th>
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function Mappings({ source }) {
|
export default function Mappings({ source }) {
|
||||||
@ -18,13 +30,19 @@ export default function Mappings({ source }) {
|
|||||||
const [selectedRule, setSelectedRule] = useState('')
|
const [selectedRule, setSelectedRule] = useState('')
|
||||||
const [unmapped, setUnmapped] = useState([])
|
const [unmapped, setUnmapped] = useState([])
|
||||||
const [mapped, setMapped] = useState([])
|
const [mapped, setMapped] = useState([])
|
||||||
const [drafts, setDrafts] = useState({}) // key: extracted_value => [{ key, value }]
|
// drafts[valueKey][colKey] = value
|
||||||
|
const [drafts, setDrafts] = useState({})
|
||||||
|
// extraCols[ruleName] = [colName, ...] — user-added columns
|
||||||
|
const [extraCols, setExtraCols] = useState({})
|
||||||
const [saving, setSaving] = useState({})
|
const [saving, setSaving] = useState({})
|
||||||
const [sampleOpen, setSampleOpen] = useState({})
|
const [sampleOpen, setSampleOpen] = useState({})
|
||||||
const [loading, setLoading] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
const [editingId, setEditingId] = useState(null)
|
const [editingId, setEditingId] = useState(null)
|
||||||
const [editDrafts, setEditDrafts] = useState({})
|
const [editDrafts, setEditDrafts] = useState({})
|
||||||
const [importing, setImporting] = useState(false)
|
const [importing, setImporting] = useState(false)
|
||||||
|
// sortBy[ruleName] = { col, dir: 'asc'|'desc' }
|
||||||
|
const [sortBy, setSortBy] = useState({})
|
||||||
|
const [allValues, setAllValues] = useState([])
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!source) return
|
if (!source) return
|
||||||
@ -40,49 +58,93 @@ export default function Mappings({ source }) {
|
|||||||
const rule = selectedRule || undefined
|
const rule = selectedRule || undefined
|
||||||
Promise.all([
|
Promise.all([
|
||||||
api.getUnmapped(source, rule),
|
api.getUnmapped(source, rule),
|
||||||
tab === 'mapped' ? api.getMappings(source, rule) : Promise.resolve([])
|
api.getMappings(source, rule),
|
||||||
]).then(([u, m]) => {
|
api.getAllValues(source, rule)
|
||||||
|
]).then(([u, m, a]) => {
|
||||||
setUnmapped(u)
|
setUnmapped(u)
|
||||||
setMapped(m)
|
setMapped(m)
|
||||||
|
setAllValues(a)
|
||||||
setDrafts({})
|
setDrafts({})
|
||||||
|
setExtraCols({})
|
||||||
}).catch(() => {}).finally(() => setLoading(false))
|
}).catch(() => {}).finally(() => setLoading(false))
|
||||||
}, [source, selectedRule, tab])
|
}, [source, selectedRule])
|
||||||
|
|
||||||
function getDraft(extractedValue, outputField) {
|
// Derive existing output key columns from mapped values, per rule
|
||||||
return drafts[valueKey(extractedValue)] || [{ key: outputField, value: '' }]
|
const existingColsByRule = {}
|
||||||
}
|
// Distinct values already used per rule+column (for datalist suggestions)
|
||||||
|
const valuesByRuleCol = {}
|
||||||
|
mapped.forEach(m => {
|
||||||
|
if (!existingColsByRule[m.rule_name]) existingColsByRule[m.rule_name] = []
|
||||||
|
Object.entries(m.output || {}).forEach(([k, v]) => {
|
||||||
|
if (!existingColsByRule[m.rule_name].includes(k))
|
||||||
|
existingColsByRule[m.rule_name].push(k)
|
||||||
|
if (!valuesByRuleCol[m.rule_name]) valuesByRuleCol[m.rule_name] = {}
|
||||||
|
if (!valuesByRuleCol[m.rule_name][k]) valuesByRuleCol[m.rule_name][k] = new Set()
|
||||||
|
valuesByRuleCol[m.rule_name][k].add(String(v))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
function updateDraftKey(extractedValue, index, newKey) {
|
function toggleSort(ruleName, col) {
|
||||||
const k = valueKey(extractedValue)
|
setSortBy(s => {
|
||||||
setDrafts(d => {
|
const cur = s[ruleName]
|
||||||
const current = d[k] || [{ key: '', value: '' }]
|
if (cur?.col === col) return { ...s, [ruleName]: { col, dir: cur.dir === 'asc' ? 'desc' : 'asc' } }
|
||||||
const updated = current.map((pair, i) => i === index ? { ...pair, key: newKey } : pair)
|
return { ...s, [ruleName]: { col, dir: 'asc' } }
|
||||||
return { ...d, [k]: updated }
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateDraftValue(extractedValue, index, newValue) {
|
function sortRows(rows, ruleName, getCellFn) {
|
||||||
const k = valueKey(extractedValue)
|
const s = sortBy[ruleName]
|
||||||
setDrafts(d => {
|
if (!s) return rows
|
||||||
const current = d[k] || [{ key: '', value: '' }]
|
return [...rows].sort((a, b) => {
|
||||||
const updated = current.map((pair, i) => i === index ? { ...pair, value: newValue } : pair)
|
if (s.col === 'count') {
|
||||||
return { ...d, [k]: updated }
|
const av = a.record_count ?? 0
|
||||||
|
const bv = b.record_count ?? 0
|
||||||
|
return s.dir === 'asc' ? av - bv : bv - av
|
||||||
|
}
|
||||||
|
let av, bv
|
||||||
|
if (s.col === 'input_value') {
|
||||||
|
av = displayValue(a.extracted_value)
|
||||||
|
bv = displayValue(b.extracted_value)
|
||||||
|
} else {
|
||||||
|
av = String(getCellFn(a, s.col) ?? '')
|
||||||
|
bv = String(getCellFn(b, s.col) ?? '')
|
||||||
|
}
|
||||||
|
return s.dir === 'asc' ? av.localeCompare(bv) : bv.localeCompare(av)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function addDraftPair(extractedValue, outputField) {
|
function colsForRule(ruleName, outputField) {
|
||||||
|
const existing = existingColsByRule[ruleName] || [outputField].filter(Boolean)
|
||||||
|
const extra = extraCols[ruleName] || []
|
||||||
|
return [...existing, ...extra]
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCellValue(extractedValue, col) {
|
||||||
|
return drafts[valueKey(extractedValue)]?.[col] || ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function setCellValue(extractedValue, col, value) {
|
||||||
const k = valueKey(extractedValue)
|
const k = valueKey(extractedValue)
|
||||||
setDrafts(d => {
|
setDrafts(d => ({ ...d, [k]: { ...(d[k] || {}), [col]: value } }))
|
||||||
const current = d[k] || [{ key: outputField, value: '' }]
|
}
|
||||||
return { ...d, [k]: [...current, { key: '', value: '' }] }
|
|
||||||
|
function addCol(ruleName) {
|
||||||
|
setExtraCols(e => ({ ...e, [ruleName]: [...(e[ruleName] || []), ''] }))
|
||||||
|
}
|
||||||
|
|
||||||
|
function setExtraColName(ruleName, idx, name) {
|
||||||
|
setExtraCols(e => {
|
||||||
|
const cols = [...(e[ruleName] || [])]
|
||||||
|
cols[idx] = name
|
||||||
|
return { ...e, [ruleName]: cols }
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async function saveMapping(row) {
|
async function saveMapping(row, cols) {
|
||||||
const k = valueKey(row.extracted_value)
|
const k = valueKey(row.extracted_value)
|
||||||
const pairs = getDraft(row.extracted_value, row.output_field)
|
|
||||||
const output = Object.fromEntries(
|
const output = Object.fromEntries(
|
||||||
pairs.filter(p => p.key && p.value).map(p => [p.key, p.value])
|
cols.map(col => [col, getCellValue(row.extracted_value, col)])
|
||||||
|
.filter(([, v]) => v.trim())
|
||||||
)
|
)
|
||||||
if (Object.keys(output).length === 0) return
|
if (Object.keys(output).length === 0) return
|
||||||
|
|
||||||
@ -95,6 +157,7 @@ export default function Mappings({ source }) {
|
|||||||
output
|
output
|
||||||
})
|
})
|
||||||
setUnmapped(u => u.filter(x => valueKey(x.extracted_value) !== k))
|
setUnmapped(u => u.filter(x => valueKey(x.extracted_value) !== k))
|
||||||
|
setMapped(m => [...m, { rule_name: row.rule_name, input_value: row.extracted_value, output }])
|
||||||
setDrafts(d => { const n = { ...d }; delete n[k]; return n })
|
setDrafts(d => { const n = { ...d }; delete n[k]; return n })
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
alert(err.message)
|
alert(err.message)
|
||||||
@ -103,6 +166,58 @@ export default function Mappings({ source }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Save for the All tab — create if unmapped, update if already mapped
|
||||||
|
// Merges draft values over existing mapping values so unedited fields are preserved
|
||||||
|
async function saveAllRow(row, cols) {
|
||||||
|
const k = valueKey(row.extracted_value)
|
||||||
|
const output = Object.fromEntries(
|
||||||
|
cols.map(col => {
|
||||||
|
const drafted = drafts[k]?.[col]
|
||||||
|
const val = drafted !== undefined ? drafted : (row.is_mapped ? String(row.output?.[col] ?? '') : '')
|
||||||
|
return [col, val]
|
||||||
|
}).filter(([, v]) => v.trim())
|
||||||
|
)
|
||||||
|
if (Object.keys(output).length === 0) return
|
||||||
|
|
||||||
|
setSaving(s => ({ ...s, [k]: true }))
|
||||||
|
try {
|
||||||
|
if (row.is_mapped && row.mapping_id) {
|
||||||
|
const updated = await api.updateMapping(row.mapping_id, { output })
|
||||||
|
setAllValues(av => av.map(x =>
|
||||||
|
x.rule_name === row.rule_name && valueKey(x.extracted_value) === k
|
||||||
|
? { ...x, output: updated.output }
|
||||||
|
: x
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
await api.createMapping({
|
||||||
|
source_name: source,
|
||||||
|
rule_name: row.rule_name,
|
||||||
|
input_value: row.extracted_value,
|
||||||
|
output
|
||||||
|
})
|
||||||
|
setAllValues(av => av.map(x =>
|
||||||
|
x.rule_name === row.rule_name && valueKey(x.extracted_value) === k
|
||||||
|
? { ...x, is_mapped: true, output }
|
||||||
|
: x
|
||||||
|
))
|
||||||
|
setUnmapped(u => u.filter(x => valueKey(x.extracted_value) !== k))
|
||||||
|
}
|
||||||
|
setDrafts(d => { const n = { ...d }; delete n[k]; return n })
|
||||||
|
} catch (err) {
|
||||||
|
alert(err.message)
|
||||||
|
} finally {
|
||||||
|
setSaving(s => ({ ...s, [k]: false }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveAllDrafts(rows, cols) {
|
||||||
|
const dirty = rows.filter(row => {
|
||||||
|
const k = valueKey(row.extracted_value)
|
||||||
|
return drafts[k] && Object.keys(drafts[k]).length > 0
|
||||||
|
})
|
||||||
|
await Promise.all(dirty.map(row => saveAllRow(row, cols)))
|
||||||
|
}
|
||||||
|
|
||||||
async function handleImportCSV(e) {
|
async function handleImportCSV(e) {
|
||||||
const file = e.target.files[0]
|
const file = e.target.files[0]
|
||||||
if (!file) return
|
if (!file) return
|
||||||
@ -111,15 +226,13 @@ export default function Mappings({ source }) {
|
|||||||
try {
|
try {
|
||||||
const result = await api.importMappingsCSV(source, file)
|
const result = await api.importMappingsCSV(source, file)
|
||||||
alert(`Imported ${result.count} mapping${result.count !== 1 ? 's' : ''}.`)
|
alert(`Imported ${result.count} mapping${result.count !== 1 ? 's' : ''}.`)
|
||||||
// Refresh current tab
|
|
||||||
const rule = selectedRule || undefined
|
const rule = selectedRule || undefined
|
||||||
const [u, m] = await Promise.all([
|
const [u, m, a] = await Promise.all([api.getUnmapped(source, rule), api.getMappings(source, rule), api.getAllValues(source, rule)])
|
||||||
api.getUnmapped(source, rule),
|
|
||||||
tab === 'mapped' ? api.getMappings(source, rule) : Promise.resolve([])
|
|
||||||
])
|
|
||||||
setUnmapped(u)
|
setUnmapped(u)
|
||||||
setMapped(m)
|
setMapped(m)
|
||||||
|
setAllValues(a)
|
||||||
setDrafts({})
|
setDrafts({})
|
||||||
|
setExtraCols({})
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
alert(err.message)
|
alert(err.message)
|
||||||
} finally {
|
} finally {
|
||||||
@ -176,6 +289,13 @@ export default function Mappings({ source }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Group unmapped rows by rule
|
||||||
|
const unmappedByRule = {}
|
||||||
|
unmapped.forEach(row => {
|
||||||
|
if (!unmappedByRule[row.rule_name]) unmappedByRule[row.rule_name] = []
|
||||||
|
unmappedByRule[row.rule_name].push(row)
|
||||||
|
})
|
||||||
|
|
||||||
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
|
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@ -188,16 +308,16 @@ export default function Mappings({ source }) {
|
|||||||
download
|
download
|
||||||
className="text-sm px-3 py-1.5 border border-gray-200 rounded hover:bg-gray-50 text-gray-600"
|
className="text-sm px-3 py-1.5 border border-gray-200 rounded hover:bg-gray-50 text-gray-600"
|
||||||
>
|
>
|
||||||
Export CSV
|
Export TSV
|
||||||
</a>
|
</a>
|
||||||
<label className={`text-sm px-3 py-1.5 border border-gray-200 rounded cursor-pointer hover:bg-gray-50 text-gray-600 ${importing ? 'opacity-50 pointer-events-none' : ''}`}>
|
<label className={`text-sm px-3 py-1.5 border border-gray-200 rounded cursor-pointer hover:bg-gray-50 text-gray-600 ${importing ? 'opacity-50 pointer-events-none' : ''}`}>
|
||||||
{importing ? 'Importing…' : 'Import CSV'}
|
{importing ? 'Importing…' : 'Import TSV'}
|
||||||
<input type="file" accept=".tsv,.txt" className="hidden" onChange={handleImportCSV} />
|
<input type="file" accept=".tsv,.txt" className="hidden" onChange={handleImportCSV} />
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Rule filter */}
|
{/* Rule filter + tabs */}
|
||||||
<div className="flex items-center gap-3 mb-4">
|
<div className="flex items-center gap-3 mb-4">
|
||||||
<select
|
<select
|
||||||
className="text-sm border border-gray-200 rounded px-2 py-1.5 focus:outline-none focus:border-blue-400"
|
className="text-sm border border-gray-200 rounded px-2 py-1.5 focus:outline-none focus:border-blue-400"
|
||||||
@ -209,12 +329,12 @@ export default function Mappings({ source }) {
|
|||||||
</select>
|
</select>
|
||||||
|
|
||||||
<div className="flex bg-gray-100 rounded p-0.5">
|
<div className="flex bg-gray-100 rounded p-0.5">
|
||||||
{['unmapped', 'mapped'].map(t => (
|
{['unmapped', 'mapped', 'all'].map(t => (
|
||||||
<button key={t} onClick={() => setTab(t)}
|
<button key={t} onClick={() => setTab(t)}
|
||||||
className={`text-sm px-3 py-1 rounded transition-colors ${
|
className={`text-sm px-3 py-1 rounded transition-colors ${
|
||||||
tab === t ? 'bg-white text-gray-800 shadow-sm' : 'text-gray-500'
|
tab === t ? 'bg-white text-gray-800 shadow-sm' : 'text-gray-500'
|
||||||
}`}>
|
}`}>
|
||||||
{t === 'unmapped' ? `Unmapped${unmapped.length ? ` (${unmapped.length})` : ''}` : 'Mapped'}
|
{t === 'unmapped' ? `Unmapped${unmapped.length ? ` (${unmapped.length})` : ''}` : t === 'mapped' ? 'Mapped' : 'All'}
|
||||||
</button>
|
</button>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
@ -222,95 +342,312 @@ export default function Mappings({ source }) {
|
|||||||
|
|
||||||
{loading && <p className="text-sm text-gray-400">Loading…</p>}
|
{loading && <p className="text-sm text-gray-400">Loading…</p>}
|
||||||
|
|
||||||
{/* Unmapped tab */}
|
{/* Unmapped tab — spreadsheet layout */}
|
||||||
{!loading && tab === 'unmapped' && (
|
{!loading && tab === 'unmapped' && (
|
||||||
<>
|
<>
|
||||||
{unmapped.length === 0
|
{unmapped.length === 0
|
||||||
? <p className="text-sm text-gray-400">No unmapped values. Run a transform first, or all values are mapped.</p>
|
? <p className="text-sm text-gray-400">No unmapped values. Run a transform first, or all values are mapped.</p>
|
||||||
: (
|
: Object.entries(unmappedByRule).map(([ruleName, rows]) => {
|
||||||
<div className="space-y-2">
|
const cols = colsForRule(ruleName, rows[0]?.output_field)
|
||||||
{unmapped.map(row => {
|
const extra = extraCols[ruleName] || []
|
||||||
const k = valueKey(row.extracted_value)
|
const existingCount = cols.length - extra.length
|
||||||
const pairs = getDraft(row.extracted_value, row.output_field)
|
|
||||||
const isSaving = saving[k]
|
|
||||||
const sampleKey = `${row.rule_name}:${k}`
|
|
||||||
const samples = row.sample_records || []
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div key={`${row.rule_name}:${k}`}
|
<div key={ruleName} className="mb-6 overflow-x-auto">
|
||||||
className="bg-white border border-gray-200 rounded px-4 py-3">
|
{/* Datalists for column value suggestions */}
|
||||||
<div className="flex items-start gap-3">
|
{cols.map(col => (
|
||||||
{/* Left: value info */}
|
<datalist key={col} id={`dl-${ruleName}-${col}`}>
|
||||||
<div className="flex-1 min-w-0">
|
{[...(valuesByRuleCol[ruleName]?.[col] || [])].sort().map(v => (
|
||||||
<div className="flex items-baseline gap-2">
|
<option key={v} value={v} />
|
||||||
<span className="font-mono text-sm text-gray-800">{displayValue(row.extracted_value)}</span>
|
))}
|
||||||
<span className="text-xs text-gray-400">{row.record_count} records</span>
|
</datalist>
|
||||||
<span className="text-xs text-gray-300">· {row.rule_name}</span>
|
))}
|
||||||
</div>
|
<table className="w-full text-xs bg-white border border-gray-200 rounded">
|
||||||
{samples.length > 0 && (
|
<thead>
|
||||||
<button
|
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
|
||||||
className="text-xs text-blue-400 hover:text-blue-600 mt-0.5"
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={ruleName} col="rule" label="rule" />
|
||||||
onClick={() => setSampleOpen(s => ({ ...s, [sampleKey]: !s[sampleKey] }))}
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={ruleName} col="input_value" label="input_value" />
|
||||||
>
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={ruleName} col="count" label="count" className="text-right" />
|
||||||
{sampleOpen[sampleKey] ? 'hide samples' : 'show samples'}
|
{cols.slice(0, existingCount).map(col => (
|
||||||
</button>
|
<SortHeader sortBy={sortBy} onSort={toggleSort} key={col} ruleName={ruleName} col={col} label={col} />
|
||||||
)}
|
))}
|
||||||
{sampleOpen[sampleKey] && (
|
{extra.map((col, idx) => (
|
||||||
<div className="mt-2 text-xs bg-gray-50 rounded p-2 space-y-1">
|
<th key={`extra-${idx}`} className="px-3 py-2 font-medium">
|
||||||
{samples.slice(0, 3).map((s, i) => (
|
<input
|
||||||
<div key={i} className="font-mono text-gray-500 truncate">
|
className="border border-gray-200 rounded px-1 py-0.5 w-24 focus:outline-none focus:border-blue-400 font-normal"
|
||||||
{JSON.stringify(s)}
|
value={col}
|
||||||
</div>
|
placeholder="new key"
|
||||||
))}
|
onChange={e => setExtraColName(ruleName, idx, e.target.value)}
|
||||||
</div>
|
/>
|
||||||
)}
|
</th>
|
||||||
</div>
|
))}
|
||||||
|
<th className="px-2 py-2">
|
||||||
|
<button onClick={() => addCol(ruleName)} className="text-gray-300 hover:text-gray-500 font-normal" title="Add column">+</button>
|
||||||
|
</th>
|
||||||
|
<th className="px-3 py-2"></th>
|
||||||
|
<th className="px-3 py-2"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{sortRows(rows, ruleName, (row, col) => getCellValue(row.extracted_value, col)).map(row => {
|
||||||
|
const k = valueKey(row.extracted_value)
|
||||||
|
const isSaving = saving[k]
|
||||||
|
const sampleKey = `${ruleName}:${k}`
|
||||||
|
const samples = row.sample ? (Array.isArray(row.sample) ? row.sample : [row.sample]) : []
|
||||||
|
|
||||||
{/* Right: output fields */}
|
return (
|
||||||
<div className="flex items-center gap-2 flex-shrink-0">
|
<>
|
||||||
<div className="space-y-1">
|
<tr key={k} className="border-t border-gray-50 hover:bg-gray-50">
|
||||||
{pairs.map((pair, i) => (
|
<td className="px-3 py-1.5 text-gray-400">{ruleName}</td>
|
||||||
<div key={i} className="flex gap-1">
|
<td className="px-3 py-1.5 font-mono text-gray-800 whitespace-nowrap">{displayValue(row.extracted_value)}</td>
|
||||||
<input
|
<td className="px-3 py-1.5 text-right text-gray-400">{row.record_count}</td>
|
||||||
className="border border-gray-200 rounded px-2 py-1 text-xs w-24 focus:outline-none focus:border-blue-400"
|
{cols.map(col => (
|
||||||
value={pair.key}
|
<td key={col} className="px-3 py-1.5">
|
||||||
placeholder="key"
|
<input
|
||||||
onChange={e => updateDraftKey(row.extracted_value, i, e.target.value)}
|
list={`dl-${ruleName}-${col}`}
|
||||||
/>
|
className="border border-gray-200 rounded px-2 py-1 w-full min-w-24 focus:outline-none focus:border-blue-400"
|
||||||
<input
|
value={getCellValue(row.extracted_value, col)}
|
||||||
className="border border-gray-200 rounded px-2 py-1 text-xs w-32 focus:outline-none focus:border-blue-400"
|
onChange={e => setCellValue(row.extracted_value, col, e.target.value)}
|
||||||
value={pair.value}
|
onKeyDown={e => e.key === 'Enter' && saveMapping(row, cols)}
|
||||||
placeholder="value"
|
/>
|
||||||
onChange={e => updateDraftValue(row.extracted_value, i, e.target.value)}
|
</td>
|
||||||
onKeyDown={e => e.key === 'Enter' && saveMapping(row)}
|
))}
|
||||||
/>
|
{/* Empty cell under the + button */}
|
||||||
</div>
|
<td />
|
||||||
))}
|
<td className="px-3 py-1.5">
|
||||||
<button
|
{samples.length > 0 && (
|
||||||
className="text-xs text-gray-300 hover:text-gray-500"
|
<button
|
||||||
onClick={() => addDraftPair(row.extracted_value, row.output_field)}
|
className="text-blue-400 hover:text-blue-600 whitespace-nowrap"
|
||||||
>
|
onClick={() => setSampleOpen(s => ({ ...s, [sampleKey]: !s[sampleKey] }))}
|
||||||
+ field
|
>
|
||||||
</button>
|
{sampleOpen[sampleKey] ? 'hide' : 'show'}
|
||||||
</div>
|
</button>
|
||||||
<button
|
)}
|
||||||
onClick={() => saveMapping(row)}
|
</td>
|
||||||
disabled={isSaving}
|
<td className="px-3 py-1.5">
|
||||||
className="text-xs bg-blue-600 text-white px-2 py-1 rounded hover:bg-blue-700 disabled:opacity-50 self-start"
|
<button
|
||||||
>
|
onClick={() => saveMapping(row, cols)}
|
||||||
{isSaving ? '…' : 'Save'}
|
disabled={isSaving}
|
||||||
</button>
|
className="bg-blue-600 text-white px-2 py-1 rounded hover:bg-blue-700 disabled:opacity-50 whitespace-nowrap"
|
||||||
</div>
|
>
|
||||||
</div>
|
{isSaving ? '…' : 'Save'}
|
||||||
</div>
|
</button>
|
||||||
)
|
</td>
|
||||||
})}
|
</tr>
|
||||||
</div>
|
{sampleOpen[sampleKey] && (() => {
|
||||||
)
|
const sampleCols = [...new Set(samples.flatMap(r => Object.keys(r)))]
|
||||||
|
return (
|
||||||
|
<tr className="border-t border-gray-50 bg-gray-50">
|
||||||
|
<td colSpan={4 + cols.length + 3} className="px-3 py-2">
|
||||||
|
<table className="w-full text-xs border border-gray-100 rounded bg-white">
|
||||||
|
<thead>
|
||||||
|
<tr className="bg-gray-50 border-b border-gray-100">
|
||||||
|
{sampleCols.map(c => (
|
||||||
|
<th key={c} className="px-2 py-1 text-left font-medium text-gray-400 whitespace-nowrap">{c}</th>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{samples.map((rec, i) => (
|
||||||
|
<tr key={i} className="border-t border-gray-50">
|
||||||
|
{sampleCols.map(c => (
|
||||||
|
<td key={c} className="px-2 py-1 font-mono text-gray-600 whitespace-nowrap">{rec[c] != null ? String(rec[c]) : ''}</td>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)
|
||||||
|
})()}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* All tab — single SQL query, all extracted values with mapping status */}
|
||||||
|
{!loading && tab === 'all' && (() => {
|
||||||
|
// Derive columns and datalist suggestions from mapped rows in allValues
|
||||||
|
const allColsByRule = {}
|
||||||
|
const allValuesByRuleCol = {}
|
||||||
|
allValues.forEach(row => {
|
||||||
|
if (!row.is_mapped) return
|
||||||
|
if (!allColsByRule[row.rule_name]) allColsByRule[row.rule_name] = []
|
||||||
|
Object.entries(row.output || {}).forEach(([k, v]) => {
|
||||||
|
if (!allColsByRule[row.rule_name].includes(k))
|
||||||
|
allColsByRule[row.rule_name].push(k)
|
||||||
|
if (!allValuesByRuleCol[row.rule_name]) allValuesByRuleCol[row.rule_name] = {}
|
||||||
|
if (!allValuesByRuleCol[row.rule_name][k]) allValuesByRuleCol[row.rule_name][k] = new Set()
|
||||||
|
allValuesByRuleCol[row.rule_name][k].add(String(v))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
// Group rows by rule
|
||||||
|
const byRule = {}
|
||||||
|
allValues.forEach(row => {
|
||||||
|
if (!byRule[row.rule_name]) byRule[row.rule_name] = []
|
||||||
|
byRule[row.rule_name].push(row)
|
||||||
|
})
|
||||||
|
|
||||||
|
if (Object.keys(byRule).length === 0)
|
||||||
|
return <p className="text-sm text-gray-400">No extracted values. Run a transform first.</p>
|
||||||
|
|
||||||
|
return Object.entries(byRule).map(([ruleName, rows]) => {
|
||||||
|
const existing = allColsByRule[ruleName] || [rows[0]?.output_field].filter(Boolean)
|
||||||
|
const extra = extraCols[ruleName] || []
|
||||||
|
const cols = [...existing, ...extra]
|
||||||
|
const existingCount = cols.length - extra.length
|
||||||
|
const dirtyCount = rows.filter(row => {
|
||||||
|
const k = valueKey(row.extracted_value)
|
||||||
|
return drafts[k] && Object.keys(drafts[k]).length > 0
|
||||||
|
}).length
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div key={ruleName} className="mb-6 overflow-x-auto">
|
||||||
|
{dirtyCount > 0 && (
|
||||||
|
<div className="flex justify-end mb-1">
|
||||||
|
<button
|
||||||
|
onClick={() => saveAllDrafts(rows, cols)}
|
||||||
|
className="text-xs bg-blue-600 text-white px-3 py-1 rounded hover:bg-blue-700"
|
||||||
|
>
|
||||||
|
Save all ({dirtyCount})
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{cols.map(col => (
|
||||||
|
<datalist key={col} id={`dl-all-${ruleName}-${col}`}>
|
||||||
|
{[...(allValuesByRuleCol[ruleName]?.[col] || [])].sort().map(v => (
|
||||||
|
<option key={v} value={v} />
|
||||||
|
))}
|
||||||
|
</datalist>
|
||||||
|
))}
|
||||||
|
<table className="w-full text-xs bg-white border border-gray-200 rounded">
|
||||||
|
<thead>
|
||||||
|
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
|
||||||
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={`all:${ruleName}`} col="rule" label="rule" />
|
||||||
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={`all:${ruleName}`} col="input_value" label="input_value" />
|
||||||
|
<SortHeader sortBy={sortBy} onSort={toggleSort} ruleName={`all:${ruleName}`} col="count" label="count" className="text-right" />
|
||||||
|
{cols.slice(0, existingCount).map(col => (
|
||||||
|
<SortHeader sortBy={sortBy} onSort={toggleSort} key={col} ruleName={`all:${ruleName}`} col={col} label={col} />
|
||||||
|
))}
|
||||||
|
{extra.map((col, idx) => (
|
||||||
|
<th key={`extra-${idx}`} className="px-3 py-2 font-medium">
|
||||||
|
<input
|
||||||
|
className="border border-gray-200 rounded px-1 py-0.5 w-24 focus:outline-none focus:border-blue-400 font-normal"
|
||||||
|
value={col}
|
||||||
|
placeholder="new key"
|
||||||
|
onChange={e => setExtraColName(ruleName, idx, e.target.value)}
|
||||||
|
/>
|
||||||
|
</th>
|
||||||
|
))}
|
||||||
|
<th className="px-2 py-2">
|
||||||
|
<button onClick={() => addCol(ruleName)} className="text-gray-300 hover:text-gray-500" title="Add column">+</button>
|
||||||
|
</th>
|
||||||
|
<th className="px-3 py-2"></th>
|
||||||
|
<th className="px-3 py-2"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{sortRows(rows, `all:${ruleName}`, (row, col) =>
|
||||||
|
row.is_mapped ? String(row.output?.[col] ?? '') : ''
|
||||||
|
).map(row => {
|
||||||
|
const k = valueKey(row.extracted_value)
|
||||||
|
const isSaving = saving[k]
|
||||||
|
const sampleKey = `all:${ruleName}:${k}`
|
||||||
|
const samples = row.sample ? (Array.isArray(row.sample) ? row.sample : [row.sample]) : []
|
||||||
|
|
||||||
|
const cellVal = (col) => {
|
||||||
|
const drafted = drafts[k]?.[col]
|
||||||
|
if (drafted !== undefined) return drafted
|
||||||
|
return row.is_mapped ? String(row.output?.[col] ?? '') : ''
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasDraft = !!(drafts[k] && Object.keys(drafts[k]).length > 0)
|
||||||
|
const rowBg = hasDraft ? 'bg-blue-50' : row.is_mapped ? '' : 'bg-yellow-50'
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<tr key={k} className={`border-t border-gray-50 hover:bg-gray-50 ${rowBg}`}>
|
||||||
|
<td className="px-3 py-1.5 text-gray-400">{ruleName}</td>
|
||||||
|
<td className="px-3 py-1.5 font-mono text-gray-800 whitespace-nowrap">{displayValue(row.extracted_value)}</td>
|
||||||
|
<td className="px-3 py-1.5 text-right text-gray-400">{row.record_count}</td>
|
||||||
|
{cols.map(col => (
|
||||||
|
<td key={col} className="px-3 py-1.5">
|
||||||
|
<input
|
||||||
|
list={`dl-all-${ruleName}-${col}`}
|
||||||
|
className={`border rounded px-2 py-1 w-full min-w-24 focus:outline-none focus:border-blue-400 ${hasDraft ? 'border-blue-300' : row.is_mapped ? 'border-gray-200' : 'border-yellow-300'}`}
|
||||||
|
value={cellVal(col)}
|
||||||
|
onChange={e => setCellValue(row.extracted_value, col, e.target.value)}
|
||||||
|
onKeyDown={e => e.key === 'Enter' && saveAllRow(row, cols)}
|
||||||
|
/>
|
||||||
|
</td>
|
||||||
|
))}
|
||||||
|
<td />
|
||||||
|
<td className="px-3 py-1.5">
|
||||||
|
{samples.length > 0 && (
|
||||||
|
<button
|
||||||
|
className="text-blue-400 hover:text-blue-600 whitespace-nowrap"
|
||||||
|
onClick={() => setSampleOpen(s => ({ ...s, [sampleKey]: !s[sampleKey] }))}
|
||||||
|
>
|
||||||
|
{sampleOpen[sampleKey] ? 'hide' : 'show'}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td className="px-3 py-1.5">
|
||||||
|
<button
|
||||||
|
onClick={() => saveAllRow(row, cols)}
|
||||||
|
disabled={isSaving}
|
||||||
|
className="bg-blue-600 text-white px-2 py-1 rounded hover:bg-blue-700 disabled:opacity-50 whitespace-nowrap"
|
||||||
|
>
|
||||||
|
{isSaving ? '…' : 'Save'}
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{sampleOpen[sampleKey] && (() => {
|
||||||
|
const sampleCols = [...new Set(samples.flatMap(r => Object.keys(r)))]
|
||||||
|
return (
|
||||||
|
<tr className="border-t border-gray-50 bg-gray-50">
|
||||||
|
<td colSpan={3 + cols.length + 3} className="px-3 py-2">
|
||||||
|
<table className="w-full text-xs border border-gray-100 rounded bg-white">
|
||||||
|
<thead>
|
||||||
|
<tr className="bg-gray-50 border-b border-gray-100">
|
||||||
|
{sampleCols.map(c => (
|
||||||
|
<th key={c} className="px-2 py-1 text-left font-medium text-gray-400 whitespace-nowrap">{c}</th>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{samples.map((rec, i) => (
|
||||||
|
<tr key={i} className="border-t border-gray-50">
|
||||||
|
{sampleCols.map(c => (
|
||||||
|
<td key={c} className="px-2 py-1 font-mono text-gray-600 whitespace-nowrap">{rec[c] != null ? String(rec[c]) : ''}</td>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)
|
||||||
|
})()}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})()}
|
||||||
|
|
||||||
{/* Mapped tab */}
|
{/* Mapped tab */}
|
||||||
{!loading && tab === 'mapped' && (
|
{!loading && tab === 'mapped' && (
|
||||||
<>
|
<>
|
||||||
@ -338,36 +675,28 @@ export default function Mappings({ source }) {
|
|||||||
<div key={i} className="flex gap-1">
|
<div key={i} className="flex gap-1">
|
||||||
<input
|
<input
|
||||||
className="border border-gray-200 rounded px-2 py-1 text-xs w-24 focus:outline-none focus:border-blue-400"
|
className="border border-gray-200 rounded px-2 py-1 text-xs w-24 focus:outline-none focus:border-blue-400"
|
||||||
value={pair.key}
|
value={pair.key} placeholder="key"
|
||||||
placeholder="key"
|
|
||||||
onChange={e => updateEditKey(m.id, i, e.target.value)}
|
onChange={e => updateEditKey(m.id, i, e.target.value)}
|
||||||
/>
|
/>
|
||||||
<input
|
<input
|
||||||
className="border border-gray-200 rounded px-2 py-1 text-xs w-32 focus:outline-none focus:border-blue-400"
|
className="border border-gray-200 rounded px-2 py-1 text-xs w-32 focus:outline-none focus:border-blue-400"
|
||||||
value={pair.value}
|
value={pair.value} placeholder="value"
|
||||||
placeholder="value"
|
|
||||||
onChange={e => updateEditValue(m.id, i, e.target.value)}
|
onChange={e => updateEditValue(m.id, i, e.target.value)}
|
||||||
onKeyDown={e => e.key === 'Enter' && saveEdit(m)}
|
onKeyDown={e => e.key === 'Enter' && saveEdit(m)}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
<button
|
<button className="text-xs text-gray-300 hover:text-gray-500"
|
||||||
className="text-xs text-gray-300 hover:text-gray-500"
|
onClick={() => addEditPair(m.id)}>+ field</button>
|
||||||
onClick={() => addEditPair(m.id)}
|
|
||||||
>+ field</button>
|
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td className="px-4 py-2">
|
<td className="px-4 py-2">
|
||||||
<div className="flex gap-2">
|
<div className="flex gap-2">
|
||||||
<button
|
<button onClick={() => saveEdit(m)} disabled={saving[m.id]}
|
||||||
onClick={() => saveEdit(m)}
|
|
||||||
disabled={saving[m.id]}
|
|
||||||
className="text-xs bg-blue-600 text-white px-2 py-1 rounded hover:bg-blue-700 disabled:opacity-50"
|
className="text-xs bg-blue-600 text-white px-2 py-1 rounded hover:bg-blue-700 disabled:opacity-50"
|
||||||
>{saving[m.id] ? '…' : 'Save'}</button>
|
>{saving[m.id] ? '…' : 'Save'}</button>
|
||||||
<button
|
<button onClick={() => setEditingId(null)}
|
||||||
onClick={() => setEditingId(null)}
|
className="text-xs text-gray-400 hover:text-gray-600">Cancel</button>
|
||||||
className="text-xs text-gray-400 hover:text-gray-600"
|
|
||||||
>Cancel</button>
|
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
@ -375,9 +704,7 @@ export default function Mappings({ source }) {
|
|||||||
<tr key={m.id} className="border-t border-gray-50 hover:bg-gray-50">
|
<tr key={m.id} className="border-t border-gray-50 hover:bg-gray-50">
|
||||||
<td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td>
|
<td className="px-4 py-2 text-xs text-gray-400">{m.rule_name}</td>
|
||||||
<td className="px-4 py-2 font-mono text-gray-700">{displayValue(m.input_value)}</td>
|
<td className="px-4 py-2 font-mono text-gray-700">{displayValue(m.input_value)}</td>
|
||||||
<td className="px-4 py-2 font-mono text-xs text-gray-500">
|
<td className="px-4 py-2 font-mono text-xs text-gray-500">{JSON.stringify(m.output)}</td>
|
||||||
{JSON.stringify(m.output)}
|
|
||||||
</td>
|
|
||||||
<td className="px-4 py-2">
|
<td className="px-4 py-2">
|
||||||
<div className="flex gap-2">
|
<div className="flex gap-2">
|
||||||
<button onClick={() => startEdit(m)}
|
<button onClick={() => startEdit(m)}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user