pt/dataflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rule chaining: later sequence rules can read earlier outputs

Browse Source 
apply_transformations now processes rules in sequence order using a
temp table accumulator. Each sequence group reads from data merged
with all prior groups' outputs, so a rule at seq N can reference a
field written by a rule at seq < N.

preview_rule falls back to transformed when a field isn't in raw
data, so chained rules preview correctly in the UI.

Rules form field dropdown gains an "from earlier rules" optgroup
listing output fields from lower-sequence rules.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-04-16 23:04:14 -04:00
parent fd67bb03af
commit fce427ba95
3 changed files with 151 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
database/functions.sql
View File

@ -187,46 +187,58 @@ CREATE OR REPLACE FUNCTION apply_transformations(
p_record_ids INTEGER[] DEFAULT NULL, -- NULL = all eligible records p_record_ids INTEGER[] DEFAULT NULL, -- NULL = all eligible records
p_overwrite BOOLEAN DEFAULT FALSE -- FALSE = skip already-transformed, TRUE = overwrite all p_overwrite BOOLEAN DEFAULT FALSE -- FALSE = skip already-transformed, TRUE = overwrite all
) RETURNS JSON AS $$ ) RETURNS JSON AS $$
WITH DECLARE
-- All records to process v_seq INT;
qualifying AS ( v_count INT := 0;
SELECT id, data BEGIN
-- Accumulator: one row per qualifying record, additions built up across sequence steps.
-- Each sequence step reads from data || additions so later rules can reference earlier outputs.
CREATE TEMP TABLE _xform_acc ON COMMIT DROP AS
SELECT id, data, '{}'::jsonb AS additions
FROM dataflow.records FROM dataflow.records
WHERE source_name = p_source_name WHERE source_name = p_source_name
AND (p_overwrite OR transformed IS NULL) AND (p_overwrite OR transformed IS NULL)
AND (p_record_ids IS NULL OR id = ANY(p_record_ids)) AND (p_record_ids IS NULL OR id = ANY(p_record_ids));
),
-- Mirror TPS rx: fan out one row per regex match, drive from rules → records -- Process one sequence group at a time, in order.
rx AS ( -- Rules at sequence N can read fields written by rules at sequence < N.
FOR v_seq IN
SELECT DISTINCT sequence
FROM dataflow.rules
WHERE source_name = p_source_name AND enabled = true
ORDER BY sequence
LOOP
WITH
-- Current view of each record: original data merged with accumulated outputs so far
current AS (
SELECT id, data || additions AS current_data
FROM _xform_acc
),
-- Fan out one row per regex match for rules at this sequence level
rx AS (
SELECT SELECT
q.id, c.id,
r.name AS rule_name, r.name AS rule_name,
r.sequence, r.sequence,
r.output_field, r.output_field,
r.retain, r.retain,
r.function_type, r.function_type,
COALESCE(mt.rn, rp.rn, 1) AS result_number, COALESCE(mt.rn, rp.rn, 1) AS result_number,
-- extract: build map_val and retain_val per match (mirrors TPS)
CASE WHEN array_length(mt.mt, 1) = 1 THEN to_jsonb(mt.mt[1]) ELSE to_jsonb(mt.mt) END AS match_val, CASE WHEN array_length(mt.mt, 1) = 1 THEN to_jsonb(mt.mt[1]) ELSE to_jsonb(mt.mt) END AS match_val,
to_jsonb(rp.rp) AS replace_val to_jsonb(rp.rp) AS replace_val
FROM dataflow.rules r FROM dataflow.rules r
INNER JOIN qualifying q ON q.data ? r.field INNER JOIN current c ON (c.current_data ? r.field)
LEFT JOIN LATERAL regexp_matches(q.data ->> r.field, r.pattern, r.flags) LEFT JOIN LATERAL regexp_matches(c.current_data ->> r.field, r.pattern, r.flags)
WITH ORDINALITY AS mt(mt, rn) ON r.function_type = 'extract' WITH ORDINALITY AS mt(mt, rn) ON r.function_type = 'extract'
LEFT JOIN LATERAL regexp_replace(q.data ->> r.field, r.pattern, r.replace_value, r.flags) LEFT JOIN LATERAL regexp_replace(c.current_data ->> r.field, r.pattern, r.replace_value, r.flags)
WITH ORDINALITY AS rp(rp, rn) ON r.function_type = 'replace' WITH ORDINALITY AS rp(rp, rn) ON r.function_type = 'replace'
WHERE r.source_name = p_source_name WHERE r.source_name = p_source_name
AND r.sequence = v_seq
AND r.enabled = true AND r.enabled = true
), ),
-- Aggregate match rows back into one value per (record, rule) — mirrors TPS agg_to_target_items agg_matches AS (
agg_matches AS (
SELECT SELECT
id, id, rule_name, sequence, output_field, retain, function_type,
rule_name,
sequence,
output_field,
retain,
function_type,
CASE function_type CASE function_type
WHEN 'replace' THEN jsonb_agg(replace_val) -> 0 WHEN 'replace' THEN jsonb_agg(replace_val) -> 0
ELSE ELSE
@ -237,15 +249,10 @@ agg_matches AS (
END AS extracted END AS extracted
FROM rx FROM rx
GROUP BY id, rule_name, sequence, output_field, retain, function_type GROUP BY id, rule_name, sequence, output_field, retain, function_type
), ),
-- Join with mappings to find mapped output — mirrors TPS link_map linked AS (
linked AS (
SELECT SELECT
a.id, a.id, a.sequence, a.output_field, a.retain, a.extracted,
a.sequence,
a.output_field,
a.retain,
a.extracted,
m.output AS mapped m.output AS mapped
FROM agg_matches a FROM agg_matches a
LEFT JOIN dataflow.mappings m ON LEFT JOIN dataflow.mappings m ON
@ -253,49 +260,44 @@ linked AS (
AND m.rule_name = a.rule_name AND m.rule_name = a.rule_name
AND m.input_value = a.extracted AND m.input_value = a.extracted
WHERE a.extracted IS NOT NULL WHERE a.extracted IS NOT NULL
), ),
-- Build per-rule output JSONB: rule_output AS (
-- mapped → use mapping output; also write output_field if retain = true SELECT id, sequence,
-- no map → write extracted value to output_field
rule_output AS (
SELECT
id,
sequence,
CASE CASE
WHEN mapped IS NOT NULL THEN WHEN mapped IS NOT NULL THEN
mapped || mapped || CASE WHEN retain THEN jsonb_build_object(output_field, extracted) ELSE '{}'::jsonb END
CASE WHEN retain
THEN jsonb_build_object(output_field, extracted)
ELSE '{}'::jsonb
END
ELSE ELSE
jsonb_build_object(output_field, extracted) jsonb_build_object(output_field, extracted)
END AS output END AS output
FROM linked FROM linked
), ),
-- Merge all rule outputs per record in sequence order — mirrors TPS agg_to_id seq_additions AS (
record_additions AS ( SELECT id, dataflow.jsonb_concat_obj(output ORDER BY sequence) AS additions
SELECT
id,
dataflow.jsonb_concat_obj(output ORDER BY sequence) AS additions
FROM rule_output FROM rule_output
GROUP BY id GROUP BY id
), )
-- Update all qualifying records; records with no rule matches get transformed = data UPDATE _xform_acc acc
updated AS ( SET additions = additions || COALESCE(sa.additions, '{}'::jsonb)
UPDATE dataflow.records rec FROM seq_additions sa
SET transformed = rec.data || COALESCE(ra.additions, '{}'::jsonb) || COALESCE(rec.overrides, '{}'::jsonb), WHERE acc.id = sa.id;
transformed_at = CURRENT_TIMESTAMP END LOOP;
FROM qualifying q
LEFT JOIN record_additions ra ON ra.id = q.id
WHERE rec.id = q.id
RETURNING rec.id
)
SELECT json_build_object('success', true, 'transformed', count(*))
FROM updated
$$ LANGUAGE sql;
COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and mappings to records (set-based CTE)'; -- Write final result: original data + all accumulated rule outputs + any manual overrides
WITH updated AS (
UPDATE dataflow.records rec
SET transformed = rec.data || acc.additions || COALESCE(rec.overrides, '{}'::jsonb),
transformed_at = CURRENT_TIMESTAMP
FROM _xform_acc acc
WHERE rec.id = acc.id
RETURNING rec.id
)
SELECT count(*) INTO v_count FROM updated;
RETURN json_build_object('success', true, 'transformed', v_count);
END;
$$ LANGUAGE plpgsql;
COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and mappings to records. Rules are processed in sequence order — a rule at sequence N can read fields written by rules at sequence < N (chaining).';
------------------------------------------------------ ------------------------------------------------------
-- Function: get_all_values -- Function: get_all_values

22
database/queries/rules.sql
View File

@ -86,21 +86,27 @@ CREATE OR REPLACE FUNCTION preview_rule(
p_limit INT DEFAULT 20 p_limit INT DEFAULT 20
) )
RETURNS TABLE (id INT, raw_value TEXT, extracted_value JSONB) AS $$ RETURNS TABLE (id INT, raw_value TEXT, extracted_value JSONB) AS $$
-- Field is resolved from data first, then transformed (supports chained rules whose
-- input field was produced by an earlier-sequence rule rather than the raw import).
BEGIN BEGIN
IF p_function_type = 'replace' THEN IF p_function_type = 'replace' THEN
RETURN QUERY RETURN QUERY
SELECT SELECT
r.id, r.id,
r.data ->> p_field, COALESCE(r.data ->> p_field, r.transformed ->> p_field),
to_jsonb(regexp_replace(r.data ->> p_field, p_pattern, p_replace_value, p_flags)) to_jsonb(regexp_replace(
COALESCE(r.data ->> p_field, r.transformed ->> p_field),
p_pattern, p_replace_value, p_flags
))
FROM dataflow.records r FROM dataflow.records r
WHERE source_name = p_source AND data ? p_field WHERE source_name = p_source
AND (data ? p_field OR transformed ? p_field)
ORDER BY r.id DESC LIMIT p_limit; ORDER BY r.id DESC LIMIT p_limit;
ELSE ELSE
RETURN QUERY RETURN QUERY
SELECT SELECT
r.id, r.id,
r.data ->> p_field, COALESCE(r.data ->> p_field, r.transformed ->> p_field),
CASE CASE
WHEN agg.match_count = 0 THEN NULL WHEN agg.match_count = 0 THEN NULL
WHEN agg.match_count = 1 THEN agg.matches -> 0 WHEN agg.match_count = 1 THEN agg.matches -> 0
@ -114,10 +120,14 @@ BEGIN
ORDER BY rn ORDER BY rn
) AS matches, ) AS matches,
count(*)::int AS match_count count(*)::int AS match_count
FROM regexp_matches(r.data ->> p_field, p_pattern, p_flags) FROM regexp_matches(
COALESCE(r.data ->> p_field, r.transformed ->> p_field),
p_pattern, p_flags
)
WITH ORDINALITY AS m(mt, rn) WITH ORDINALITY AS m(mt, rn)
) agg ) agg
WHERE r.source_name = p_source AND r.data ? p_field WHERE r.source_name = p_source
AND (r.data ? p_field OR r.transformed ? p_field)
ORDER BY r.id DESC LIMIT p_limit; ORDER BY r.id DESC LIMIT p_limit;
END IF; END IF;
END; END;

25
ui/src/pages/Rules.jsx
View File

@ -42,7 +42,7 @@ function PreviewModal({ rows, onClose }) {
) )
} }
function FormPanel({ form, setForm, editing, error, loading, fields, source, onSubmit, onCancel }) { function FormPanel({ form, setForm, editing, error, loading, fields, rules, source, onSubmit, onCancel }) {
const [preview, setPreview] = useState([]) const [preview, setPreview] = useState([])
const [previewing, setPreviewing] = useState(false) const [previewing, setPreviewing] = useState(false)
const [modalOpen, setModalOpen] = useState(false) const [modalOpen, setModalOpen] = useState(false)
@ -91,15 +91,28 @@ function FormPanel({ form, setForm, editing, error, loading, fields, source, onS
<div className="grid grid-cols-2 gap-3"> <div className="grid grid-cols-2 gap-3">
<div> <div>
<label className="text-xs text-gray-500 block mb-1">Input field</label> <label className="text-xs text-gray-500 block mb-1">Input field</label>
{fields.length > 0 ? ( {fields.length > 0 ? (() => {
// Output fields from rules at a lower sequence — available as chained inputs
const chainedFields = [...new Set(
(rules || [])
.filter(r => r.sequence < form.sequence && r.output_field && (!editing || r.id !== editing))
.map(r => r.output_field)
)].filter(f => !fields.includes(f))
return (
<select <select
className="w-full border border-gray-200 rounded px-3 py-1.5 text-sm focus:outline-none focus:border-blue-400" className="w-full border border-gray-200 rounded px-3 py-1.5 text-sm focus:outline-none focus:border-blue-400"
value={form.field} onChange={e => setForm(f => ({ ...f, field: e.target.value }))} value={form.field} onChange={e => setForm(f => ({ ...f, field: e.target.value }))}
> >
<option value="">— select field —</option> <option value="">— select field —</option>
{fields.map(f => <option key={f} value={f}>{f}</option>)} {fields.map(f => <option key={f} value={f}>{f}</option>)}
{chainedFields.length > 0 && (
<optgroup label="from earlier rules">
{chainedFields.map(f => <option key={f} value={f}>{f}</option>)}
</optgroup>
)}
</select> </select>
) : ( )
})() : (
<input <input
className="w-full border border-gray-200 rounded px-3 py-1.5 text-sm focus:outline-none focus:border-blue-400" className="w-full border border-gray-200 rounded px-3 py-1.5 text-sm focus:outline-none focus:border-blue-400"
value={form.field} onChange={e => setForm(f => ({ ...f, field: e.target.value }))} value={form.field} onChange={e => setForm(f => ({ ...f, field: e.target.value }))}
@ -322,7 +335,7 @@ export default function Rules({ source }) {
{creating && ( {creating && (
<FormPanel <FormPanel
form={form} setForm={setForm} editing={false} form={form} setForm={setForm} editing={false}
error={error} loading={loading} fields={fields} source={source} error={error} loading={loading} fields={fields} rules={rules} source={source}
onSubmit={handleSubmit} onSubmit={handleSubmit}
onCancel={() => { setCreating(false); setError('') }} onCancel={() => { setCreating(false); setError('') }}
/> />
@ -377,8 +390,8 @@ export default function Rules({ source }) {
</div> </div>
<div className="px-4 pb-4"> <div className="px-4 pb-4">
<FormPanel <FormPanel
form={form} setForm={setForm} editing={true} form={form} setForm={setForm} editing={rule.id}
error={error} loading={loading} fields={fields} source={source} error={error} loading={loading} fields={fields} rules={rules} source={source}
onSubmit={e => handleSubmit(e, rule.id)} onSubmit={e => handleSubmit(e, rule.id)}
onCancel={() => { setEditing(null); setExpanded(null) }} onCancel={() => { setEditing(null); setExpanded(null) }}
/> />