pt/dataflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add retain flag to rules for preserving extracted values alongside mappings

Browse Source 
Mirrors TPS's retain: y behaviour — when a mapping is applied, the extracted
value is also written to output_field so both the raw extraction and the
mapped result are available in transformed data.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-04-04 20:48:52 -04:00
parent 3be5ccc435
commit f59908aaa3
5 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
api/routes/rules.js
View File

@ -145,7 +145,7 @@ module.exports = (pool) => {
// Create rule
router.post('/', async (req, res, next) => {
try {
const { source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
const { source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence } = req.body;
if (!source_name || !name || !field || !pattern || !output_field) {
return res.status(400).json({
@ -158,10 +158,10 @@ module.exports = (pool) => {
}
const result = await pool.query(
`INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
`INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
RETURNING *`,
[source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', replace_value || '', enabled !== false, sequence || 0]
[source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', replace_value || '', enabled !== false, retain === true, sequence || 0]
);
res.status(201).json(result.rows[0]);
@ -179,7 +179,7 @@ module.exports = (pool) => {
// Update rule
router.put('/:id', async (req, res, next) => {
try {
const { name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
const { name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence } = req.body;
if (function_type && !['extract', 'replace'].includes(function_type)) {
return res.status(400).json({ error: 'function_type must be "extract" or "replace"' });
@ -195,10 +195,11 @@ module.exports = (pool) => {
flags = COALESCE($7, flags),
replace_value = COALESCE($8, replace_value),
enabled = COALESCE($9, enabled),
sequence = COALESCE($10, sequence)
retain = COALESCE($10, retain),
sequence = COALESCE($11, sequence)
WHERE id = $1
RETURNING *`,
[req.params.id, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence]
[req.params.id, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence]
);
if (result.rows.length === 0) {

4
database/functions.sql
View File

@ -149,6 +149,10 @@ BEGIN
IF v_mapping IS NOT NULL THEN
-- Apply mapping (merge mapped fields into result)
v_transformed := v_transformed || v_mapping;
-- If retain is set, also write the extracted value to output_field
IF v_rule.retain THEN
v_transformed := jsonb_set(v_transformed, ARRAY[v_rule.output_field], v_extracted);
END IF;
ELSE
-- No mapping, store extracted value (scalar or array)
v_transformed := jsonb_set(

5
database/migrate_tps.sql
View File

@ -51,7 +51,7 @@ FROM dataflow.sources ORDER BY name;
\echo '=== 2. Rules ==='
INSERT INTO dataflow.rules
(source_name, name, field, pattern, output_field, function_type, flags, replace_value, sequence, enabled)
(source_name, name, field, pattern, output_field, function_type, flags, replace_value, sequence, enabled, retain)
SELECT
srce AS source_name,
target AS name,
@ -63,7 +63,8 @@ SELECT
COALESCE(regex->'regex'->'defn'->0->>'flag', '') AS flags,
'' AS replace_value,
seq AS sequence,
true AS enabled
true AS enabled,
(regex->'regex'->'defn'->0->>'retain') = 'y' AS retain
FROM dblink(:'tps_conn',
'SELECT srce, target, seq, regex FROM tps.map_rm'
) AS t(srce TEXT, target TEXT, seq INT, regex JSONB)

1
database/schema.sql
View File

@ -76,6 +76,7 @@ CREATE TABLE rules (
-- Options
enabled BOOLEAN DEFAULT true,
retain BOOLEAN DEFAULT false, -- Write output_field even when a mapping is applied
sequence INTEGER DEFAULT 0, -- Execution order
-- Metadata

13
ui/src/pages/Rules.jsx
View File

@ -1,7 +1,7 @@
import { useState, useEffect, useRef } from 'react'
import { api } from '../api'
const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', replace_value: '', sequence: 0 }
const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', replace_value: '', retain: false, sequence: 0 }
function PreviewModal({ rows, onClose }) {
const matched = rows.filter(r => r.extracted_value != null).length
@ -144,6 +144,16 @@ function FormPanel({ form, setForm, editing, error, loading, fields, source, onS
/>
</div>
</div>
{form.function_type === 'extract' && (
<label className="flex items-center gap-2 text-xs text-gray-600 cursor-pointer select-none">
<input
type="checkbox"
checked={!!form.retain}
onChange={e => setForm(f => ({ ...f, retain: e.target.checked }))}
/>
Retain extracted value in output field even when a mapping is applied
</label>
)}
{form.function_type === 'replace' && (
<div>
<label className="text-xs text-gray-500 block mb-1">Replacement string</label>
@ -237,6 +247,7 @@ export default function Rules({ source }) {
function_type: rule.function_type || 'extract',
flags: rule.flags || '',
replace_value: rule.replace_value || '',
retain: rule.retain || false,
sequence: rule.sequence,
})
setEditing(rule.id)