Add retain flag to rules for preserving extracted values alongside mappings

Mirrors TPS's retain: y behaviour — when a mapping is applied, the extracted
value is also written to output_field so both the raw extraction and the
mapped result are available in transformed data.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/routes/rules.js

@@ -145,7 +145,7 @@ module.exports = (pool) => {
     // Create rule
     router.post('/', async (req, res, next) => {
         try {
-            const { source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
+            const { source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence } = req.body;
 
             if (!source_name || !name || !field || !pattern || !output_field) {
                 return res.status(400).json({
@@ -158,10 +158,10 @@ module.exports = (pool) => {
             }
 
             const result = await pool.query(
-                `INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence)
+                `INSERT INTO rules (source_name, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence)
-                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
                  RETURNING *`,
-                [source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', replace_value || '', enabled !== false, sequence || 0]
+                [source_name, name, field, pattern, output_field, function_type || 'extract', flags || '', replace_value || '', enabled !== false, retain === true, sequence || 0]
             );
 
             res.status(201).json(result.rows[0]);
@@ -179,7 +179,7 @@ module.exports = (pool) => {
     // Update rule
     router.put('/:id', async (req, res, next) => {
         try {
-            const { name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence } = req.body;
+            const { name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence } = req.body;
 
             if (function_type && !['extract', 'replace'].includes(function_type)) {
                 return res.status(400).json({ error: 'function_type must be "extract" or "replace"' });
@@ -195,10 +195,11 @@ module.exports = (pool) => {
                      flags = COALESCE($7, flags),
                      replace_value = COALESCE($8, replace_value),
                      enabled = COALESCE($9, enabled),
-                     sequence = COALESCE($10, sequence)
+                     retain = COALESCE($10, retain),
+                     sequence = COALESCE($11, sequence)
                  WHERE id = $1
                  RETURNING *`,
-                [req.params.id, name, field, pattern, output_field, function_type, flags, replace_value, enabled, sequence]
+                [req.params.id, name, field, pattern, output_field, function_type, flags, replace_value, enabled, retain, sequence]
             );
 
             if (result.rows.length === 0) {

database/functions.sql

@@ -149,6 +149,10 @@ BEGIN
                     IF v_mapping IS NOT NULL THEN
                         -- Apply mapping (merge mapped fields into result)
                         v_transformed := v_transformed || v_mapping;
+                        -- If retain is set, also write the extracted value to output_field
+                        IF v_rule.retain THEN
+                            v_transformed := jsonb_set(v_transformed, ARRAY[v_rule.output_field], v_extracted);
+                        END IF;
                     ELSE
                         -- No mapping, store extracted value (scalar or array)
                         v_transformed := jsonb_set(

database/migrate_tps.sql

@@ -51,7 +51,7 @@ FROM dataflow.sources ORDER BY name;
 \echo '=== 2. Rules ==='
 
 INSERT INTO dataflow.rules
-    (source_name, name, field, pattern, output_field, function_type, flags, replace_value, sequence, enabled)
+    (source_name, name, field, pattern, output_field, function_type, flags, replace_value, sequence, enabled, retain)
 SELECT
     srce                                                                AS source_name,
     target                                                              AS name,
@@ -63,7 +63,8 @@ SELECT
     COALESCE(regex->'regex'->'defn'->0->>'flag', '')                   AS flags,
     ''                                                                  AS replace_value,
     seq                                                                 AS sequence,
-    true                                                                AS enabled
+    true                                                                AS enabled,
+    (regex->'regex'->'defn'->0->>'retain') = 'y'                       AS retain
 FROM dblink(:'tps_conn',
     'SELECT srce, target, seq, regex FROM tps.map_rm'
 ) AS t(srce TEXT, target TEXT, seq INT, regex JSONB)

database/schema.sql

@@ -76,6 +76,7 @@ CREATE TABLE rules (
 
     -- Options
     enabled BOOLEAN DEFAULT true,
+    retain BOOLEAN DEFAULT false,           -- Write output_field even when a mapping is applied
     sequence INTEGER DEFAULT 0,             -- Execution order
 
     -- Metadata

ui/src/pages/Rules.jsx

@@ -1,7 +1,7 @@
 import { useState, useEffect, useRef } from 'react'
 import { api } from '../api'
 
-const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', replace_value: '', sequence: 0 }
+const EMPTY_FORM = { name: '', field: '', pattern: '', output_field: '', function_type: 'extract', flags: '', replace_value: '', retain: false, sequence: 0 }
 
 function PreviewModal({ rows, onClose }) {
   const matched = rows.filter(r => r.extracted_value != null).length
@@ -144,6 +144,16 @@ function FormPanel({ form, setForm, editing, error, loading, fields, source, onS
             />
           </div>
         </div>
+        {form.function_type === 'extract' && (
+          <label className="flex items-center gap-2 text-xs text-gray-600 cursor-pointer select-none">
+            <input
+              type="checkbox"
+              checked={!!form.retain}
+              onChange={e => setForm(f => ({ ...f, retain: e.target.checked }))}
+            />
+            Retain extracted value in output field even when a mapping is applied
+          </label>
+        )}
         {form.function_type === 'replace' && (
           <div>
             <label className="text-xs text-gray-500 block mb-1">Replacement string</label>
@@ -237,6 +247,7 @@ export default function Rules({ source }) {
       function_type: rule.function_type || 'extract',
       flags: rule.flags || '',
       replace_value: rule.replace_value || '',
+      retain: rule.retain || false,
       sequence: rule.sequence,
     })
     setEditing(rule.id)