Flatten database/queries into database/ and fix five stale functions

database/functions.sql held a full pg_dump appended onto the original
hand-written file, duplicating ~50 functions that had since been split into
database/queries/. Nothing deployed it, but CLAUDE.md and the tutorial still
told you to psql it, which would have reverted the split versions.

The reverse had also happened: five functions in queries/ were behind the live
database, all of them undoing the May 2026 split of the transformed column.
preview_rule lost its data -> transformed fallback for chained rules;
set_/clear_/bulk_set_record_overrides wrote overrides back into transformed and
returned the wrong type (which would have made the redeploy error outright);
generate_source_view read only transformed instead of merging all three layers.
Those are corrected here from the live definitions.

generate_source_view additionally regains the _overridden column that queries/
had and live lacked — Records.jsx reads row._overridden to highlight manually
edited rows, so that indicator had been dead.

The seven functions that existed only in functions.sql move to two new files,
import.sql (import + audit trail) and transform.sql (the rule/mapping engine),
leaving database/ flat: schema.sql plus one file per route. The four already
applied migrate_*.sql scripts are removed.

manage.py picks up the new files in QUERY_FILES, and its DB_ACTIONS set now
keys off the action functions rather than duplicated label strings that no
longer matched any menu entry, so the "into database X" hint renders again.

uninstall.sh is folded into manage.py as menu option 10. Beyond what the script
did, it stops/disables/removes the systemd unit, removes the nginx site with an
nginx -t check before reloading, and deletes public/.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

database/import.sql

@@ -0,0 +1,159 @@
+--
+-- Import queries
+-- CSV import and the import audit trail; SQL for the import/log endpoints in
+-- api/routes/sources.js
+--
+
+SET search_path TO dataflow, public;
+
+-- ── Import ────────────────────────────────────────────────────────────────────
+
+-- Import records, skipping any whose constraint key already exists in the table.
+--
+-- Dedup is enforced here, in the CTE — there is no unique constraint on
+-- constraint_key and ON CONFLICT must never be used. Within one batch every row
+-- inserts even if two rows share a constraint key, because banks legitimately send
+-- identical-looking transactions (same date, description, amount) on the same day.
+-- The key exists only to stop a re-imported overlapping date range from
+-- double-counting rows already in the table.
+CREATE OR REPLACE FUNCTION import_records(
+    p_source_name TEXT,
+    p_data JSONB  -- Array of records
+) RETURNS JSON AS $$
+DECLARE
+    v_constraint_fields TEXT[];
+    v_inserted          INTEGER;
+    v_duplicates        INTEGER;
+    v_log_id            INTEGER;
+BEGIN
+    SELECT constraint_fields INTO v_constraint_fields
+    FROM dataflow.sources
+    WHERE name = p_source_name;
+
+    IF v_constraint_fields IS NULL THEN
+        RETURN json_build_object(
+            'success', false,
+            'error', 'Source not found: ' || p_source_name
+        );
+    END IF;
+
+    WITH
+    -- All incoming records with their constraint keys
+    pending AS (
+        SELECT
+            rec.value      AS data,
+            rec.ordinality AS seq,
+            (SELECT jsonb_object_agg(f, rec.value->>f)
+             FROM unnest(v_constraint_fields) AS f) AS constraint_key
+        FROM jsonb_array_elements(p_data) WITH ORDINALITY AS rec
+    ),
+    -- Keys already in the database (excluded)
+    existing AS (
+        SELECT DISTINCT r.constraint_key
+        FROM dataflow.records r
+        INNER JOIN pending p ON p.constraint_key = r.constraint_key
+        WHERE r.source_name = p_source_name
+    ),
+    -- Rows whose constraint key is not yet in the database
+    new_records AS (
+        SELECT p.data, p.constraint_key, p.seq
+        FROM pending p
+        WHERE NOT EXISTS (SELECT 1 FROM existing e WHERE e.constraint_key = p.constraint_key)
+    ),
+    -- Write the log entry
+    log_entry AS (
+        INSERT INTO dataflow.import_log (source_name, records_imported, records_duplicate, info)
+        VALUES (
+            p_source_name,
+            (SELECT count(*) FROM new_records),
+            (SELECT count(*) FROM pending) - (SELECT count(*) FROM new_records),
+            jsonb_build_object(
+                'total',          jsonb_array_length(p_data),
+                'inserted_keys',  (SELECT jsonb_agg(constraint_key ORDER BY constraint_key) FROM new_records),
+                'excluded_keys',  (SELECT jsonb_agg(constraint_key) FROM existing)
+            )
+        )
+        RETURNING id, records_imported, records_duplicate
+    ),
+    -- Insert new records
+    inserted AS (
+        INSERT INTO dataflow.records (source_name, data, constraint_key, import_id)
+        SELECT p_source_name, nr.data, nr.constraint_key, (SELECT id FROM log_entry)
+        FROM new_records nr
+        ORDER BY nr.seq
+        RETURNING id
+    )
+    SELECT le.id, le.records_imported, le.records_duplicate
+    INTO   v_log_id, v_inserted, v_duplicates
+    FROM   log_entry le;
+
+    RETURN json_build_object(
+        'success',    true,
+        'imported',   v_inserted,
+        'duplicates', v_duplicates,
+        'log_id',     v_log_id
+    );
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION import_records IS 'Import records with automatic deduplication';
+
+-- ── Audit trail ───────────────────────────────────────────────────────────────
+
+CREATE OR REPLACE FUNCTION get_import_log(p_source_name TEXT)
+RETURNS TABLE (
+    id              INTEGER,
+    source_name     TEXT,
+    records_imported INTEGER,
+    records_duplicate INTEGER,
+    imported_at     TIMESTAMPTZ,
+    info            JSONB
+) AS $$
+    SELECT id, source_name, records_imported, records_duplicate, imported_at, info
+    FROM dataflow.import_log
+    WHERE source_name = p_source_name
+    ORDER BY imported_at DESC;
+$$ LANGUAGE sql;
+
+COMMENT ON FUNCTION get_import_log IS 'Return import history for a source, newest first, including inserted/excluded key lists';
+
+CREATE OR REPLACE FUNCTION get_all_import_logs()
+RETURNS TABLE (
+    id               INTEGER,
+    source_name      TEXT,
+    records_imported INTEGER,
+    records_duplicate INTEGER,
+    imported_at      TIMESTAMPTZ,
+    info             JSONB
+) AS $$
+    SELECT id, source_name, records_imported, records_duplicate, imported_at, info
+    FROM dataflow.import_log
+    ORDER BY imported_at DESC;
+$$ LANGUAGE sql;
+
+COMMENT ON FUNCTION get_all_import_logs IS 'Return import history across all sources, newest first';
+
+-- Records are removed by the import_id FK's ON DELETE CASCADE
+CREATE OR REPLACE FUNCTION delete_import(p_log_id INTEGER)
+RETURNS JSON AS $$
+DECLARE
+    v_deleted INTEGER;
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM dataflow.import_log WHERE id = p_log_id) THEN
+        RETURN json_build_object('success', false, 'error', 'Import log entry not found');
+    END IF;
+
+    SELECT count(*) INTO v_deleted FROM dataflow.records WHERE import_id = p_log_id;
+
+    -- Cascade handles deleting records via FK ON DELETE CASCADE
+    DELETE FROM dataflow.import_log WHERE id = p_log_id;
+
+    RETURN json_build_object(
+        'success',         true,
+        'records_deleted', v_deleted,
+        'log_id',          p_log_id
+    );
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION delete_import IS 'Delete all records belonging to an import batch and remove the log entry';

database/migrate_input_value_jsonb.sql

@@ -1,22 +0,0 @@
---
--- Migration: Change mappings.input_value from TEXT to JSONB
--- Allows multi-capture-group regex results to be used as mapping keys
---
-
-SET search_path TO dataflow, public;
-
--- Drop dependent constraint and index first
-ALTER TABLE dataflow.mappings DROP CONSTRAINT mappings_source_name_rule_name_input_value_key;
-DROP INDEX IF EXISTS dataflow.idx_mappings_input;
-
--- Convert column: existing TEXT values become JSONB strings e.g. "MEIJER"
-ALTER TABLE dataflow.mappings
-    ALTER COLUMN input_value TYPE JSONB
-    USING to_jsonb(input_value);
-
--- Recreate constraint and index
-ALTER TABLE dataflow.mappings
-    ADD CONSTRAINT mappings_source_name_rule_name_input_value_key
-    UNIQUE (source_name, rule_name, input_value);
-
-CREATE INDEX idx_mappings_input ON dataflow.mappings(source_name, rule_name, input_value);

database/migrate_overrides_column.sql

@@ -1,40 +0,0 @@
---
--- Migration: add overrides column to records
---
--- Separates the three data layers:
---   data        — original import values, never mutated
---   transformed — rule/mapping output fields only (delta)
---   overrides   — manual user overrides (highest precedence)
---
--- Consumers merge as: data || COALESCE(transformed,'{}') || COALESCE(overrides,'{}')
---
--- Safe to run multiple times (IF NOT EXISTS guards).
---
-
-SET search_path TO dataflow, public;
-
--- 1. Add overrides column
-ALTER TABLE dataflow.records
-    ADD COLUMN IF NOT EXISTS overrides JSONB;
-
--- 2. Add partial GIN index (only indexes rows that have overrides)
-CREATE INDEX IF NOT EXISTS idx_records_overrides
-    ON dataflow.records USING gin(overrides)
-    WHERE overrides IS NOT NULL;
-
--- 3. Redeploy functions (CREATE OR REPLACE — non-destructive)
-\i functions.sql
-
--- 4. Reprocess all sources to strip stale data keys from transformed
---    (apply_transformations now writes only rule additions, not data || additions)
-DO $$
-DECLARE
-    src TEXT;
-    result JSON;
-BEGIN
-    FOR src IN SELECT name FROM dataflow.sources ORDER BY name LOOP
-        SELECT dataflow.reprocess_records(src) INTO result;
-        RAISE NOTICE 'Reprocessed %: %', src, result;
-    END LOOP;
-END;
-$$;

database/migrate_pivot_layouts_drop_fk.sql

@@ -1,4 +0,0 @@
--- Drop the foreign key from pivot_layouts.source_name so stack view names can also
--- be used as layout keys (stacks are not rows in the sources table).
-ALTER TABLE dataflow.pivot_layouts
-    DROP CONSTRAINT pivot_layouts_source_name_fkey;

database/migrate_tps.sql

@@ -1,121 +0,0 @@
---
--- TPS → Dataflow Migration
---
--- Migrates sources, rules, mappings, and records from the TPS system.
--- Run against the dataflow database:
---   PGPASSWORD=dataflow psql -U dataflow -d dataflow -h localhost -f database/migrate_tps.sql
---
--- Existing rows are skipped (ON CONFLICT DO NOTHING) so the script is safe to re-run.
--- NOTE: dcard already configured in dataflow will NOT be overwritten.
---
-
-SET search_path TO dataflow, public;
-
-CREATE EXTENSION IF NOT EXISTS dblink;
-
--- Connection string to the TPS database
-\set tps_conn 'host=192.168.1.110 dbname=ubm user=api password=gyaswddh1983'
-
-\echo ''
-\echo '=== 1. Sources ==='
-
-INSERT INTO dataflow.sources (name, constraint_fields, config)
-SELECT
-    srce AS name,
-    -- Strip {} wrappers from constraint paths → constraint field names
-    ARRAY(
-        SELECT regexp_replace(c, '^\{|\}$', '', 'g')
-        FROM jsonb_array_elements_text(defn->'constraint') AS c
-    ) AS constraint_fields,
-    -- Build config.fields from the first schema (index 0 = "mapped" for dcard, "default" for others)
-    jsonb_build_object('fields',
-        (SELECT jsonb_agg(
-            jsonb_build_object(
-                'name', regexp_replace(col->>'path', '^\{|\}$', '', 'g'),
-                'type', COALESCE(NULLIF(col->>'type', ''), 'text')
-            ) ORDER BY ord
-        )
-        FROM jsonb_array_elements(defn->'schemas'->0->'columns')
-             WITH ORDINALITY AS t(col, ord)
-        )
-    ) AS config
-FROM dblink(:'tps_conn',
-    'SELECT srce, defn FROM tps.srce'
-) AS t(srce TEXT, defn JSONB)
-ON CONFLICT (name) DO NOTHING;
-
-SELECT name, constraint_fields, jsonb_array_length(config->'fields') AS field_count
-FROM dataflow.sources ORDER BY name;
-
-\echo ''
-\echo '=== 2. Rules ==='
-
-INSERT INTO dataflow.rules
-    (source_name, name, field, pattern, output_field, function_type, flags, replace_value, sequence, enabled, retain)
-SELECT
-    srce                                                                AS source_name,
-    target                                                              AS name,
-    -- Strip {} from the input field key
-    regexp_replace(regex->'regex'->'defn'->0->>'key', '^\{|\}$', '', 'g') AS field,
-    regex->'regex'->'defn'->0->>'regex'                                AS pattern,
-    regex->'regex'->'defn'->0->>'field'                                AS output_field,
-    COALESCE(NULLIF(regex->'regex'->>'function', ''), 'extract')       AS function_type,
-    COALESCE(regex->'regex'->'defn'->0->>'flag', '')                   AS flags,
-    ''                                                                  AS replace_value,
-    seq                                                                 AS sequence,
-    true                                                                AS enabled,
-    (regex->'regex'->'defn'->0->>'retain') = 'y'                       AS retain
-FROM dblink(:'tps_conn',
-    'SELECT srce, target, seq, regex FROM tps.map_rm'
-) AS t(srce TEXT, target TEXT, seq INT, regex JSONB)
-ON CONFLICT (source_name, name) DO NOTHING;
-
-SELECT source_name, name, field, pattern, output_field, sequence
-FROM dataflow.rules ORDER BY source_name, sequence;
-
-\echo ''
-\echo '=== 3. Mappings ==='
-
-INSERT INTO dataflow.mappings (source_name, rule_name, input_value, output)
-SELECT
-    srce        AS source_name,
-    target      AS rule_name,
-    -- retval is {"f20": "<extracted string>"} — pull out the value as JSONB
-    (SELECT value FROM jsonb_each(retval) LIMIT 1) AS input_value,
-    map         AS output
-FROM dblink(:'tps_conn',
-    'SELECT srce, target, retval, map FROM tps.map_rv'
-) AS t(srce TEXT, target TEXT, retval JSONB, map JSONB)
-ON CONFLICT (source_name, rule_name, input_value) DO NOTHING;
-
-SELECT source_name, rule_name, COUNT(*) AS mapping_count
-FROM dataflow.mappings GROUP BY source_name, rule_name ORDER BY source_name, rule_name;
-
-\echo ''
-\echo '=== 4. Records ==='
-\echo '    (13 000+ rows — may take a moment)'
-
-INSERT INTO dataflow.records (source_name, data, constraint_key, transformed, imported_at, transformed_at)
-SELECT
-    t.srce                                                  AS source_name,
-    t.rec                                                   AS data,
-    (SELECT jsonb_object_agg(f, t.rec->>f) FROM unnest(s.constraint_fields) AS f) AS constraint_key,
-    t.allj                                                  AS transformed,
-    CURRENT_TIMESTAMP                                       AS imported_at,
-    CASE WHEN t.allj IS NOT NULL THEN CURRENT_TIMESTAMP END AS transformed_at
-FROM dblink(:'tps_conn',
-    'SELECT srce, rec, allj FROM tps.trans'
-) AS t(srce TEXT, rec JSONB, allj JSONB)
-JOIN dataflow.sources s ON s.name = t.srce
-ON CONFLICT (source_name, constraint_key) DO NOTHING;
-
-SELECT source_name, COUNT(*) AS records, COUNT(transformed) AS transformed
-FROM dataflow.records GROUP BY source_name ORDER BY source_name;
-
-\echo ''
-\echo '=== Migration complete ==='
-SELECT
-    (SELECT COUNT(*) FROM dataflow.sources)  AS sources,
-    (SELECT COUNT(*) FROM dataflow.rules)    AS rules,
-    (SELECT COUNT(*) FROM dataflow.mappings) AS mappings,
-    (SELECT COUNT(*) FROM dataflow.records)  AS records;

database/records.sql

@@ -41,37 +41,45 @@ $$ LANGUAGE sql STABLE;
 
 -- ── Overrides ─────────────────────────────────────────────────────────────────
 
--- Store manual overrides and immediately merge into transformed
-CREATE OR REPLACE FUNCTION set_record_overrides(p_id INT, p_overrides JSONB)
-RETURNS dataflow.records AS $$
-    UPDATE dataflow.records
-    SET overrides  = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END,
-        transformed = COALESCE(transformed, data) || COALESCE(p_overrides, '{}'::jsonb)
-    WHERE id = p_id
-    RETURNING *;
+-- Store manual overrides. Overrides stay in their own column — never merged into
+-- transformed — so reprocessing rules cannot clobber a manual edit.
+DROP FUNCTION IF EXISTS set_record_overrides(INTEGER, JSONB);
+CREATE OR REPLACE FUNCTION set_record_overrides(p_id INTEGER, p_overrides JSONB)
+RETURNS JSON AS $$
+    WITH updated AS (
+        UPDATE dataflow.records
+        SET overrides = CASE WHEN p_overrides = '{}'::jsonb THEN NULL ELSE p_overrides END
+        WHERE id = p_id
+        RETURNING *
+    )
+    SELECT row_to_json(updated) FROM updated;
 $$ LANGUAGE sql;
 
 -- Merge overrides into multiple records at once; returns actual updated count
-CREATE OR REPLACE FUNCTION bulk_set_record_overrides(p_source_name TEXT, p_ids INT[], p_overrides JSONB)
-RETURNS BIGINT AS $$
+DROP FUNCTION IF EXISTS bulk_set_record_overrides(TEXT, INTEGER[], JSONB);
+CREATE OR REPLACE FUNCTION bulk_set_record_overrides(p_source_name TEXT, p_ids INTEGER[], p_overrides JSONB)
+RETURNS JSON AS $$
     WITH updated AS (
         UPDATE dataflow.records
-        SET overrides   = COALESCE(overrides, '{}'::jsonb) || p_overrides,
-            transformed = COALESCE(transformed, data) || p_overrides
+        SET overrides = COALESCE(overrides, '{}'::jsonb) || p_overrides
         WHERE id = ANY(p_ids)
           AND source_name = p_source_name
         RETURNING id
     )
-    SELECT count(*) FROM updated;
+    SELECT json_build_object('updated', count(*)) FROM updated;
 $$ LANGUAGE sql;
 
--- Clear overrides; caller should reprocess to restore computed transformed value
-CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INT)
-RETURNS dataflow.records AS $$
-    UPDATE dataflow.records
-    SET overrides = NULL
-    WHERE id = p_id
-    RETURNING *;
+-- Clear overrides; the computed values in transformed are untouched
+DROP FUNCTION IF EXISTS clear_record_overrides(INTEGER);
+CREATE OR REPLACE FUNCTION clear_record_overrides(p_id INTEGER)
+RETURNS JSON AS $$
+    WITH updated AS (
+        UPDATE dataflow.records
+        SET overrides = NULL
+        WHERE id = p_id
+        RETURNING *
+    )
+    SELECT row_to_json(updated) FROM updated;
 $$ LANGUAGE sql;
 
 -- ── Delete ────────────────────────────────────────────────────────────────────

database/rules.sql

@@ -86,21 +86,27 @@ CREATE OR REPLACE FUNCTION preview_rule(
     p_limit         INT  DEFAULT 20
 )
 RETURNS TABLE (id INT, raw_value TEXT, extracted_value JSONB) AS $$
+-- Field is resolved from data first, then transformed (supports chained rules whose
+-- input field was produced by an earlier-sequence rule rather than the raw import).
 BEGIN
     IF p_function_type = 'replace' THEN
         RETURN QUERY
         SELECT
             r.id,
-            r.data ->> p_field,
-            to_jsonb(regexp_replace(r.data ->> p_field, p_pattern, p_replace_value, p_flags))
+            COALESCE(r.data ->> p_field, r.transformed ->> p_field),
+            to_jsonb(regexp_replace(
+                COALESCE(r.data ->> p_field, r.transformed ->> p_field),
+                p_pattern, p_replace_value, p_flags
+            ))
         FROM dataflow.records r
-        WHERE source_name = p_source AND data ? p_field
+        WHERE source_name = p_source
+          AND (data ? p_field OR transformed ? p_field)
         ORDER BY r.id DESC LIMIT p_limit;
     ELSE
         RETURN QUERY
         SELECT
             r.id,
-            r.data ->> p_field,
+            COALESCE(r.data ->> p_field, r.transformed ->> p_field),
             CASE
                 WHEN agg.match_count = 0 THEN NULL
                 WHEN agg.match_count = 1 THEN agg.matches -> 0
@@ -114,10 +120,14 @@ BEGIN
                     ORDER BY rn
                 ) AS matches,
                 count(*)::int AS match_count
-            FROM regexp_matches(r.data ->> p_field, p_pattern, p_flags)
+            FROM regexp_matches(
+                COALESCE(r.data ->> p_field, r.transformed ->> p_field),
+                p_pattern, p_flags
+            )
             WITH ORDINALITY AS m(mt, rn)
         ) agg
-        WHERE r.source_name = p_source AND r.data ? p_field
+        WHERE r.source_name = p_source
+          AND (r.data ? p_field OR r.transformed ? p_field)
         ORDER BY r.id DESC LIMIT p_limit;
     END IF;
 END;

database/sources.sql

@@ -40,8 +40,6 @@ RETURNS TEXT AS $$
     DELETE FROM dataflow.sources WHERE name = p_name RETURNING name;
 $$ LANGUAGE sql;
 
--- ── Import log ────────────────────────────────────────────────────────────────
-
 -- ── Stats ─────────────────────────────────────────────────────────────────────
 
 CREATE OR REPLACE FUNCTION get_source_stats(p_source_name TEXT)
@@ -161,6 +159,7 @@ BEGIN
         RETURN json_build_object('success', false, 'error', 'No schema fields defined for this source');
     END IF;
 
+    -- Columns read from r, the merged data || transformed || overrides object
     FOR v_field IN SELECT * FROM jsonb_array_elements(v_config->'fields') LOOP
         IF v_cols != '' THEN v_cols := v_cols || ', '; END IF;
 
@@ -171,24 +170,27 @@ BEGIN
             BEGIN
                 WHILE v_expr ~ '\{[^}]+\}' LOOP
                     v_ref  := substring(v_expr FROM '\{([^}]+)\}');
-                    v_expr := replace(v_expr, '{' || v_ref || '}', format('(transformed->>%L)::numeric', v_ref));
+                    v_expr := replace(v_expr, '{' || v_ref || '}', format('(r->>%L)::numeric', v_ref));
                 END LOOP;
                 v_cols := v_cols || format('%s AS %I', v_expr, v_field->>'name');
             END;
         ELSE
             CASE v_field->>'type'
-                WHEN 'date'    THEN v_cols := v_cols || format('(transformed->>%L)::date    AS %I', v_field->>'name', v_field->>'name');
-                WHEN 'numeric' THEN v_cols := v_cols || format('(transformed->>%L)::numeric AS %I', v_field->>'name', v_field->>'name');
-                ELSE                v_cols := v_cols || format('transformed->>%L             AS %I', v_field->>'name', v_field->>'name');
+                WHEN 'date'    THEN v_cols := v_cols || format('(r->>%L)::date    AS %I', v_field->>'name', v_field->>'name');
+                WHEN 'numeric' THEN v_cols := v_cols || format('(r->>%L)::numeric AS %I', v_field->>'name', v_field->>'name');
+                ELSE                v_cols := v_cols || format('r->>%L             AS %I', v_field->>'name', v_field->>'name');
             END CASE;
         END IF;
     END LOOP;
 
     CREATE SCHEMA IF NOT EXISTS dfv;
     v_view := 'dfv.' || quote_ident(p_source_name);
-    EXECUTE format('DROP VIEW IF EXISTS %s', v_view);
+    EXECUTE format('DROP VIEW IF EXISTS %s CASCADE', v_view);
     v_sql := format(
-        'CREATE VIEW %s AS SELECT id, overrides IS NOT NULL AS _overridden, %s FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL',
+        'CREATE VIEW %s AS SELECT id, _overridden, %s FROM ('
+        || 'SELECT id, overrides IS NOT NULL AS _overridden, '
+        || 'data || COALESCE(transformed, ''{}''::jsonb) || COALESCE(overrides, ''{}''::jsonb) AS r '
+        || 'FROM dataflow.records WHERE source_name = %L AND transformed IS NOT NULL) rec',
         v_view, v_cols, p_source_name
     );
     EXECUTE v_sql;

database/transform.sql

@@ -0,0 +1,156 @@
+--
+-- Transform queries
+-- The rule/mapping engine; SQL for the transform endpoints in api/routes/sources.js,
+-- api/routes/rules.js and api/routes/records.js
+--
+-- Order matters within this file: the aggregate is used by apply_transformations,
+-- which in turn is called by reprocess_records.
+--
+
+SET search_path TO dataflow, public;
+
+-- ── Merge aggregate ───────────────────────────────────────────────────────────
+
+-- Merge JSONB objects across rows (later rows win on key conflicts)
+-- Usage: jsonb_concat_obj(col ORDER BY sequence)
+CREATE OR REPLACE FUNCTION dataflow.jsonb_merge(a JSONB, b JSONB)
+RETURNS JSONB AS $$
+    SELECT COALESCE(a, '{}') || COALESCE(b, '{}')
+$$ LANGUAGE sql IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS dataflow.jsonb_concat_obj(JSONB);
+CREATE AGGREGATE dataflow.jsonb_concat_obj(JSONB) (
+    sfunc    = dataflow.jsonb_merge,
+    stype    = JSONB,
+    initcond = '{}'
+);
+
+-- ── Apply rules and mappings ──────────────────────────────────────────────────
+
+-- Writes only the rule/mapping output into records.transformed. Raw values stay in
+-- data and manual edits stay in overrides; readers merge the three layers.
+DROP FUNCTION IF EXISTS apply_transformations(TEXT, INTEGER[]);
+CREATE OR REPLACE FUNCTION apply_transformations(
+    p_source_name TEXT,
+    p_record_ids INTEGER[] DEFAULT NULL,  -- NULL = all eligible records
+    p_overwrite   BOOLEAN  DEFAULT FALSE  -- FALSE = skip already-transformed, TRUE = overwrite all
+) RETURNS JSON AS $$
+WITH
+-- All records to process
+qualifying AS (
+    SELECT id, data
+    FROM dataflow.records
+    WHERE source_name = p_source_name
+      AND (p_overwrite OR transformed IS NULL)
+      AND (p_record_ids IS NULL OR id = ANY(p_record_ids))
+),
+-- Mirror TPS rx: fan out one row per regex match, drive from rules → records
+rx AS (
+    SELECT
+        q.id,
+        r.name          AS rule_name,
+        r.sequence,
+        r.output_field,
+        r.retain,
+        r.function_type,
+        COALESCE(mt.rn, rp.rn, 1) AS result_number,
+        -- extract: build map_val and retain_val per match (mirrors TPS)
+        CASE WHEN array_length(mt.mt, 1) = 1 THEN to_jsonb(mt.mt[1]) ELSE to_jsonb(mt.mt) END AS match_val,
+        to_jsonb(rp.rp) AS replace_val
+    FROM dataflow.rules r
+    INNER JOIN qualifying q ON q.data ? r.field
+    LEFT JOIN LATERAL regexp_matches(q.data ->> r.field, r.pattern, r.flags)
+        WITH ORDINALITY AS mt(mt, rn) ON r.function_type = 'extract'
+    LEFT JOIN LATERAL regexp_replace(q.data ->> r.field, r.pattern, r.replace_value, r.flags)
+        WITH ORDINALITY AS rp(rp, rn) ON r.function_type = 'replace'
+    WHERE r.source_name = p_source_name
+      AND r.enabled = true
+),
+-- Aggregate match rows back into one value per (record, rule) — mirrors TPS agg_to_target_items
+agg_matches AS (
+    SELECT
+        id,
+        rule_name,
+        sequence,
+        output_field,
+        retain,
+        function_type,
+        CASE function_type
+            WHEN 'replace' THEN jsonb_agg(replace_val) -> 0
+            ELSE
+                CASE WHEN max(result_number) = 1
+                    THEN jsonb_agg(match_val ORDER BY result_number) -> 0
+                    ELSE jsonb_agg(match_val ORDER BY result_number)
+                END
+        END AS extracted
+    FROM rx
+    GROUP BY id, rule_name, sequence, output_field, retain, function_type
+),
+-- Join with mappings to find mapped output — mirrors TPS link_map
+linked AS (
+    SELECT
+        a.id,
+        a.sequence,
+        a.output_field,
+        a.retain,
+        a.extracted,
+        m.output AS mapped
+    FROM agg_matches a
+    LEFT JOIN dataflow.mappings m ON
+        m.source_name = p_source_name
+        AND m.rule_name = a.rule_name
+        AND m.input_value = a.extracted
+    WHERE a.extracted IS NOT NULL
+),
+-- Build per-rule output JSONB:
+--   mapped  → use mapping output; also write output_field if retain = true
+--   no map  → write extracted value to output_field
+rule_output AS (
+    SELECT
+        id,
+        sequence,
+        CASE
+            WHEN mapped IS NOT NULL THEN
+                mapped ||
+                CASE WHEN retain
+                     THEN jsonb_build_object(output_field, extracted)
+                     ELSE '{}'::jsonb
+                END
+            ELSE
+                jsonb_build_object(output_field, extracted)
+        END AS output
+    FROM linked
+),
+-- Merge all rule outputs per record in sequence order — mirrors TPS agg_to_id
+record_additions AS (
+    SELECT
+        id,
+        dataflow.jsonb_concat_obj(output ORDER BY sequence) AS additions
+    FROM rule_output
+    GROUP BY id
+),
+-- Update all qualifying records; records with no rule matches get an empty object
+updated AS (
+    UPDATE dataflow.records rec
+    SET transformed    = COALESCE(ra.additions, '{}'::jsonb),
+        transformed_at = CURRENT_TIMESTAMP
+    FROM qualifying q
+    LEFT JOIN record_additions ra ON ra.id = q.id
+    WHERE rec.id = q.id
+    RETURNING rec.id
+)
+SELECT json_build_object('success', true, 'transformed', count(*))
+FROM updated
+$$ LANGUAGE sql;
+
+COMMENT ON FUNCTION apply_transformations IS 'Apply transformation rules and mappings to records (set-based CTE)';
+
+-- ── Reprocess ─────────────────────────────────────────────────────────────────
+
+CREATE OR REPLACE FUNCTION reprocess_records(p_source_name TEXT)
+RETURNS JSON AS $$
+    -- Overwrite all records directly — no clear step, mirrors TPS srce_map_overwrite
+    SELECT dataflow.apply_transformations(p_source_name, NULL, TRUE)
+$$ LANGUAGE sql;
+
+COMMENT ON FUNCTION reprocess_records IS 'Reapply all transformations for a source, overwriting existing values';

manage.py

@@ -18,13 +18,16 @@ SERVICE_FILE = Path('/etc/systemd/system/dataflow.service')
 SERVICE_SRC  = ROOT / 'dataflow.service'
 NGINX_DIR    = Path('/etc/nginx/sites-enabled')
 
-# Deployed in order — stacks.sql creates tables that reference sources
-QUERIES_DIR  = ROOT / 'database' / 'queries'
+# Deployed in order — stacks.sql creates tables that reference sources, and
+# transform.sql defines an aggregate its own functions depend on
+QUERIES_DIR  = ROOT / 'database'
 QUERY_FILES  = [
     QUERIES_DIR / 'sources.sql',
     QUERIES_DIR / 'rules.sql',
     QUERIES_DIR / 'mappings.sql',
     QUERIES_DIR / 'records.sql',
+    QUERIES_DIR / 'import.sql',
+    QUERIES_DIR / 'transform.sql',
     QUERIES_DIR / 'stacks.sql',
     QUERIES_DIR / 'status.sql',
 ]
@@ -164,23 +167,31 @@ def ui_build_time():
         return datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
     return None
 
-def nginx_domain(port):
-    """Find nginx site proxying to our port."""
+def nginx_conf_path(port):
+    """Path of the nginx site proxying to our port, if any."""
     if not NGINX_DIR.exists():
         return None
     for f in NGINX_DIR.iterdir():
         try:
-            text = f.read_text()
-            if f':{port}' in text:
-                for line in text.splitlines():
-                    if 'server_name' in line:
-                        parts = line.split()
-                        if len(parts) >= 2:
-                            return parts[1].rstrip(';')
+            if f':{port}' in f.read_text():
+                return f
         except Exception:
             pass
     return None
 
+
+def nginx_domain(port):
+    """server_name of the nginx site proxying to our port."""
+    conf = nginx_conf_path(port)
+    if not conf:
+        return None
+    for line in conf.read_text().splitlines():
+        if 'server_name' in line:
+            parts = line.split()
+            if len(parts) >= 2:
+                return parts[1].rstrip(';')
+    return None
+
 def sudo_run(args, **kwargs):
     return subprocess.run(['sudo'] + args, **kwargs)
 
@@ -433,7 +444,7 @@ def action_deploy_schema(cfg):
 
 
 def action_deploy_functions(cfg):
-    header('Deploy SQL functions (database/queries/)')
+    header('Deploy SQL functions (database/*.sql)')
     if not cfg:
         err(f'{ENV_FILE} not found — run option 1 to configure the database connection first')
         return
@@ -729,6 +740,116 @@ def action_stop_service():
     ok('dataflow.service stopped')
 
 
+def action_uninstall(cfg):
+    """Reverse everything this script installs, outside the repo itself."""
+    header('Uninstall dataflow')
+
+    port      = cfg.get('API_PORT', '3020') if cfg else '3020'
+    db_name   = cfg.get('DB_NAME', 'dataflow') if cfg else 'dataflow'
+    db_user   = cfg.get('DB_USER', 'dataflow') if cfg else 'dataflow'
+    conf_path = nginx_conf_path(port)
+
+    # Everything that exists right now, in reverse install order
+    targets = []
+    if service_installed():
+        targets.append(f'systemd service   {SERVICE_FILE}' +
+                       ('  (running)' if service_running() else ''))
+    if conf_path:
+        targets.append(f'nginx site        {conf_path}')
+    if cfg and can_connect(cfg):
+        targets.append(f'database          "{db_name}" on {cfg["DB_HOST"]}:{cfg["DB_PORT"]}  (ALL DATA)')
+        targets.append(f'database user     {db_user}')
+    if ENV_FILE.exists():
+        targets.append(f'config            {ENV_FILE}')
+    if (ROOT / 'public').exists():
+        targets.append(f'built UI          {ROOT / "public"}')
+    if (ROOT / 'node_modules').exists():
+        targets.append(f'dependencies      {ROOT / "node_modules"}')
+
+    if not targets:
+        info('Nothing installed to remove.')
+        return cfg
+
+    print('  This will permanently remove:')
+    for t in targets:
+        print(f'    {t}')
+    print()
+    info(f'The repository itself ({ROOT}) is left alone — delete it manually if you want it gone.')
+    print()
+
+    if input("  Type 'delete' to confirm: ").strip() != 'delete':
+        info('Cancelled — no changes made')
+        return cfg
+
+    # ── Service ───────────────────────────────────────────────────────────────
+    if service_installed():
+        print()
+        print('  Removing systemd service...')
+        sudo_run(['systemctl', 'stop', 'dataflow'])
+        sudo_run(['systemctl', 'disable', 'dataflow'])
+        r = sudo_run(['rm', '-f', str(SERVICE_FILE)])
+        if r.returncode != 0:
+            err(f'Could not remove {SERVICE_FILE} — check sudo permissions')
+        else:
+            sudo_run(['systemctl', 'daemon-reload'])
+            ok(f'Service stopped, disabled, and {SERVICE_FILE} removed')
+
+    # ── nginx ─────────────────────────────────────────────────────────────────
+    if conf_path:
+        print()
+        print('  Removing nginx site...')
+        r = sudo_run(['rm', '-f', str(conf_path)])
+        if r.returncode != 0:
+            err(f'Could not remove {conf_path} — check sudo permissions')
+        elif sudo_run(['nginx', '-t'], capture_output=True).returncode != 0:
+            err('nginx config test failed after removal — not reloading; check nginx manually')
+        else:
+            sudo_run(['systemctl', 'reload', 'nginx'])
+            ok(f'{conf_path} removed and nginx reloaded')
+
+    # ── Database ──────────────────────────────────────────────────────────────
+    if cfg and can_connect(cfg):
+        print()
+        print(f'  Dropping the database requires PostgreSQL admin credentials.')
+        admin = {
+            'user':     prompt('PostgreSQL admin username', 'postgres'),
+            'password': prompt('PostgreSQL admin password', secret=True),
+            'host':     cfg['DB_HOST'],
+            'port':     cfg['DB_PORT'],
+        }
+        r = psql_admin(admin, 'SELECT 1')
+        if r.returncode != 0:
+            err(f'Cannot connect as admin — database and user left in place\n{r.stderr.strip()}')
+        else:
+            r = psql_admin(admin, f'DROP DATABASE IF EXISTS {db_name}')
+            if r.returncode != 0:
+                err(f'Could not drop database "{db_name}"\n{r.stderr.strip()}')
+            else:
+                ok(f'Database "{db_name}" dropped')
+                r = psql_admin(admin, f'DROP USER IF EXISTS {db_user}')
+                if r.returncode != 0:
+                    err(f'Could not drop user {db_user}\n{r.stderr.strip()}')
+                else:
+                    ok(f'User {db_user} dropped')
+
+    # ── Generated files ───────────────────────────────────────────────────────
+    print()
+    for path, label in [(ENV_FILE, 'config'),
+                        (ROOT / 'public', 'built UI'),
+                        (ROOT / 'node_modules', 'dependencies')]:
+        if not path.exists():
+            continue
+        if path.is_dir():
+            shutil.rmtree(path, ignore_errors=True)
+        else:
+            path.unlink()
+        ok(f'Removed {label} ({path})')
+
+    print()
+    ok('Uninstall complete')
+    return None
+
+
 def action_set_login_credentials(cfg):
     header('Set login credentials (LOGIN_USER / LOGIN_PASSWORD_HASH in .env)')
 
@@ -786,13 +907,14 @@ def action_set_login_credentials(cfg):
 MENU = [
     ('Database configuration and deployment dialog  (.env)',       action_configure),
     ('Redeploy "dataflow" schema only  (database/schema.sql)',    action_deploy_schema),
-    ('Redeploy SQL functions only      (database/queries/)',      action_deploy_functions),
+    ('Redeploy SQL functions only      (database/*.sql)',         action_deploy_functions),
     ('Build UI                  (ui/ → public/)',           action_build_ui),
     ('Set up nginx reverse proxy',                         action_setup_nginx),
     ('Install dataflow systemd service unit',               action_install_service),
     ('Start / restart dataflow.service',                    action_restart_service),
     ('Stop dataflow.service',                               action_stop_service),
     ('Set login credentials',                               action_set_login_credentials),
+    ('Uninstall  (service, nginx, database, .env, build)',  action_uninstall),
 ]
 
 def main():
@@ -805,14 +927,11 @@ def main():
         show_status(cfg)
 
         db_target = f'into "{cfg["DB_NAME"]}" on {cfg["DB_HOST"]}' if cfg else '(not configured)'
-        DB_ACTIONS = {
-            'Deploy "dataflow" schema  (database/schema.sql)',
-            'Deploy SQL functions      (database/functions.sql)',
-        }
+        DB_ACTIONS = {action_deploy_schema, action_deploy_functions}
 
         print(bold('Actions'))
-        for i, (label, _) in enumerate(MENU, 1):
-            suffix = f'  {dim(db_target)}' if label in DB_ACTIONS else ''
+        for i, (label, fn) in enumerate(MENU, 1):
+            suffix = f'  {dim(db_target)}' if fn in DB_ACTIONS else ''
             print(f'  {cyan(str(i))}. {label}{suffix}')
         print(f'  {cyan("q")}. Quit')
         print()
@@ -828,13 +947,12 @@ def main():
             if 0 <= idx < len(MENU):
                 label, fn = MENU[idx]
                 import inspect
-                sig = inspect.signature(fn)
-                if len(sig.parameters) == 0:
-                    result = fn()
-                elif len(sig.parameters) == 1:
-                    result = fn(cfg)
-                if label.startswith('Configure') and result is not None:
-                    cfg = result
+                # cfg is reloaded from .env at the top of every loop, so a return
+                # value is only ever informational
+                if len(inspect.signature(fn).parameters) == 0:
+                    fn()
+                else:
+                    fn(cfg)
                 pause()
             else:
                 warn('Invalid choice — enter a number from the list above')

uninstall.sh

@@ -1,85 +0,0 @@
-#!/bin/bash
-#
-# Dataflow Uninstall Script
-# Removes database user, database, and optionally .env
-#
-
-echo "⚠️  Dataflow Uninstall"
-echo "====================="
-echo ""
-
-# Load .env if it exists
-if [ -f .env ]; then
-    export $(cat .env | grep -v '^#' | xargs)
-fi
-
-DB_NAME=${DB_NAME:-dataflow}
-DB_USER=${DB_USER:-dataflow}
-
-echo "⚠️  This will permanently delete:"
-echo "  - Database: $DB_NAME"
-echo "  - User: $DB_USER"
-echo ""
-read -p "Type 'delete' to confirm: " CONFIRM
-if [ "$CONFIRM" != "delete" ]; then
-    echo "Cancelled."
-    exit 0
-fi
-
-# Prompt for admin credentials
-echo ""
-echo "📋 PostgreSQL Admin Credentials"
-echo ""
-read -p "Admin username [postgres]: " ADMIN_USER
-ADMIN_USER=${ADMIN_USER:-postgres}
-read -s -p "Admin password: " ADMIN_PASS
-echo ""
-
-DB_HOST=${DB_HOST:-localhost}
-DB_PORT=${DB_PORT:-5432}
-DB_NAME=${DB_NAME:-dataflow}
-DB_USER=${DB_USER:-dataflow}
-
-# Test admin connection
-echo ""
-echo "🔍 Testing PostgreSQL admin connection..."
-export PGPASSWORD="$ADMIN_PASS"
-if ! psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -c '\q' 2>/dev/null; then
-    echo "✗ Cannot connect to PostgreSQL"
-    exit 1
-fi
-echo "✓ Connected"
-
-# Drop database
-echo ""
-echo "🗄️  Dropping database..."
-psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true
-echo "✓ Database dropped"
-
-# Drop user
-echo ""
-echo "👤 Dropping user..."
-psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -c "DROP USER IF EXISTS $DB_USER;" 2>/dev/null || true
-echo "✓ User dropped"
-
-unset PGPASSWORD
-
-# Optionally remove .env
-echo ""
-read -p "Remove .env file? [y/N]: " REMOVE_ENV
-if [ "$REMOVE_ENV" == "y" ] || [ "$REMOVE_ENV" == "Y" ]; then
-    rm -f .env
-    echo "✓ .env removed"
-fi
-
-# Optionally remove node_modules
-echo ""
-read -p "Remove node_modules? [y/N]: " REMOVE_MODULES
-if [ "$REMOVE_MODULES" == "y" ] || [ "$REMOVE_MODULES" == "Y" ]; then
-    rm -rf node_modules
-    echo "✓ node_modules removed"
-fi
-
-echo ""
-echo "✅ Uninstall complete!"
-echo ""