Records filters, global picklist, autocomplete, and rule reprocess

- Records tab: regex filter bar (postgres ~*), add/remove filters, debounced,
  ANDed together; get_view_data gains p_filters JSONB param
- Global picklist: sources.global_picklist flag (default true) controls whether
  a source's mapped output values feed the cross-source autocomplete suggestion pool;
  toggle on Sources page; get_global_output_values() SQL function
- Mappings: replace native datalist with custom AutocompleteInput component —
  Alt+Down opens, Tab cycles, Enter selects, arrow keys navigate, Escape closes
- Rules: auto-reprocess source records when a rule is created or updated
- preview_rule: fix BIGINT/INT return type mismatch
- Stale get_import_log removed from sources.sql
- TSV export: fetch with auth headers instead of plain <a href> (fixes 401)
- + column button: more visible styling

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/routes/mappings.js

@@ -39,6 +39,21 @@ module.exports = (pool) => {
         }
     });
 
+    // Get global output values (for autocomplete across all global_picklist=true sources)
+    router.get('/global-values', async (req, res, next) => {
+        try {
+            const result = await pool.query(`SELECT * FROM get_global_output_values()`);
+            const map = {};
+            for (const { col, val } of result.rows) {
+                if (!map[col]) map[col] = [];
+                map[col].push(val);
+            }
+            res.json(map);
+        } catch (err) {
+            next(err);
+        }
+    });
+
     // Get unmapped values
     router.get('/source/:source_name/unmapped', async (req, res, next) => {
         try {

api/routes/rules.js

@@ -73,7 +73,9 @@ module.exports = (pool) => {
             const result = await pool.query(
                 `SELECT * FROM create_rule(${lit(source_name)}, ${lit(name)}, ${lit(field)}, ${lit(pattern)}, ${lit(output_field)}, ${lit(function_type || 'extract')}, ${lit(flags || '')}, ${lit(replace_value || '')}, ${lit(enabled !== false)}, ${lit(retain === true)}, ${lit(sequence || 0)})`
             );
-            res.status(201).json(result.rows[0]);
+            const rule = result.rows[0];
+            await pool.query(`SELECT reprocess_records(${lit(source_name)})`);
+            res.status(201).json(rule);
         } catch (err) {
             if (err.code === '23505') return res.status(409).json({ error: 'Rule already exists for this source' });
             if (err.code === '23503') return res.status(404).json({ error: 'Source not found' });
@@ -93,7 +95,9 @@ module.exports = (pool) => {
                 `SELECT * FROM update_rule(${lit(parseInt(req.params.id))}, ${n(name)}, ${n(field)}, ${n(pattern)}, ${n(output_field)}, ${n(function_type)}, ${n(flags)}, ${n(replace_value)}, ${n(enabled)}, ${n(retain)}, ${n(sequence)})`
             );
             if (result.rows.length === 0) return res.status(404).json({ error: 'Rule not found' });
-            res.json(result.rows[0]);
+            const rule = result.rows[0];
+            await pool.query(`SELECT reprocess_records(${lit(rule.source_name)})`);
+            res.json(rule);
         } catch (err) {
             next(err);
         }

api/routes/sources.js

@@ -73,12 +73,12 @@ module.exports = (pool) => {
     // Create source
     router.post('/', async (req, res, next) => {
         try {
-            const { name, constraint_fields, config } = req.body;
+            const { name, constraint_fields, config, global_picklist } = req.body;
             if (!name || !constraint_fields || !Array.isArray(constraint_fields)) {
                 return res.status(400).json({ error: 'Missing required fields: name, constraint_fields (array)' });
             }
             const result = await pool.query(
-                `SELECT * FROM create_source(${lit(name)}, ${arr(constraint_fields)}, ${lit(config || {})})`
+                `SELECT * FROM create_source(${lit(name)}, ${arr(constraint_fields)}, ${lit(config || {})}, ${lit(global_picklist !== false)})`
             );
             res.status(201).json(result.rows[0]);
         } catch (err) {
@@ -90,9 +90,10 @@ module.exports = (pool) => {
     // Update source
     router.put('/:name', async (req, res, next) => {
         try {
-            const { constraint_fields, config } = req.body;
+            const { constraint_fields, config, global_picklist } = req.body;
+            const gpVal = global_picklist !== undefined ? lit(global_picklist) : 'NULL';
             const result = await pool.query(
-                `SELECT * FROM update_source(${lit(req.params.name)}, ${constraint_fields ? arr(constraint_fields) : 'NULL'}, ${config ? lit(config) : 'NULL'})`
+                `SELECT * FROM update_source(${lit(req.params.name)}, ${constraint_fields ? arr(constraint_fields) : 'NULL'}, ${config ? lit(config) : 'NULL'}, ${gpVal})`
             );
             if (result.rows.length === 0) return res.status(404).json({ error: 'Source not found' });
             res.json(result.rows[0]);
@@ -212,9 +213,13 @@ module.exports = (pool) => {
     // Get view data (paginated, sortable)
     router.get('/:name/view-data', async (req, res, next) => {
         try {
-            const { limit = 100, offset = 0, sort_col, sort_dir } = req.query;
+            const { limit = 100, offset = 0, sort_col, sort_dir, filters } = req.query;
+            let parsedFilters = null;
+            if (filters) {
+                try { parsedFilters = JSON.parse(filters); } catch { /* ignore bad JSON */ }
+            }
             const result = await pool.query(
-                `SELECT get_view_data(${lit(req.params.name)}, ${lit(parseInt(limit))}, ${lit(parseInt(offset))}, ${lit(sort_col || null)}, ${lit(sort_dir || 'asc')}) as result`
+                `SELECT get_view_data(${lit(req.params.name)}, ${lit(parseInt(limit))}, ${lit(parseInt(offset))}, ${lit(sort_col || null)}, ${lit(sort_dir || 'asc')}, ${parsedFilters ? lit(parsedFilters) : 'NULL'}) as result`
             );
             res.json(result.rows[0].result);
         } catch (err) {

database/queries/mappings.sql

@@ -206,3 +206,18 @@ BEGIN
     ORDER BY count(*) DESC;
 END;
 $$ LANGUAGE plpgsql;
+
+
+-- ── Global picklist ───────────────────────────────────────────────────────────
+
+CREATE OR REPLACE FUNCTION get_global_output_values()
+RETURNS TABLE (col TEXT, val TEXT) AS $$
+    SELECT DISTINCT e.key AS col, e.value AS val
+    FROM dataflow.mappings m
+    JOIN dataflow.sources s ON s.name = m.source_name
+    CROSS JOIN LATERAL jsonb_each_text(m.output) AS e(key, value)
+    WHERE s.global_picklist = true
+      AND e.value IS NOT NULL
+      AND e.value <> ''
+    ORDER BY e.key, e.value;
+$$ LANGUAGE sql STABLE;

database/queries/rules.sql

@@ -85,7 +85,7 @@ CREATE OR REPLACE FUNCTION preview_rule(
     p_replace_value TEXT DEFAULT '',
     p_limit         INT  DEFAULT 20
 )
-RETURNS TABLE (id BIGINT, raw_value TEXT, extracted_value JSONB) AS $$
+RETURNS TABLE (id INT, raw_value TEXT, extracted_value JSONB) AS $$
 BEGIN
     IF p_function_type = 'replace' THEN
         RETURN QUERY

database/queries/sources.sql

@@ -17,18 +17,19 @@ RETURNS dataflow.sources AS $$
     SELECT * FROM dataflow.sources WHERE name = p_name;
 $$ LANGUAGE sql STABLE;
 
-CREATE OR REPLACE FUNCTION create_source(p_name TEXT, p_constraint_fields TEXT[], p_config JSONB DEFAULT '{}')
+CREATE OR REPLACE FUNCTION create_source(p_name TEXT, p_constraint_fields TEXT[], p_config JSONB DEFAULT '{}', p_global_picklist BOOLEAN DEFAULT true)
 RETURNS dataflow.sources AS $$
-    INSERT INTO dataflow.sources (name, constraint_fields, config)
+    INSERT INTO dataflow.sources (name, constraint_fields, config, global_picklist)
-    VALUES (p_name, p_constraint_fields, p_config)
+    VALUES (p_name, p_constraint_fields, p_config, p_global_picklist)
     RETURNING *;
 $$ LANGUAGE sql;
 
-CREATE OR REPLACE FUNCTION update_source(p_name TEXT, p_constraint_fields TEXT[] DEFAULT NULL, p_config JSONB DEFAULT NULL)
+CREATE OR REPLACE FUNCTION update_source(p_name TEXT, p_constraint_fields TEXT[] DEFAULT NULL, p_config JSONB DEFAULT NULL, p_global_picklist BOOLEAN DEFAULT NULL)
 RETURNS dataflow.sources AS $$
     UPDATE dataflow.sources
     SET constraint_fields = COALESCE(p_constraint_fields, constraint_fields),
         config            = COALESCE(p_config, config),
+        global_picklist   = COALESCE(p_global_picklist, global_picklist),
         updated_at        = CURRENT_TIMESTAMP
     WHERE name = p_name
     RETURNING *;
@@ -41,13 +42,6 @@ $$ LANGUAGE sql;
 
 -- ── Import log ────────────────────────────────────────────────────────────────
 
-CREATE OR REPLACE FUNCTION get_import_log(p_source_name TEXT)
-RETURNS SETOF dataflow.import_log AS $$
-    SELECT * FROM dataflow.import_log
-    WHERE source_name = p_source_name
-    ORDER BY imported_at DESC;
-$$ LANGUAGE sql STABLE;
-
 -- ── Stats ─────────────────────────────────────────────────────────────────────
 
 CREATE OR REPLACE FUNCTION get_source_stats(p_source_name TEXT)
@@ -90,13 +84,18 @@ CREATE OR REPLACE FUNCTION get_view_data(
     p_limit       INT     DEFAULT 100,
     p_offset      INT     DEFAULT 0,
     p_sort_col    TEXT    DEFAULT NULL,
-    p_sort_dir    TEXT DEFAULT 'asc'
+    p_sort_dir    TEXT    DEFAULT 'asc',
+    p_filters     JSONB   DEFAULT NULL   -- [{col, pattern}, ...] — postgres regex (~*)
 )
 RETURNS JSON AS $$
 DECLARE
     v_exists  BOOLEAN;
+    v_where   TEXT := '';
     v_order   TEXT := '';
     v_rows    JSON;
+    v_filter  JSONB;
+    v_col     TEXT;
+    v_pattern TEXT;
 BEGIN
     SELECT EXISTS (
         SELECT 1 FROM information_schema.views
@@ -107,6 +106,24 @@ BEGIN
         RETURN json_build_object('exists', FALSE, 'rows', '[]'::json);
     END IF;
 
+    -- Build WHERE from filters (validate each column exists in the view)
+    IF p_filters IS NOT NULL THEN
+        FOR v_filter IN SELECT value FROM jsonb_array_elements(p_filters) LOOP
+            v_col     := v_filter->>'col';
+            v_pattern := v_filter->>'pattern';
+            IF v_pattern IS NOT NULL AND v_pattern <> '' AND EXISTS (
+                SELECT 1 FROM information_schema.columns
+                WHERE table_schema = 'dfv'
+                  AND table_name   = p_source_name
+                  AND column_name  = v_col
+            ) THEN
+                v_where := v_where ||
+                    CASE WHEN v_where = '' THEN ' WHERE ' ELSE ' AND ' END ||
+                    quote_ident(v_col) || '::text ~* ' || quote_literal(v_pattern);
+            END IF;
+        END LOOP;
+    END IF;
+
     IF p_sort_col IS NOT NULL AND EXISTS (
         SELECT 1 FROM information_schema.columns
         WHERE table_schema = 'dfv'
@@ -118,11 +135,9 @@ BEGIN
                 || ' NULLS LAST';
     END IF;
 
-    -- Subquery applies ORDER BY + LIMIT first, then json_agg collects in that order.
-    -- json_agg on the outer query preserves column order (json not jsonb).
     EXECUTE format(
-        'SELECT COALESCE(json_agg(row_to_json(t)), ''[]''::json) FROM (SELECT * FROM dfv.%I%s LIMIT %s OFFSET %s) t',
+        'SELECT COALESCE(json_agg(row_to_json(t)), ''[]''::json) FROM (SELECT * FROM dfv.%I%s%s LIMIT %s OFFSET %s) t',
-        p_source_name, v_order, p_limit, p_offset
+        p_source_name, v_where, v_order, p_limit, p_offset
     ) INTO v_rows;
 
     RETURN json_build_object('exists', TRUE, 'rows', v_rows);

database/schema.sql

@@ -17,6 +17,7 @@ CREATE TABLE sources (
     name TEXT PRIMARY KEY,
     constraint_fields TEXT[] NOT NULL,  -- Fields that uniquely identify a record (e.g., ['date', 'amount', 'description'])
     config JSONB DEFAULT '{}'::jsonb,
+    global_picklist BOOLEAN NOT NULL DEFAULT true,  -- Contribute output values to global autocomplete suggestions
     created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
 );

ui/src/api.js

@@ -10,6 +10,11 @@ export function clearCredentials() {
   _credentials = null
 }
 
+export function authHeaders() {
+  if (!_credentials) return {}
+  return { 'Authorization': `Basic ${btoa(`${_credentials.user}:${_credentials.pass}`)}` }
+}
+
 async function request(method, path, body, isFormData = false) {
   const opts = { method, headers: {} }
 
@@ -65,9 +70,10 @@ export const api = {
   reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
   generateView: (name) => request('POST', `/sources/${name}/view`),
   getFields: (name) => request('GET', `/sources/${name}/fields`),
-  getViewData: (name, limit = 100, offset = 0, sortCol = null, sortDir = 'asc') => {
+  getViewData: (name, limit = 100, offset = 0, sortCol = null, sortDir = 'asc', filters = null) => {
     const params = new URLSearchParams({ limit, offset })
     if (sortCol) { params.set('sort_col', sortCol); params.set('sort_dir', sortDir) }
+    if (filters && filters.length > 0) params.set('filters', JSON.stringify(filters))
     return request('GET', `/sources/${name}/view-data?${params}`)
   },
 
@@ -81,6 +87,7 @@ export const api = {
     request('GET', `/rules/preview?source=${encodeURIComponent(source)}&field=${encodeURIComponent(field)}&pattern=${encodeURIComponent(pattern)}&flags=${encodeURIComponent(flags || '')}&function_type=${function_type}&replace_value=${encodeURIComponent(replace_value)}&limit=${limit}`),
 
   // Mappings
+  getGlobalValues: () => request('GET', '/mappings/global-values'),
   getMappings: (source, rule) => request('GET', `/mappings/source/${source}${rule ? `?rule_name=${rule}` : ''}`),
   getMappingCounts: (source, rule) => request('GET', `/mappings/source/${source}/counts${rule ? `?rule_name=${rule}` : ''}`),
   getUnmapped: (source, rule) => request('GET', `/mappings/source/${source}/unmapped${rule ? `?rule_name=${rule}` : ''}`),

ui/src/pages/Mappings.jsx

@@ -1,5 +1,86 @@
-import { useState, useEffect } from 'react'
+import { useState, useEffect, useRef } from 'react'
-import { api } from '../api'
+import { api, authHeaders } from '../api'
+
+function AutocompleteInput({ value, onChange, onEnter, suggestions = [], className, placeholder }) {
+  const [open, setOpen] = useState(false)
+  const [highlighted, setHighlighted] = useState(0)
+  const inputRef = useRef()
+  const listRef = useRef()
+
+  const filtered = value
+    ? suggestions.filter(s => s.toLowerCase().includes(value.toLowerCase()))
+    : suggestions
+
+  function openList() {
+    setOpen(true)
+    setHighlighted(0)
+  }
+
+  function select(val) {
+    onChange(val)
+    setOpen(false)
+    inputRef.current?.focus()
+  }
+
+  function handleKeyDown(e) {
+    if (e.altKey && e.key === 'ArrowDown') {
+      e.preventDefault()
+      openList()
+      return
+    }
+    if (open && filtered.length > 0) {
+      if (e.key === 'Tab') {
+        e.preventDefault()
+        setHighlighted(h => (h + 1) % filtered.length)
+        return
+      }
+      if (e.key === 'ArrowDown') { e.preventDefault(); setHighlighted(h => Math.min(h + 1, filtered.length - 1)); return }
+      if (e.key === 'ArrowUp')   { e.preventDefault(); setHighlighted(h => Math.max(h - 1, 0)); return }
+      if (e.key === 'Enter')     { e.preventDefault(); select(filtered[highlighted]); return }
+      if (e.key === 'Escape')    { setOpen(false); return }
+    }
+    if (e.key === 'Enter') onEnter?.()
+  }
+
+  // Scroll highlighted item into view
+  useEffect(() => {
+    if (!open || !listRef.current) return
+    const item = listRef.current.children[highlighted]
+    item?.scrollIntoView({ block: 'nearest' })
+  }, [highlighted, open])
+
+  return (
+    <div className="relative">
+      <input
+        ref={inputRef}
+        className={className}
+        value={value}
+        placeholder={placeholder}
+        onChange={e => { onChange(e.target.value); if (!open && e.target.value) openList() }}
+        onKeyDown={handleKeyDown}
+        onBlur={e => { if (!listRef.current?.contains(e.relatedTarget)) setOpen(false) }}
+      />
+      {open && filtered.length > 0 && (
+        <div
+          ref={listRef}
+          className="absolute z-50 left-0 top-full mt-0.5 bg-white border border-gray-200 rounded shadow-lg max-h-48 overflow-y-auto min-w-full"
+        >
+          {filtered.map((s, i) => (
+            <div
+              key={s}
+              className={`px-2 py-1 text-xs cursor-pointer whitespace-nowrap ${
+                i === highlighted ? 'bg-blue-50 text-blue-700' : 'text-gray-700 hover:bg-gray-50'
+              }`}
+              onMouseDown={e => { e.preventDefault(); select(s) }}
+            >
+              {s}
+            </div>
+          ))}
+        </div>
+      )}
+    </div>
+  )
+}
 
 function valueKey(v) {
   return Array.isArray(v) ? JSON.stringify(v) : String(v)
@@ -35,9 +116,11 @@ export default function Mappings({ source }) {
   const [loading, setLoading] = useState(false)
   const [importing, setImporting] = useState(false)
   const [sortBy, setSortBy] = useState(null)
+  const [globalValues, setGlobalValues] = useState({})
 
   useEffect(() => {
     if (!source) return
+    api.getGlobalValues().then(setGlobalValues).catch(() => {})
     api.getRules(source).then(r => setRules(r)).catch(() => {})
   }, [source])
 
@@ -57,7 +140,7 @@ export default function Mappings({ source }) {
       .finally(() => setLoading(false))
   }, [source, selectedRule])
 
-  // Derive output columns and datalist suggestions from mapped rows
+  // Derive output columns and datalist suggestions from mapped rows + global pool
   const existingCols = []
   const valuesByCol = {}
   allValues.forEach(row => {
@@ -68,6 +151,11 @@ export default function Mappings({ source }) {
       valuesByCol[k].add(String(v))
     })
   })
+  // Merge global picklist values into suggestions
+  Object.entries(globalValues).forEach(([k, vals]) => {
+    if (!valuesByCol[k]) valuesByCol[k] = new Set()
+    vals.forEach(v => valuesByCol[k].add(v))
+  })
   const cols = [...existingCols, ...extraCols]
 
   const unmappedCount = allValues.filter(r => !r.is_mapped).length
@@ -241,13 +329,26 @@ export default function Mappings({ source }) {
 
         <div className="ml-auto flex items-center gap-2">
           {selectedRule && (
-            <a
+            <button
-              href={api.exportMappingsUrl(source, selectedRule)}
+              onClick={async () => {
-              download
+                try {
+                  const url = api.exportMappingsUrl(source, selectedRule)
+                  const res = await fetch(url, { headers: authHeaders() })
+                  if (!res.ok) throw new Error('Export failed')
+                  const blob = await res.blob()
+                  const a = document.createElement('a')
+                  a.href = URL.createObjectURL(blob)
+                  a.download = `mappings_${source}.tsv`
+                  a.click()
+                  URL.revokeObjectURL(a.href)
+                } catch (err) {
+                  alert(err.message)
+                }
+              }}
               className="text-sm px-3 py-1.5 border border-gray-200 rounded hover:bg-gray-50 text-gray-600"
             >
               Export TSV
-            </a>
+            </button>
           )}
           <label className={`text-sm px-3 py-1.5 border border-gray-200 rounded cursor-pointer hover:bg-gray-50 text-gray-600 ${importing ? 'opacity-50 pointer-events-none' : ''}`}>
             {importing ? 'Importing…' : 'Import TSV'}
@@ -269,13 +370,6 @@ export default function Mappings({ source }) {
         )}
         {selectedRule && !loading && allValues.length > 0 && (
           <div className="overflow-x-auto">
-            {cols.map(col => (
-              <datalist key={col} id={`dl-${col}`}>
-                {[...(valuesByCol[col] || [])].sort().map(v => (
-                  <option key={v} value={v} />
-                ))}
-              </datalist>
-            ))}
             <table className="w-full text-xs bg-white border border-gray-200 rounded">
               <thead>
                 <tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
@@ -297,7 +391,7 @@ export default function Mappings({ source }) {
                   <th className="px-2 py-2">
                     <button
                       onClick={() => setExtraCols(ec => [...ec, ''])}
-                      className="text-gray-300 hover:text-gray-500"
+                      className="text-gray-400 hover:text-gray-700 font-medium"
                       title="Add column"
                     >+</button>
                   </th>
@@ -328,14 +422,14 @@ export default function Mappings({ source }) {
                         <td className="px-3 py-1.5 text-right text-gray-400">{row.record_count}</td>
                         {cols.map(col => (
                           <td key={col} className="px-3 py-1.5">
-                            <input
+                            <AutocompleteInput
-                              list={`dl-${col}`}
                               className={`border rounded px-2 py-1 w-full min-w-24 focus:outline-none focus:border-blue-400 ${
                                 hasDraft ? 'border-blue-300' : row.is_mapped ? 'border-gray-200' : 'border-yellow-300'
                               }`}
                               value={cellVal(col)}
-                              onChange={e => setCellValue(row.extracted_value, col, e.target.value)}
+                              onChange={v => setCellValue(row.extracted_value, col, v)}
-                              onKeyDown={e => e.key === 'Enter' && saveRow(row)}
+                              onEnter={() => saveRow(row)}
+                              suggestions={[...(valuesByCol[col] || [])].sort()}
                             />
                           </td>
                         ))}

ui/src/pages/Records.jsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react'
+import { useState, useEffect, useRef } from 'react'
 import { api } from '../api'
 
 const DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/
@@ -20,25 +20,32 @@ function formatVal(val) {
 
 export default function Records({ source }) {
   const [rows, setRows] = useState([])
+  const [cols, setCols] = useState([])
   const [exists, setExists] = useState(null)
   const [offset, setOffset] = useState(0)
   const [loading, setLoading] = useState(false)
   const [sort, setSort] = useState({ col: null, dir: 'asc' })
+  const [filters, setFilters] = useState([])
+  const debounceRef = useRef(null)
   const LIMIT = 100
 
   useEffect(() => {
     if (!source) return
     setOffset(0)
     setSort({ col: null, dir: 'asc' })
-    load(0, null, 'asc')
+    setFilters([])
+    load(0, null, 'asc', [])
   }, [source])
 
-  async function load(off, col, dir) {
+  async function load(off, col, dir, filt) {
     setLoading(true)
     try {
-      const res = await api.getViewData(source, LIMIT, off, col, dir)
+      const active = (filt || []).filter(f => f.col && f.pattern)
+      const res = await api.getViewData(source, LIMIT, off, col, dir, active)
       setExists(res.exists)
       setRows(res.rows)
+      if (res.rows.length > 0 && cols.length === 0) setCols(Object.keys(res.rows[0]))
+      else if (res.rows.length > 0) setCols(Object.keys(res.rows[0]))
     } catch (err) {
       console.error(err)
     } finally {
@@ -46,21 +53,44 @@ export default function Records({ source }) {
     }
   }
 
+  function triggerLoad(off, col, dir, filt) {
+    clearTimeout(debounceRef.current)
+    debounceRef.current = setTimeout(() => load(off, col, dir, filt), 350)
+  }
+
   function toggleSort(col) {
     const next = sort.col === col
       ? { col, dir: sort.dir === 'asc' ? 'desc' : 'asc' }
       : { col, dir: 'asc' }
     setSort(next)
     setOffset(0)
-    load(0, next.col, next.dir)
+    load(0, next.col, next.dir, filters)
   }
 
-  function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir) }
+  function addFilter() {
-  function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir) }
+    setFilters(f => [...f, { col: cols[0] || '', pattern: '' }])
+  }
+
+  function removeFilter(i) {
+    const next = filters.filter((_, idx) => idx !== i)
+    setFilters(next)
+    setOffset(0)
+    load(0, sort.col, sort.dir, next)
+  }
+
+  function updateFilter(i, key, val) {
+    const next = filters.map((f, idx) => idx === i ? { ...f, [key]: val } : f)
+    setFilters(next)
+    setOffset(0)
+    triggerLoad(0, sort.col, sort.dir, next)
+  }
+
+  function prev() { const o = Math.max(0, offset - LIMIT); setOffset(o); load(o, sort.col, sort.dir, filters) }
+  function next() { const o = offset + LIMIT; setOffset(o); load(o, sort.col, sort.dir, filters) }
 
   if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
 
-  const cols = rows.length > 0 ? Object.keys(rows[0]) : []
+  const displayCols = rows.length > 0 ? Object.keys(rows[0]) : cols
 
   return (
     <div className="p-6">
@@ -71,6 +101,48 @@ export default function Records({ source }) {
         )}
       </div>
 
+      {/* Filter bar */}
+      {exists !== false && displayCols.length > 0 && (
+        <div className="mb-4 flex flex-wrap gap-2 items-center">
+          {filters.map((f, i) => (
+            <div key={i} className="flex items-center gap-1 bg-white border border-gray-200 rounded px-2 py-1">
+              <select
+                className="text-xs text-gray-600 border-0 focus:outline-none bg-transparent"
+                value={f.col}
+                onChange={e => updateFilter(i, 'col', e.target.value)}
+              >
+                {displayCols.map(c => <option key={c} value={c}>{c}</option>)}
+              </select>
+              <span className="text-xs text-gray-300 mx-0.5">~*</span>
+              <input
+                className="text-xs font-mono border-0 focus:outline-none w-36 bg-transparent"
+                placeholder="regex…"
+                value={f.pattern}
+                onChange={e => updateFilter(i, 'pattern', e.target.value)}
+              />
+              <button
+                onClick={() => removeFilter(i)}
+                className="text-gray-300 hover:text-gray-500 ml-1 leading-none"
+              >×</button>
+            </div>
+          ))}
+          <button
+            onClick={addFilter}
+            className="text-xs text-gray-400 hover:text-gray-600 border border-dashed border-gray-200 rounded px-2 py-1"
+          >
+            + filter
+          </button>
+          {filters.length > 0 && (
+            <button
+              onClick={() => { setFilters([]); setOffset(0); load(0, sort.col, sort.dir, []) }}
+              className="text-xs text-gray-400 hover:text-red-500"
+            >
+              clear
+            </button>
+          )}
+        </div>
+      )}
+
       {loading && <p className="text-sm text-gray-400">Loading…</p>}
 
       {!loading && exists === false && (
@@ -80,7 +152,9 @@ export default function Records({ source }) {
       )}
 
       {!loading && exists && rows.length === 0 && (
-        <p className="text-sm text-gray-400">View exists but no transformed records yet. Import data and run a transform first.</p>
+        <p className="text-sm text-gray-400">
+          {filters.some(f => f.col && f.pattern) ? 'No records match the current filters.' : 'View exists but no transformed records yet. Import data and run a transform first.'}
+        </p>
       )}
 
       {!loading && exists && rows.length > 0 && (
@@ -89,7 +163,7 @@ export default function Records({ source }) {
             <table className="w-full text-sm">
               <thead>
                 <tr className="text-left text-xs text-gray-400 border-b border-gray-100 bg-gray-50">
-                  {cols.map(col => {
+                  {displayCols.map(col => {
                     const active = sort.col === col
                     return (
                       <th
@@ -109,7 +183,7 @@ export default function Records({ source }) {
               <tbody>
                 {rows.map((row, i) => (
                   <tr key={i} className="border-t border-gray-50 hover:bg-gray-50">
-                    {cols.map((col, j) => {
+                    {displayCols.map((col, j) => {
                       const formatted = formatVal(row[col])
                       return (
                         <td key={j} className="px-3 py-2 text-xs text-gray-600 whitespace-nowrap max-w-48 truncate">

ui/src/pages/Sources.jsx

@@ -5,6 +5,7 @@ const FIELD_TYPES = ['text', 'numeric', 'date']
 
 export default function Sources({ source, sources, setSources, setSource }) {
   const [constraintFields, setConstraintFields] = useState('')
+  const [globalPicklist, setGlobalPicklist] = useState(true)
   const [schemaFields, setSchemaFields] = useState([])
   const [stats, setStats] = useState(null)
   const [saving, setSaving] = useState(false)
@@ -25,6 +26,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
   useEffect(() => {
     if (!sourceObj) return
     setConstraintFields(sourceObj.constraint_fields?.join(', ') || '')
+    setGlobalPicklist(sourceObj.global_picklist !== false)
     setSchemaFields((sourceObj.config?.fields || []).map((f, i) => ({ seq: i + 1, ...f })))
     setViewName(sourceObj.config?.fields?.length ? `dfv.${sourceObj.name}` : '')
     setResult('')
@@ -43,7 +45,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
       const constraint_fields = constraintFields.split(',').map(s => s.trim()).filter(Boolean)
       const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
       const config = { ...(sourceObj.config || {}), fields }
-      await api.updateSource(sourceObj.name, { constraint_fields, config })
+      await api.updateSource(sourceObj.name, { constraint_fields, config, global_picklist: globalPicklist })
       const updated = await api.getSources()
       setSources(updated)
       setResult('Saved.')
@@ -62,7 +64,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
       const constraint_fields = constraintFields.split(',').map(s => s.trim()).filter(Boolean)
       const fields = [...schemaFields.filter(f => f.name)].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
       const config = { ...(sourceObj.config || {}), fields }
-      await api.updateSource(sourceObj.name, { constraint_fields, config })
+      await api.updateSource(sourceObj.name, { constraint_fields, config, global_picklist: globalPicklist })
       const res = await api.generateView(sourceObj.name)
       if (res.success) {
         setViewName(res.view)
@@ -133,11 +135,11 @@ export default function Sources({ source, sources, setSources, setSource }) {
     setCreateLoading(true)
     try {
       const config = form.schema.length > 0 ? { fields: form.schema } : {}
-      await api.createSource({ name: form.name, constraint_fields: constraintArr, config })
+      await api.createSource({ name: form.name, constraint_fields: constraintArr, config, global_picklist: form.global_picklist !== false })
       const updated = await api.getSources()
       setSources(updated)
       setSource(form.name)
-      setForm({ name: '', constraint_fields: '', fields: [], schema: [] })
+      setForm({ name: '', constraint_fields: '', fields: [], schema: [], global_picklist: true })
       setCreating(false)
     } catch (err) {
       setCreateError(err.message)
@@ -273,7 +275,11 @@ export default function Sources({ source, sources, setSources, setSource }) {
                 </tbody>
               </table>
 
-              <div className="flex items-center gap-3 pt-1">
+              <div className="flex items-center gap-3 pt-1 flex-wrap">
+                <label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
+                  <input type="checkbox" checked={globalPicklist} onChange={e => setGlobalPicklist(e.target.checked)} />
+                  Global picklist
+                </label>
                 <form onSubmit={handleSave}>
                   <button type="submit" disabled={saving}
                     className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
@@ -300,12 +306,18 @@ export default function Sources({ source, sources, setSources, setSource }) {
 
           {/* Save button when no fields loaded yet */}
           {availableFields.length === 0 && (
+            <div className="flex items-center gap-3">
+              <label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
+                <input type="checkbox" checked={globalPicklist} onChange={e => setGlobalPicklist(e.target.checked)} />
+                Global picklist
+              </label>
               <form onSubmit={handleSave}>
                 <button type="submit" disabled={saving}
                   className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
                   {saving ? 'Saving…' : 'Save'}
                 </button>
               </form>
+            </div>
           )}
 
           {/* Reprocess */}
@@ -400,6 +412,15 @@ export default function Sources({ source, sources, setSources, setSource }) {
               </div>
             )}
 
+            <label className="flex items-center gap-1.5 text-xs text-gray-500 cursor-pointer">
+              <input
+                type="checkbox"
+                checked={form.global_picklist !== false}
+                onChange={e => setForm(f => ({ ...f, global_picklist: e.target.checked }))}
+              />
+              Global picklist
+            </label>
+
             {createError && <p className="text-xs text-red-500">{createError}</p>}
 
             <div className="flex gap-2">