Add SimpleFIN Bridge sync as an alternative to CSV import

Sources with a `simplefin` block in their config can pull transactions
straight from the bridge instead of taking a CSV upload. Only the fetch
differs — dedupe, logging, and transformation reuse the import path.

The access URL is the whole credential, so it lives in .env rather than
the database that manage.py offers to reset. Claiming a setup token is
exposed as an endpoint because the token is single-use and easy to burn.

The bridge answers 200 with a populated `errors` array when a bank is
failing, which would otherwise read as a successful empty pull — those
errors ride along in the sync response and show on the Import page.

Pending transactions are skipped by default: they get a new id once they
post, which would import the same charge twice under two keys. Sources
should use ['id'] as constraint_fields — the transaction id makes
overlapping pulls free while keeping genuinely repeated charges distinct.

Verified against a stubbed bridge response, not a live account.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01G2HFeU5neCKagTnmA6o9Tu

.env.example

@@ -8,3 +8,10 @@ DB_PASSWORD=your_password_here
 # API Configuration
 API_PORT=3000
 NODE_ENV=development
+
+# SimpleFIN (optional — only needed for API-based bank feeds)
+# The access URL is the credential: claim a setup token once via
+# POST /api/sources/simplefin-claim and paste the result here.
+# A source picks its bridge with config.simplefin.access_url_env;
+# SIMPLEFIN_ACCESS_URL is the default.
+SIMPLEFIN_ACCESS_URL=https://user:pass@bridge.simplefin.org/simplefin

api/lib/simplefin.js

@@ -0,0 +1,182 @@
+/**
+ * SimpleFIN Bridge client
+ *
+ * Read-only access to linked bank accounts. Auth is a single "access URL" with
+ * credentials embedded (https://user:pass@bridge.simplefin.org/simplefin),
+ * obtained once by claiming a setup token. No certificates, no refresh flow —
+ * the URL is the credential, so it lives in .env and never in the database.
+ *
+ * Protocol: https://www.simplefin.org/protocol.html
+ */
+
+const REQUEST_TIMEOUT = 30000;
+
+// The bridge refreshes from banks roughly daily and transactions can land a few
+// days late, so pulls overlap by design and lean on constraint_key for dedupe.
+const DEFAULT_DAYS = 10;
+
+class SimpleFinError extends Error {
+    constructor(message, status) {
+        super(message);
+        this.name = 'SimpleFinError';
+        this.status = status;
+    }
+}
+
+// Resolve an access URL from the environment. Sources name their own variable
+// via config.simplefin.access_url_env so several bridges can coexist.
+function getAccessUrl(urlEnv) {
+    const name = urlEnv || 'SIMPLEFIN_ACCESS_URL';
+    const value = process.env[name];
+    if (!value) {
+        throw new SimpleFinError(`SimpleFIN access URL not found — set ${name} in .env`, 500);
+    }
+
+    let parsed;
+    try {
+        parsed = new URL(value);
+    } catch {
+        throw new SimpleFinError(`${name} is not a valid URL`, 500);
+    }
+    if (!parsed.username) {
+        throw new SimpleFinError(`${name} has no credentials — expected https://user:pass@host/path`, 500);
+    }
+    return parsed;
+}
+
+async function get(accessUrl, path, params) {
+    // Credentials travel in the Authorization header, not the request line.
+    const target = new URL(accessUrl.pathname + path, accessUrl.origin);
+    for (const [k, v] of Object.entries(params || {})) {
+        if (v !== undefined && v !== null) target.searchParams.append(k, String(v));
+    }
+    const auth = Buffer.from(`${decodeURIComponent(accessUrl.username)}:${decodeURIComponent(accessUrl.password)}`).toString('base64');
+
+    let res;
+    try {
+        res = await fetch(target, {
+            headers: { Authorization: `Basic ${auth}`, Accept: 'application/json' },
+            signal: AbortSignal.timeout(REQUEST_TIMEOUT),
+        });
+    } catch (err) {
+        throw new SimpleFinError(`SimpleFIN request failed: ${err.message}`, 502);
+    }
+
+    if (res.status === 401 || res.status === 403) {
+        throw new SimpleFinError(
+            'SimpleFIN rejected the access URL — it may have been revoked; claim a new setup token', res.status);
+    }
+    if (!res.ok) {
+        throw new SimpleFinError(`SimpleFIN returned ${res.status}: ${(await res.text()).slice(0, 200)}`, res.status);
+    }
+
+    try {
+        return await res.json();
+    } catch {
+        throw new SimpleFinError('SimpleFIN returned a non-JSON response', 502);
+    }
+}
+
+/**
+ * Exchange a setup token for a permanent access URL. Run once per bridge; the
+ * returned URL goes in .env. The token is base64 of a one-shot claim URL and is
+ * consumed by this call.
+ */
+async function claimSetupToken(setupToken) {
+    let claimUrl;
+    try {
+        claimUrl = Buffer.from(String(setupToken).trim(), 'base64').toString('utf8');
+        new URL(claimUrl);
+    } catch {
+        throw new SimpleFinError('Setup token is not valid base64 of a claim URL', 400);
+    }
+
+    let res;
+    try {
+        res = await fetch(claimUrl, { method: 'POST', signal: AbortSignal.timeout(REQUEST_TIMEOUT) });
+    } catch (err) {
+        throw new SimpleFinError(`Claim request failed: ${err.message}`, 502);
+    }
+    if (!res.ok) {
+        throw new SimpleFinError(
+            `Claim returned ${res.status} — setup tokens can only be claimed once`, res.status);
+    }
+    return (await res.text()).trim();
+}
+
+// Epoch seconds → YYYY-MM-DD, so dates sort and compare as plain text the way
+// CSV-imported dates already do.
+function toDate(epochSeconds) {
+    if (epochSeconds === undefined || epochSeconds === null) return null;
+    return new Date(epochSeconds * 1000).toISOString().slice(0, 10);
+}
+
+// Flatten a SimpleFIN transaction into the shallow string map the rule engine
+// expects. Account context is folded in so records stay self-describing.
+function flatten(txn, account) {
+    return {
+        id: txn.id,
+        date: toDate(txn.posted),
+        transacted_at: toDate(txn.transacted_at),
+        description: txn.description,
+        payee: txn.payee,
+        memo: txn.memo,
+        amount: txn.amount,
+        pending: txn.pending ? 'true' : 'false',
+        account_id: account.id,
+        account_name: account.name,
+        organization: account.org?.name || account.org?.domain,
+    };
+}
+
+async function listAccounts(urlEnv) {
+    const data = await get(getAccessUrl(urlEnv), '/accounts', { 'balances-only': 1 });
+    return {
+        errors: data.errors || [],
+        accounts: (data.accounts || []).map(a => ({
+            id: a.id,
+            name: a.name,
+            organization: a.org?.name || a.org?.domain,
+            currency: a.currency,
+            balance: a.balance,
+            available_balance: a['available-balance'],
+            balance_date: toDate(a['balance-date']),
+        })),
+    };
+}
+
+/**
+ * Fetch transactions for one account.
+ *
+ * days           — how far back to ask for; 0 lets the bridge return its default window
+ * includePending — pending transactions get a new id once they post, so they are excluded by default
+ */
+async function fetchTransactions({ accountId, accessUrlEnv, days, includePending = false }) {
+    days = Number.isFinite(days) ? days : DEFAULT_DAYS;
+
+    const params = { account: accountId };
+    if (days > 0) params['start-date'] = Math.floor(Date.now() / 1000) - days * 86400;
+    if (includePending) params.pending = 1;
+
+    const data = await get(getAccessUrl(accessUrlEnv), '/accounts', params);
+
+    // The bridge reports per-institution problems here and still returns 200 —
+    // one bank being down must not look like a successful empty pull.
+    const errors = data.errors || [];
+    const account = (data.accounts || []).find(a => a.id === accountId);
+    if (!account) {
+        const detail = errors.length ? ` — bridge reported: ${errors.join('; ')}` : '';
+        throw new SimpleFinError(`Account ${accountId} not returned by SimpleFIN${detail}`, 502);
+    }
+
+    const txns = account.transactions || [];
+    const kept = includePending ? txns : txns.filter(t => !t.pending);
+
+    return {
+        fetched: txns.length,
+        errors,
+        records: kept.map(t => flatten(t, account)),
+    };
+}
+
+module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS };

api/routes/sources.js

@@ -7,6 +7,7 @@ const express = require('express');
 const multer = require('multer');
 const { parse } = require('csv-parse/sync');
 const { lit, arr } = require('../lib/sql');
+const simplefin = require('../lib/simplefin');
 
 const upload = multer({ storage: multer.memoryStorage() });
 
@@ -23,6 +24,30 @@ module.exports = (pool) => {
         }
     });
 
+    // SimpleFIN helpers. Declared before /:name so they aren't shadowed by it.
+
+    // List the accounts behind a bridge — used to find the account_id for a source
+    router.get('/simplefin-accounts', async (req, res, next) => {
+        try {
+            res.json(await simplefin.listAccounts(req.query.access_url_env));
+        } catch (err) {
+            if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
+            next(err);
+        }
+    });
+
+    // Exchange a one-shot setup token for the permanent access URL to put in .env
+    router.post('/simplefin-claim', async (req, res, next) => {
+        try {
+            const { setup_token } = req.body || {};
+            if (!setup_token) return res.status(400).json({ error: 'setup_token is required' });
+            res.json({ access_url: await simplefin.claimSetupToken(setup_token) });
+        } catch (err) {
+            if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
+            next(err);
+        }
+    });
+
     // List all sources
     router.get('/', async (req, res, next) => {
         try {
@@ -139,6 +164,51 @@ module.exports = (pool) => {
         }
     });
 
+    // Pull transactions from SimpleFIN and import them, same as a CSV upload.
+    // Safe to re-run: overlapping transactions are skipped by constraint key.
+    router.post('/:name/sync', async (req, res, next) => {
+        try {
+            const sourceResult = await pool.query(`SELECT * FROM get_source(${lit(req.params.name)})`);
+            const source = sourceResult.rows[0];
+            if (!source || !source.name) return res.status(404).json({ error: 'Source not found' });
+
+            const cfg = (source.config || {}).simplefin;
+            if (!cfg || !cfg.account_id) {
+                return res.status(400).json({
+                    error: `Source "${req.params.name}" has no simplefin.account_id in its config`
+                });
+            }
+
+            const opts = { ...req.query, ...req.body };
+            const { fetched, errors, records } = await simplefin.fetchTransactions({
+                accountId: cfg.account_id,
+                accessUrlEnv: cfg.access_url_env,
+                days: opts.days !== undefined ? parseInt(opts.days) : cfg.days,
+                includePending: opts.include_pending === true || opts.include_pending === 'true',
+            });
+
+            if (records.length === 0) {
+                return res.json({ success: true, fetched, errors, imported: 0, duplicates: 0 });
+            }
+
+            const importResult = await pool.query(
+                `SELECT import_records(${lit(req.params.name)}, ${lit(records)}) as result`
+            );
+            const importData = importResult.rows[0].result;
+
+            if (!importData.success) return res.json({ ...importData, fetched, errors });
+
+            const transformResult = await pool.query(
+                `SELECT apply_transformations(${lit(req.params.name)}) as result`
+            );
+
+            res.json({ ...importData, fetched, errors, transform: transformResult.rows[0].result });
+        } catch (err) {
+            if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
+            next(err);
+        }
+    });
+
     // Get import log
     router.get('/:name/import-log', async (req, res, next) => {
         try {

docs/spec.md

@@ -48,6 +48,7 @@ api/
     auth.js            — Basic Auth enforcement on all /api routes
   lib/
     sql.js             — lit() and arr() helpers for SQL literal building
+    simplefin.js       — SimpleFIN Bridge client for bank transaction pulls
   routes/
     sources.js         — HTTP handlers for source management
     rules.js           — HTTP handlers for rule management
@@ -62,7 +63,7 @@ ui/
     pages/
       Login.jsx        — username/password form
       Sources.jsx      — source CRUD, field config, view generation
-      Import.jsx       — CSV upload and import log
+      Import.jsx       — CSV upload, SimpleFIN sync, and import log
       Rules.jsx        — rule CRUD with live pattern preview
       Mappings.jsx     — mapping table with TSV import/export
       Records.jsx      — paginated, sortable view of transformed records
@@ -112,6 +113,41 @@ CSV file → parse in Node.js → import_records(source, data)
   → apply_transformations() runs automatically on new records
 ```
 
+### SimpleFIN sync (API-based bank feeds)
+```
+POST /api/sources/:name/sync → api/lib/simplefin.js
+  → GET {access_url}/accounts?account=…&start-date=… (Basic auth)
+  → drop pending, flatten transactions, fold in account context
+  → import_records(source, data) — identical path to a CSV import from here on
+```
+An alternative to CSV upload for sources that read from a bank API. Only the
+fetching differs: dedup, logging, and transformation are the same code.
+
+- **Authentication.** A SimpleFIN access URL *is* the credential — it carries
+  its own username and password (`https://user:pass@bridge.simplefin.org/simplefin`).
+  You claim it once from a setup token (`POST /api/sources/simplefin-claim`,
+  which consumes the token) and store it in `.env`, one variable per bridge. It
+  is deliberately **not** stored in the database, which `manage.py` offers to reset.
+- **Source config.** A source opts in by having `simplefin` in its `config` JSONB:
+  `{"simplefin": {"account_id": "ACT-…", "access_url_env": "SIMPLEFIN_ACCESS_URL",
+  "days": 10}}`. Only `account_id` is required. `GET /api/sources/simplefin-accounts`
+  lists the accounts behind a bridge so you can find the id.
+- **`constraint_fields` should be `['id']`.** SimpleFIN assigns each transaction a
+  stable id, which makes overlapping pulls free and — unlike date + amount +
+  description — keeps genuinely repeated charges as separate records.
+- **Pending transactions are skipped** (`?include_pending=true` overrides). A
+  pending transaction gets a different id once it posts, so importing it would
+  produce a duplicate under a different key a day or two later.
+- **Bridge errors are surfaced, not swallowed.** SimpleFIN returns HTTP 200 with
+  an `errors` array when an institution is failing. Those errors ride along in
+  the sync response so a broken connection doesn't read as a successful empty
+  pull; the Import page shows them in orange above the counts.
+- **Refresh cadence.** The bridge polls banks roughly daily and transactions can
+  take a few days to appear, so the pull asks for a rolling window (`days`,
+  default 10, `0` for whatever the bridge returns) rather than tracking a cursor.
+- **Cron.** A daily pull is just the endpoint:
+  `curl -sS -u user:pass -X POST http://localhost:3000/api/sources/NAME/sync`
+
 ### Transform
 ```
 apply_transformations(source) — pure SQL CTE
@@ -173,6 +209,9 @@ All routes are under `/api`. Every route requires HTTP Basic Auth. The `GET /hea
 | DELETE | /api/sources/:name | Delete source and all its data |
 | POST | /api/sources/suggest | Suggest source config from an uploaded CSV |
 | POST | /api/sources/:name/import | Import CSV; transformations are applied to the new records |
+| POST | /api/sources/:name/sync | Pull transactions from SimpleFIN and import them (`?days=`, `?include_pending=`) |
+| GET | /api/sources/simplefin-accounts | List accounts behind a bridge (`?access_url_env=`) |
+| POST | /api/sources/simplefin-claim | Exchange a setup token for a permanent access URL |
 | GET | /api/sources/import-log | Import history across all sources |
 | GET | /api/sources/:name/import-log | Import history for one source |
 | DELETE | /api/sources/:name/import-log/:id | Delete an import batch and every record in it |
@@ -288,7 +327,7 @@ Built with React + Vite + Tailwind CSS. Compiled output goes to `public/`. The s
 
 - **Sources** — View and edit source configuration. Shows all known field names and their origins (raw data, schema, rules, mappings). Checkboxes control which fields are constraint fields and which appear in the output view. Supports CSV upload to auto-detect fields.
 
-- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation.
+- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation. Sources with `config.simplefin.account_id` also get a Sync panel — a window selector (10/30/90 days or everything) and a "Sync now" button that pulls from the bank API through the same import path.
 
 - **Rules** — Create and manage regex rules. Live preview fires automatically (debounced 500ms) as pattern/field/flags are edited, showing match results against real records. Rules can be enabled/disabled by toggle.
 
@@ -377,6 +416,8 @@ API_PORT             Port the Express server listens on (default 3020)
 NODE_ENV             development | production
 LOGIN_USER           Username for Basic Auth
 LOGIN_PASSWORD_HASH  bcrypt hash of the password
+
+SIMPLEFIN_ACCESS_URL Default SimpleFIN bridge URL; per-source override via config.simplefin.access_url_env
 ```
 
 ---

ui/src/api.js

@@ -66,6 +66,13 @@ export const api = {
     fd.append('file', file)
     return request('POST', `/sources/${name}/import`, fd, true)
   },
+  syncSimpleFin: (name, opts = {}) => {
+    const params = new URLSearchParams(opts)
+    return request('POST', `/sources/${name}/sync${params.toString() ? `?${params}` : ''}`)
+  },
+  getSimpleFinAccounts: (accessUrlEnv) =>
+    request('GET', `/sources/simplefin-accounts${accessUrlEnv ? `?access_url_env=${encodeURIComponent(accessUrlEnv)}` : ''}`),
+  claimSimpleFinToken: (setup_token) => request('POST', '/sources/simplefin-claim', { setup_token }),
   transform: (name) => request('POST', `/sources/${name}/transform`),
   reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
   generateView: (name) => request('POST', `/sources/${name}/view`),

ui/src/pages/Import.jsx

@@ -67,12 +67,15 @@ export default function Import({ source }) {
   const [error, setError] = useState('')
   const [dragOver, setDragOver] = useState(false)
   const [selected, setSelected] = useState(new Set())
+  const [simplefin, setSimplefin] = useState(null)
+  const [days, setDays] = useState('10')
   const fileRef = useRef()
 
   useEffect(() => {
     if (!source) return
     api.getStats(source).then(setStats).catch(() => {})
     api.getImportLog(source).then(setLog).catch(() => {})
+    api.getSource(source).then(s => setSimplefin(s.config?.simplefin || null)).catch(() => setSimplefin(null))
     setSelected(new Set())
   }, [source])
 
@@ -93,6 +96,23 @@ export default function Import({ source }) {
     }
   }
 
+  async function handleSync() {
+    if (!source) return
+    setLoading(true)
+    setError('')
+    setResult(null)
+    try {
+      const res = await api.syncSimpleFin(source, { days })
+      setResult(res)
+      api.getStats(source).then(setStats)
+      api.getImportLog(source).then(setLog)
+    } catch (err) {
+      setError(err.message)
+    } finally {
+      setLoading(false)
+    }
+  }
+
   async function handleTransform() {
     if (!source) return
     setLoading(true)
@@ -170,6 +190,30 @@ export default function Import({ source }) {
         </div>
       )}
 
+      {/* SimpleFIN sync — only for sources with a bridge account in their config */}
+      {simplefin?.account_id && (
+        <div className="bg-white border border-gray-200 rounded p-4 mb-4 flex items-center gap-3">
+          <div className="flex-1 min-w-0">
+            <div className="text-sm font-medium text-gray-700">SimpleFIN</div>
+            <div className="text-xs text-gray-400 font-mono truncate">{simplefin.account_id}</div>
+          </div>
+          <select
+            value={days}
+            onChange={e => setDays(e.target.value)}
+            className="text-sm border border-gray-200 rounded px-2 py-1.5 bg-white text-gray-700"
+          >
+            <option value="10">Last 10 days</option>
+            <option value="30">Last 30 days</option>
+            <option value="90">Last 90 days</option>
+            <option value="0">Everything available</option>
+          </select>
+          <button onClick={handleSync} disabled={loading}
+            className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
+            {loading ? 'Syncing…' : 'Sync now'}
+          </button>
+        </div>
+      )}
+
       {/* Drop zone */}
       <div
         className={`border-2 border-dashed rounded-lg p-8 text-center mb-4 cursor-pointer transition-colors ${
@@ -215,6 +259,17 @@ export default function Import({ source }) {
             </>
           ) : result.imported !== undefined ? (
             <>
+              {result.errors?.length > 0 && (
+                <div className="mb-2 text-xs text-orange-600">
+                  {result.errors.map((e, i) => <div key={i}>Bridge: {e}</div>)}
+                </div>
+              )}
+              {result.fetched !== undefined && (
+                <>
+                  <span className="text-gray-500">{result.fetched} fetched</span>
+                  <span className="text-gray-400 mx-2">·</span>
+                </>
+              )}
               <span className="text-green-600 font-medium">{result.imported} imported</span>
               <span className="text-gray-400 mx-2">·</span>
               <span className="text-gray-500">{result.duplicates} duplicates skipped</span>