Add Teller API sync as an alternative to CSV import

Sources with a `teller` block in their config can pull transactions
straight from the bank instead of taking a CSV upload. Only the fetch
differs — dedupe, logging, and transformation reuse the import path.

api/lib/teller.js speaks Teller's mutual-TLS protocol (client cert plus
the access token as the HTTP Basic username) and flattens transactions
into the shallow map the rule engine expects. Access tokens live in .env,
one per enrollment, not in the database that manage.py offers to reset.

Pending transactions are skipped by default: their ids change when they
post, which would import the same charge twice under two keys. Sources
should use ['id'] as constraint_fields — Teller's transaction id makes
overlapping pulls free while keeping genuinely repeated charges distinct.

Untested against the live API — Teller has no self-serve signup at the
moment, so no account to verify against.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01G2HFeU5neCKagTnmA6o9Tu

.env.example

@@ -8,3 +8,14 @@ DB_PASSWORD=your_password_here
 # API Configuration
 API_PORT=3000
 NODE_ENV=development
+
+# Teller (optional — only needed for API-based bank feeds)
+# Client certificate + key downloaded from the Teller dashboard; every API call
+# presents them (mutual TLS). Keep both outside the repo.
+TELLER_CERT_PATH=/etc/dataflow/teller/certificate.pem
+TELLER_KEY_PATH=/etc/dataflow/teller/private_key.pem
+
+# One access token per enrollment, produced by Teller Connect. A source picks
+# its token with config.teller.token_env; TELLER_TOKEN is the default.
+TELLER_TOKEN=token_here
+# TELLER_TOKEN_HUNTINGTON=token_here

api/lib/teller.js

@@ -0,0 +1,167 @@
+/**
+ * Teller API client
+ *
+ * Read-only access to enrolled bank accounts. Teller uses mutual TLS: every
+ * request presents the client certificate downloaded from the Teller dashboard,
+ * and the enrollment's access token is sent as the HTTP Basic *username* with an
+ * empty password.
+ *
+ * Uses https.request rather than fetch — Node's global fetch is undici, which
+ * ignores an https.Agent and needs its own dispatcher type for client certs.
+ */
+
+const fs = require('fs');
+const https = require('https');
+
+const API_HOST = 'api.teller.io';
+
+// Teller returns transactions newest-first with no date filter — only a count
+// and a cursor. We over-fetch and trim by date on our side.
+const DEFAULT_COUNT = 200;
+const DEFAULT_DAYS = 10;
+
+class TellerError extends Error {
+    constructor(message, status) {
+        super(message);
+        this.name = 'TellerError';
+        this.status = status;
+    }
+}
+
+let _agent = null;
+
+// The mTLS agent is built once and reused; the cert never changes at runtime.
+function getAgent() {
+    if (_agent) return _agent;
+
+    const certPath = process.env.TELLER_CERT_PATH;
+    const keyPath = process.env.TELLER_KEY_PATH;
+    if (!certPath || !keyPath) {
+        throw new TellerError('TELLER_CERT_PATH and TELLER_KEY_PATH must be set in .env', 500);
+    }
+
+    let cert, key;
+    try {
+        cert = fs.readFileSync(certPath);
+        key = fs.readFileSync(keyPath);
+    } catch (err) {
+        throw new TellerError(`Cannot read Teller certificate: ${err.message}`, 500);
+    }
+
+    _agent = new https.Agent({ cert, key, keepAlive: true });
+    return _agent;
+}
+
+// Resolve an enrollment's access token from the environment. Sources name their
+// own variable via config.teller.token_env so several banks can coexist.
+function getToken(tokenEnv) {
+    const name = tokenEnv || 'TELLER_TOKEN';
+    const token = process.env[name];
+    if (!token) {
+        throw new TellerError(`Teller access token not found — set ${name} in .env`, 500);
+    }
+    return token;
+}
+
+function get(path, token) {
+    return new Promise((resolve, reject) => {
+        const req = https.request({
+            host: API_HOST,
+            path,
+            method: 'GET',
+            agent: getAgent(),
+            auth: `${token}:`,
+            headers: { Accept: 'application/json' },
+        }, (res) => {
+            let body = '';
+            res.setEncoding('utf8');
+            res.on('data', chunk => { body += chunk; });
+            res.on('end', () => {
+                if (res.statusCode === 401) {
+                    return reject(new TellerError(
+                        'Teller rejected the access token (401) — the enrollment likely needs to be reconnected through Teller Connect', 401));
+                }
+                if (res.statusCode >= 400) {
+                    let detail = body.slice(0, 200);
+                    try { detail = JSON.parse(body).error?.message || detail; } catch { /* keep raw body */ }
+                    return reject(new TellerError(`Teller returned ${res.statusCode}: ${detail}`, res.statusCode));
+                }
+                try {
+                    resolve(JSON.parse(body));
+                } catch {
+                    reject(new TellerError('Teller returned a non-JSON response', 502));
+                }
+            });
+        });
+
+        req.on('error', err => reject(new TellerError(`Teller request failed: ${err.message}`, 502)));
+        req.end();
+    });
+}
+
+// Flatten a Teller transaction into the shallow string map the rule engine
+// expects. Nested `details` is hoisted; `links` is dropped.
+function flatten(txn) {
+    const details = txn.details || {};
+    const counterparty = details.counterparty || {};
+
+    return {
+        id: txn.id,
+        date: txn.date,
+        description: txn.description,
+        amount: txn.amount,
+        status: txn.status,
+        type: txn.type,
+        running_balance: txn.running_balance,
+        account_id: txn.account_id,
+        category: details.category,
+        processing_status: details.processing_status,
+        counterparty_name: counterparty.name,
+        counterparty_type: counterparty.type,
+    };
+}
+
+async function listAccounts(tokenEnv) {
+    const accounts = await get('/accounts', getToken(tokenEnv));
+    return accounts.map(a => ({
+        id: a.id,
+        name: a.name,
+        type: a.type,
+        subtype: a.subtype,
+        last_four: a.last_four,
+        status: a.status,
+        institution: a.institution?.name,
+    }));
+}
+
+/**
+ * Fetch transactions for one account.
+ *
+ * days           — keep transactions on or after today minus this many days; 0 fetches everything returned
+ * includePending — pending transaction ids change when they post, so they are excluded by default
+ */
+async function fetchTransactions({ accountId, tokenEnv, count, days, includePending = false }) {
+    // Callers pass values straight off the query string, so anything unparseable
+    // falls back to the default rather than becoming NaN.
+    count = Number.isFinite(count) ? count : DEFAULT_COUNT;
+    days = Number.isFinite(days) ? days : DEFAULT_DAYS;
+
+    const token = getToken(tokenEnv);
+    const txns = await get(`/accounts/${encodeURIComponent(accountId)}/transactions?count=${count}`, token);
+
+    let kept = txns;
+    if (!includePending) {
+        kept = kept.filter(t => t.status === 'posted');
+    }
+    if (days > 0) {
+        const cutoff = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
+        kept = kept.filter(t => t.date >= cutoff);  // ISO dates compare correctly as strings
+    }
+
+    return {
+        fetched: txns.length,
+        records: kept.map(flatten),
+    };
+}
+
+module.exports = { listAccounts, fetchTransactions, TellerError, DEFAULT_COUNT, DEFAULT_DAYS };

api/routes/sources.js

@@ -7,6 +7,7 @@ const express = require('express');
 const multer = require('multer');
 const { parse } = require('csv-parse/sync');
 const { lit, arr } = require('../lib/sql');
+const teller = require('../lib/teller');
 
 const upload = multer({ storage: multer.memoryStorage() });
 
@@ -23,6 +24,18 @@ module.exports = (pool) => {
         }
     });
 
+    // List the accounts behind a Teller enrollment — used to find the account_id
+    // to put in a source's config. Declared before /:name so it isn't shadowed.
+    router.get('/teller-accounts', async (req, res, next) => {
+        try {
+            const accounts = await teller.listAccounts(req.query.token_env);
+            res.json(accounts);
+        } catch (err) {
+            if (err instanceof teller.TellerError) return res.status(err.status || 502).json({ error: err.message });
+            next(err);
+        }
+    });
+
     // List all sources
     router.get('/', async (req, res, next) => {
         try {
@@ -139,6 +152,52 @@ module.exports = (pool) => {
         }
     });
 
+    // Pull transactions from Teller and import them, same as a CSV upload.
+    // Safe to re-run: overlapping transactions are skipped by constraint key.
+    router.post('/:name/sync', async (req, res, next) => {
+        try {
+            const sourceResult = await pool.query(`SELECT * FROM get_source(${lit(req.params.name)})`);
+            const source = sourceResult.rows[0];
+            if (!source || !source.name) return res.status(404).json({ error: 'Source not found' });
+
+            const cfg = (source.config || {}).teller;
+            if (!cfg || !cfg.account_id) {
+                return res.status(400).json({
+                    error: `Source "${req.params.name}" has no teller.account_id in its config`
+                });
+            }
+
+            const opts = { ...req.query, ...req.body };
+            const { fetched, records } = await teller.fetchTransactions({
+                accountId: cfg.account_id,
+                tokenEnv: cfg.token_env,
+                count: opts.count !== undefined ? parseInt(opts.count) : cfg.count,
+                days: opts.days !== undefined ? parseInt(opts.days) : cfg.days,
+                includePending: opts.include_pending === true || opts.include_pending === 'true',
+            });
+
+            if (records.length === 0) {
+                return res.json({ success: true, fetched, imported: 0, duplicates: 0 });
+            }
+
+            const importResult = await pool.query(
+                `SELECT import_records(${lit(req.params.name)}, ${lit(records)}) as result`
+            );
+            const importData = importResult.rows[0].result;
+
+            if (!importData.success) return res.json({ ...importData, fetched });
+
+            const transformResult = await pool.query(
+                `SELECT apply_transformations(${lit(req.params.name)}) as result`
+            );
+
+            res.json({ ...importData, fetched, transform: transformResult.rows[0].result });
+        } catch (err) {
+            if (err instanceof teller.TellerError) return res.status(err.status || 502).json({ error: err.message });
+            next(err);
+        }
+    });
+
     // Get import log
     router.get('/:name/import-log', async (req, res, next) => {
         try {

docs/spec.md

@@ -48,6 +48,7 @@ api/
     auth.js            — Basic Auth enforcement on all /api routes
   lib/
     sql.js             — lit() and arr() helpers for SQL literal building
+    teller.js          — Teller API client (mutual TLS) for bank transaction pulls
   routes/
     sources.js         — HTTP handlers for source management
     rules.js           — HTTP handlers for rule management
@@ -62,7 +63,7 @@ ui/
     pages/
       Login.jsx        — username/password form
       Sources.jsx      — source CRUD, field config, view generation
-      Import.jsx       — CSV upload and import log
+      Import.jsx       — CSV upload, Teller sync, and import log
       Rules.jsx        — rule CRUD with live pattern preview
       Mappings.jsx     — mapping table with TSV import/export
       Records.jsx      — paginated, sortable view of transformed records
@@ -112,6 +113,41 @@ CSV file → parse in Node.js → import_records(source, data)
   → apply_transformations() runs automatically on new records
 ```
 
+### Teller sync (API-based bank feeds)
+```
+POST /api/sources/:name/sync → api/lib/teller.js
+  → GET api.teller.io/accounts/:id/transactions (mutual TLS + access token)
+  → drop pending, trim to the last N days, flatten nested `details`
+  → import_records(source, data) — identical path to a CSV import from here on
+```
+An alternative to CSV upload for sources that read from a bank API. Only the
+fetching differs: dedup, logging, and transformation are the same code.
+
+- **Authentication.** Teller uses mutual TLS. Every request presents the client
+  certificate and key from the Teller dashboard (`TELLER_CERT_PATH`,
+  `TELLER_KEY_PATH`), with the enrollment's access token as the HTTP Basic
+  *username* and an empty password. Access tokens come from the Teller Connect
+  browser flow, run once per bank, and live in `.env` — one variable per
+  enrollment. They are deliberately **not** stored in the database, which
+  `manage.py` offers to reset.
+- **Source config.** A source opts in by having `teller` in its `config` JSONB:
+  `{"teller": {"account_id": "acc_…", "token_env": "TELLER_TOKEN_HUNTINGTON",
+  "days": 10, "count": 200}}`. Only `account_id` is required; `token_env`
+  defaults to `TELLER_TOKEN`. `GET /api/sources/teller-accounts` lists the
+  accounts behind a token so you can find the id.
+- **`constraint_fields` should be `['id']`.** Teller assigns each transaction a
+  stable id, which makes overlapping pulls free and — unlike date + amount +
+  description — keeps genuinely repeated charges as separate records.
+- **Pending transactions are skipped** (`?include_pending=true` overrides). A
+  pending transaction's id changes when it posts, so importing it would produce
+  a duplicate under a different key a day or two later.
+- **Rolling window, not a cursor.** Teller's transaction endpoint takes a count,
+  not a date range, so the route over-fetches (`count`, default 200) and trims
+  to `days` (default 10, `0` for everything returned). Re-running the same
+  window is harmless; late-arriving transactions get picked up.
+- **Cron.** A daily pull is just the endpoint:
+  `curl -sS -u user:pass -X POST http://localhost:3000/api/sources/NAME/sync`
+
 ### Transform
 ```
 apply_transformations(source) — pure SQL CTE
@@ -173,6 +209,8 @@ All routes are under `/api`. Every route requires HTTP Basic Auth. The `GET /hea
 | DELETE | /api/sources/:name | Delete source and all its data |
 | POST | /api/sources/suggest | Suggest source config from an uploaded CSV |
 | POST | /api/sources/:name/import | Import CSV; transformations are applied to the new records |
+| POST | /api/sources/:name/sync | Pull transactions from Teller and import them (`?days=`, `?count=`, `?include_pending=`) |
+| GET | /api/sources/teller-accounts | List accounts behind a Teller enrollment (`?token_env=`) |
 | GET | /api/sources/import-log | Import history across all sources |
 | GET | /api/sources/:name/import-log | Import history for one source |
 | DELETE | /api/sources/:name/import-log/:id | Delete an import batch and every record in it |
@@ -288,7 +326,7 @@ Built with React + Vite + Tailwind CSS. Compiled output goes to `public/`. The s
 
 - **Sources** — View and edit source configuration. Shows all known field names and their origins (raw data, schema, rules, mappings). Checkboxes control which fields are constraint fields and which appear in the output view. Supports CSV upload to auto-detect fields.
 
-- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation.
+- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation. Sources with `config.teller.account_id` also get a Sync panel — a window selector (10/30/90 days or everything) and a "Sync now" button that pulls from the bank API through the same import path.
 
 - **Rules** — Create and manage regex rules. Live preview fires automatically (debounced 500ms) as pattern/field/flags are edited, showing match results against real records. Rules can be enabled/disabled by toggle.
 
@@ -377,6 +415,10 @@ API_PORT             Port the Express server listens on (default 3020)
 NODE_ENV             development | production
 LOGIN_USER           Username for Basic Auth
 LOGIN_PASSWORD_HASH  bcrypt hash of the password
+
+TELLER_CERT_PATH     Teller client certificate (only for Teller sync)
+TELLER_KEY_PATH      Teller private key
+TELLER_TOKEN         Default Teller access token; per-source override via config.teller.token_env
 ```
 
 ---

ui/src/api.js

@@ -66,6 +66,12 @@ export const api = {
     fd.append('file', file)
     return request('POST', `/sources/${name}/import`, fd, true)
   },
+  syncTeller: (name, opts = {}) => {
+    const params = new URLSearchParams(opts)
+    return request('POST', `/sources/${name}/sync${params.toString() ? `?${params}` : ''}`)
+  },
+  getTellerAccounts: (tokenEnv) =>
+    request('GET', `/sources/teller-accounts${tokenEnv ? `?token_env=${encodeURIComponent(tokenEnv)}` : ''}`),
   transform: (name) => request('POST', `/sources/${name}/transform`),
   reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
   generateView: (name) => request('POST', `/sources/${name}/view`),

ui/src/pages/Import.jsx

@@ -67,12 +67,15 @@ export default function Import({ source }) {
   const [error, setError] = useState('')
   const [dragOver, setDragOver] = useState(false)
   const [selected, setSelected] = useState(new Set())
+  const [teller, setTeller] = useState(null)
+  const [days, setDays] = useState('10')
   const fileRef = useRef()
 
   useEffect(() => {
     if (!source) return
     api.getStats(source).then(setStats).catch(() => {})
     api.getImportLog(source).then(setLog).catch(() => {})
+    api.getSource(source).then(s => setTeller(s.config?.teller || null)).catch(() => setTeller(null))
     setSelected(new Set())
   }, [source])
 
@@ -93,6 +96,23 @@ export default function Import({ source }) {
     }
   }
 
+  async function handleSync() {
+    if (!source) return
+    setLoading(true)
+    setError('')
+    setResult(null)
+    try {
+      const res = await api.syncTeller(source, { days })
+      setResult(res)
+      api.getStats(source).then(setStats)
+      api.getImportLog(source).then(setLog)
+    } catch (err) {
+      setError(err.message)
+    } finally {
+      setLoading(false)
+    }
+  }
+
   async function handleTransform() {
     if (!source) return
     setLoading(true)
@@ -170,6 +190,30 @@ export default function Import({ source }) {
         </div>
       )}
 
+      {/* Teller sync — only for sources with a teller account in their config */}
+      {teller?.account_id && (
+        <div className="bg-white border border-gray-200 rounded p-4 mb-4 flex items-center gap-3">
+          <div className="flex-1">
+            <div className="text-sm font-medium text-gray-700">Teller</div>
+            <div className="text-xs text-gray-400 font-mono">{teller.account_id}</div>
+          </div>
+          <select
+            value={days}
+            onChange={e => setDays(e.target.value)}
+            className="text-sm border border-gray-200 rounded px-2 py-1.5 bg-white text-gray-700"
+          >
+            <option value="10">Last 10 days</option>
+            <option value="30">Last 30 days</option>
+            <option value="90">Last 90 days</option>
+            <option value="0">Everything available</option>
+          </select>
+          <button onClick={handleSync} disabled={loading}
+            className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
+            {loading ? 'Syncing…' : 'Sync now'}
+          </button>
+        </div>
+      )}
+
       {/* Drop zone */}
       <div
         className={`border-2 border-dashed rounded-lg p-8 text-center mb-4 cursor-pointer transition-colors ${
@@ -215,6 +259,12 @@ export default function Import({ source }) {
             </>
           ) : result.imported !== undefined ? (
             <>
+              {result.fetched !== undefined && (
+                <>
+                  <span className="text-gray-500">{result.fetched} fetched</span>
+                  <span className="text-gray-400 mx-2">·</span>
+                </>
+              )}
               <span className="text-green-600 font-medium">{result.imported} imported</span>
               <span className="text-gray-400 mx-2">·</span>
               <span className="text-gray-500">{result.duplicates} duplicates skipped</span>