Merge SimpleFIN Bridge bank feed integration
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01G2HFeU5neCKagTnmA6o9Tu
This commit is contained in:
commit
43d968b248
@ -8,3 +8,10 @@ DB_PASSWORD=your_password_here
|
||||
# API Configuration
|
||||
API_PORT=3000
|
||||
NODE_ENV=development
|
||||
|
||||
# SimpleFIN (optional — only needed for API-based bank feeds)
|
||||
# The access URL is the credential: claim a setup token once via
|
||||
# POST /api/sources/simplefin-claim and paste the result here.
|
||||
# A source picks its bridge with config.simplefin.access_url_env;
|
||||
# SIMPLEFIN_ACCESS_URL is the default.
|
||||
SIMPLEFIN_ACCESS_URL=https://user:pass@bridge.simplefin.org/simplefin
|
||||
|
||||
42
api/lib/fields.js
Normal file
42
api/lib/fields.js
Normal file
@ -0,0 +1,42 @@
|
||||
/**
|
||||
* Field inference
|
||||
* Derives a field list and column types from sample records, so a source can be
|
||||
* configured from real data rather than an assumed shape. Used by the CSV
|
||||
* suggest endpoint and by bank-feed sampling.
|
||||
*/
|
||||
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/;
|
||||
|
||||
/**
|
||||
* @param records array of flat objects
|
||||
* @param limit how many rows to look at
|
||||
* @returns { fields: [{name, type}], sampleRows }
|
||||
*
|
||||
* Keys are unioned across the sample, not read off the first row — API feeds
|
||||
* omit optional keys entirely on records that lack them.
|
||||
*/
|
||||
function inferFields(records, limit = 50) {
|
||||
const sampleRows = records.slice(0, limit);
|
||||
|
||||
const keys = [];
|
||||
for (const row of sampleRows) {
|
||||
for (const key of Object.keys(row)) {
|
||||
if (!keys.includes(key)) keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const fields = keys.map(key => {
|
||||
const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null);
|
||||
let type = 'text';
|
||||
if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) {
|
||||
type = 'numeric';
|
||||
} else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) {
|
||||
type = 'date';
|
||||
}
|
||||
return { name: key, type };
|
||||
});
|
||||
|
||||
return { fields, sampleRows };
|
||||
}
|
||||
|
||||
module.exports = { inferFields };
|
||||
219
api/lib/simplefin.js
Normal file
219
api/lib/simplefin.js
Normal file
@ -0,0 +1,219 @@
|
||||
/**
|
||||
* SimpleFIN Bridge client
|
||||
*
|
||||
* Read-only access to linked bank accounts. Auth is a single "access URL" with
|
||||
* credentials embedded (https://user:pass@bridge.simplefin.org/simplefin),
|
||||
* obtained once by claiming a setup token. No certificates, no refresh flow —
|
||||
* the URL is the credential, so it lives in .env and never in the database.
|
||||
*
|
||||
* Protocol: https://www.simplefin.org/protocol.html
|
||||
*/
|
||||
|
||||
const REQUEST_TIMEOUT = 30000;
|
||||
|
||||
// The bridge refreshes from banks roughly daily and transactions can land a few
|
||||
// days late, so pulls overlap by design and lean on constraint_key for dedupe.
|
||||
const DEFAULT_DAYS = 10;
|
||||
|
||||
// The bridge hard-caps ranges at 90 days and reports the capping in `errors`;
|
||||
// 89 stays just inside it so a routine backfill doesn't look like data loss. It
|
||||
// also advises staying under 45 days, which it says on anything longer — that
|
||||
// notice is passed through, since it is a real hint about future behaviour.
|
||||
// Omitting start-date entirely is not "everything": it returns only the most
|
||||
// recent handful of transactions, so a start-date is always sent.
|
||||
const MAX_DAYS = 89;
|
||||
|
||||
class SimpleFinError extends Error {
|
||||
constructor(message, status) {
|
||||
super(message);
|
||||
this.name = 'SimpleFinError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve an access URL from the environment. Sources name their own variable
|
||||
// via config.simplefin.access_url_env so several bridges can coexist.
|
||||
function getAccessUrl(urlEnv) {
|
||||
const name = urlEnv || 'SIMPLEFIN_ACCESS_URL';
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
throw new SimpleFinError(`SimpleFIN access URL not found — set ${name} in .env`, 500);
|
||||
}
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch {
|
||||
throw new SimpleFinError(`${name} is not a valid URL`, 500);
|
||||
}
|
||||
if (!parsed.username) {
|
||||
throw new SimpleFinError(`${name} has no credentials — expected https://user:pass@host/path`, 500);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function get(accessUrl, path, params) {
|
||||
// Credentials travel in the Authorization header, not the request line.
|
||||
const target = new URL(accessUrl.pathname + path, accessUrl.origin);
|
||||
for (const [k, v] of Object.entries(params || {})) {
|
||||
if (v !== undefined && v !== null) target.searchParams.append(k, String(v));
|
||||
}
|
||||
const auth = Buffer.from(`${decodeURIComponent(accessUrl.username)}:${decodeURIComponent(accessUrl.password)}`).toString('base64');
|
||||
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(target, {
|
||||
headers: { Authorization: `Basic ${auth}`, Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT),
|
||||
});
|
||||
} catch (err) {
|
||||
throw new SimpleFinError(`SimpleFIN request failed: ${err.message}`, 502);
|
||||
}
|
||||
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
throw new SimpleFinError(
|
||||
'SimpleFIN rejected the access URL — it may have been revoked; claim a new setup token', res.status);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new SimpleFinError(`SimpleFIN returned ${res.status}: ${(await res.text()).slice(0, 200)}`, res.status);
|
||||
}
|
||||
|
||||
try {
|
||||
return await res.json();
|
||||
} catch {
|
||||
throw new SimpleFinError('SimpleFIN returned a non-JSON response', 502);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exchange a setup token for a permanent access URL. Run once per bridge; the
|
||||
* returned URL goes in .env. The token is base64 of a one-shot claim URL and is
|
||||
* consumed by this call.
|
||||
*/
|
||||
async function claimSetupToken(setupToken) {
|
||||
let claimUrl;
|
||||
try {
|
||||
claimUrl = Buffer.from(String(setupToken).trim(), 'base64').toString('utf8');
|
||||
new URL(claimUrl);
|
||||
} catch {
|
||||
throw new SimpleFinError('Setup token is not valid base64 of a claim URL', 400);
|
||||
}
|
||||
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(claimUrl, { method: 'POST', signal: AbortSignal.timeout(REQUEST_TIMEOUT) });
|
||||
} catch (err) {
|
||||
throw new SimpleFinError(`Claim request failed: ${err.message}`, 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new SimpleFinError(
|
||||
`Claim returned ${res.status} — setup tokens can only be claimed once`, res.status);
|
||||
}
|
||||
return (await res.text()).trim();
|
||||
}
|
||||
|
||||
// Epoch seconds → YYYY-MM-DD, so dates sort and compare as plain text the way
|
||||
// CSV-imported dates already do. Pending transactions carry posted=0 rather than
|
||||
// omitting it, so falsy epochs are "no date", not 1970-01-01.
|
||||
function toDate(epochSeconds) {
|
||||
if (!epochSeconds) return null;
|
||||
return new Date(epochSeconds * 1000).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
// Flatten a SimpleFIN transaction into the shallow map the rule engine expects.
|
||||
//
|
||||
// Every scalar the bridge sends is passed through rather than whitelisted, so
|
||||
// institution-specific fields (mcc, and whatever a given bank adds) show up in
|
||||
// the data and in field discovery without a code change here. Only the epoch
|
||||
// timestamps are reshaped, and account context is folded in so records stay
|
||||
// self-describing.
|
||||
function flatten(txn, account) {
|
||||
const out = {};
|
||||
|
||||
for (const [key, value] of Object.entries(txn)) {
|
||||
if (value === null || value === undefined) continue;
|
||||
if (typeof value === 'object') continue; // nested objects handled below
|
||||
out[key] = value;
|
||||
}
|
||||
|
||||
// `extra` is free-form and institution-specific; hoist its scalars under a
|
||||
// prefix so they can't collide with the documented fields
|
||||
for (const [key, value] of Object.entries(txn.extra || {})) {
|
||||
if (value !== null && value !== undefined && typeof value !== 'object') {
|
||||
out[`extra_${key}`] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const posted = toDate(txn.posted);
|
||||
const transacted = toDate(txn.transacted_at);
|
||||
|
||||
delete out.posted; // replaced by the YYYY-MM-DD forms below
|
||||
|
||||
// A pending transaction has no posted date yet; fall back to when it was
|
||||
// transacted so every record has something usable to sort and filter on
|
||||
out.date = posted || transacted;
|
||||
out.posted_date = posted;
|
||||
out.transacted_at = transacted;
|
||||
out.pending = txn.pending ? 'true' : 'false';
|
||||
|
||||
out.account_id = account.id;
|
||||
out.account_name = account.name;
|
||||
out.organization = account.org?.name || account.org?.domain;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
async function listAccounts(urlEnv) {
|
||||
const data = await get(getAccessUrl(urlEnv), '/accounts', { 'balances-only': 1 });
|
||||
return {
|
||||
errors: data.errors || [],
|
||||
accounts: (data.accounts || []).map(a => ({
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
organization: a.org?.name || a.org?.domain,
|
||||
currency: a.currency,
|
||||
balance: a.balance,
|
||||
available_balance: a['available-balance'],
|
||||
balance_date: toDate(a['balance-date']),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch transactions for one account.
|
||||
*
|
||||
* days — how far back to ask for; 0 or more than MAX_DAYS means the full 90
|
||||
* includePending — pending transactions get a new id once they post, so they are excluded by default
|
||||
*/
|
||||
async function fetchTransactions({ accountId, accessUrlEnv, days, includePending = false }) {
|
||||
days = Number.isFinite(days) ? days : DEFAULT_DAYS;
|
||||
if (days <= 0 || days > MAX_DAYS) days = MAX_DAYS;
|
||||
|
||||
const params = {
|
||||
account: accountId,
|
||||
'start-date': Math.floor(Date.now() / 1000) - days * 86400,
|
||||
};
|
||||
if (includePending) params.pending = 1;
|
||||
|
||||
const data = await get(getAccessUrl(accessUrlEnv), '/accounts', params);
|
||||
|
||||
// The bridge reports per-institution problems here and still returns 200 —
|
||||
// one bank being down must not look like a successful empty pull.
|
||||
const errors = data.errors || [];
|
||||
const account = (data.accounts || []).find(a => a.id === accountId);
|
||||
if (!account) {
|
||||
const detail = errors.length ? ` — bridge reported: ${errors.join('; ')}` : '';
|
||||
throw new SimpleFinError(`Account ${accountId} not returned by SimpleFIN${detail}`, 502);
|
||||
}
|
||||
|
||||
const txns = account.transactions || [];
|
||||
const kept = includePending ? txns : txns.filter(t => !t.pending);
|
||||
|
||||
return {
|
||||
fetched: txns.length,
|
||||
errors,
|
||||
records: kept.map(t => flatten(t, account)),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS, MAX_DAYS };
|
||||
@ -7,6 +7,8 @@ const express = require('express');
|
||||
const multer = require('multer');
|
||||
const { parse } = require('csv-parse/sync');
|
||||
const { lit, arr } = require('../lib/sql');
|
||||
const simplefin = require('../lib/simplefin');
|
||||
const { inferFields } = require('../lib/fields');
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage() });
|
||||
|
||||
@ -23,6 +25,56 @@ module.exports = (pool) => {
|
||||
}
|
||||
});
|
||||
|
||||
// SimpleFIN helpers. Declared before /:name so they aren't shadowed by it.
|
||||
|
||||
// List the accounts behind a bridge — used to find the account_id for a source
|
||||
router.get('/simplefin-accounts', async (req, res, next) => {
|
||||
try {
|
||||
res.json(await simplefin.listAccounts(req.query.access_url_env));
|
||||
} catch (err) {
|
||||
if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Sample an account's real transactions and infer its field list — the API
|
||||
// equivalent of uploading a CSV to /suggest. Whatever the account actually
|
||||
// returns is what gets offered, so investment or loan accounts describe
|
||||
// themselves rather than being forced into a checking-account shape.
|
||||
router.get('/simplefin-sample', async (req, res, next) => {
|
||||
try {
|
||||
const { account_id, access_url_env } = req.query;
|
||||
if (!account_id) return res.status(400).json({ error: 'account_id is required' });
|
||||
|
||||
const { fetched, records, errors } = await simplefin.fetchTransactions({
|
||||
accountId: account_id,
|
||||
accessUrlEnv: access_url_env,
|
||||
// The bridge advises staying under 45 days and warns at exactly
|
||||
// 45, so 44 is the largest quiet sample window
|
||||
days: req.query.days !== undefined ? parseInt(req.query.days) : 44,
|
||||
includePending: true, // widen the sample; pending rows can carry extra keys
|
||||
});
|
||||
|
||||
const { fields, sampleRows } = inferFields(records);
|
||||
res.json({ fields, sampleRows, fetched, errors });
|
||||
} catch (err) {
|
||||
if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Exchange a one-shot setup token for the permanent access URL to put in .env
|
||||
router.post('/simplefin-claim', async (req, res, next) => {
|
||||
try {
|
||||
const { setup_token } = req.body || {};
|
||||
if (!setup_token) return res.status(400).json({ error: 'setup_token is required' });
|
||||
res.json({ access_url: await simplefin.claimSetupToken(setup_token) });
|
||||
} catch (err) {
|
||||
if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// List all sources
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
@ -52,21 +104,7 @@ module.exports = (pool) => {
|
||||
const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true });
|
||||
if (records.length === 0) return res.status(400).json({ error: 'CSV file is empty' });
|
||||
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/;
|
||||
const sample = records[0];
|
||||
const sampleRows = records.slice(0, 50);
|
||||
|
||||
const fields = Object.keys(sample).map(key => {
|
||||
const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null);
|
||||
let type = 'text';
|
||||
if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) {
|
||||
type = 'numeric';
|
||||
} else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) {
|
||||
type = 'date';
|
||||
}
|
||||
return { name: key, type };
|
||||
});
|
||||
|
||||
const { fields, sampleRows } = inferFields(records);
|
||||
res.json({ name: '', constraint_fields: [], fields, sampleRows });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
@ -139,6 +177,51 @@ module.exports = (pool) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Pull transactions from SimpleFIN and import them, same as a CSV upload.
|
||||
// Safe to re-run: overlapping transactions are skipped by constraint key.
|
||||
router.post('/:name/sync', async (req, res, next) => {
|
||||
try {
|
||||
const sourceResult = await pool.query(`SELECT * FROM get_source(${lit(req.params.name)})`);
|
||||
const source = sourceResult.rows[0];
|
||||
if (!source || !source.name) return res.status(404).json({ error: 'Source not found' });
|
||||
|
||||
const cfg = (source.config || {}).simplefin;
|
||||
if (!cfg || !cfg.account_id) {
|
||||
return res.status(400).json({
|
||||
error: `Source "${req.params.name}" has no simplefin.account_id in its config`
|
||||
});
|
||||
}
|
||||
|
||||
const opts = { ...req.query, ...req.body };
|
||||
const { fetched, errors, records } = await simplefin.fetchTransactions({
|
||||
accountId: cfg.account_id,
|
||||
accessUrlEnv: cfg.access_url_env,
|
||||
days: opts.days !== undefined ? parseInt(opts.days) : cfg.days,
|
||||
includePending: opts.include_pending === true || opts.include_pending === 'true',
|
||||
});
|
||||
|
||||
if (records.length === 0) {
|
||||
return res.json({ success: true, fetched, errors, imported: 0, duplicates: 0 });
|
||||
}
|
||||
|
||||
const importResult = await pool.query(
|
||||
`SELECT import_records(${lit(req.params.name)}, ${lit(records)}) as result`
|
||||
);
|
||||
const importData = importResult.rows[0].result;
|
||||
|
||||
if (!importData.success) return res.json({ ...importData, fetched, errors });
|
||||
|
||||
const transformResult = await pool.query(
|
||||
`SELECT apply_transformations(${lit(req.params.name)}) as result`
|
||||
);
|
||||
|
||||
res.json({ ...importData, fetched, errors, transform: transformResult.rows[0].result });
|
||||
} catch (err) {
|
||||
if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Get import log
|
||||
router.get('/:name/import-log', async (req, res, next) => {
|
||||
try {
|
||||
|
||||
53
docs/spec.md
53
docs/spec.md
@ -48,6 +48,7 @@ api/
|
||||
auth.js — Basic Auth enforcement on all /api routes
|
||||
lib/
|
||||
sql.js — lit() and arr() helpers for SQL literal building
|
||||
simplefin.js — SimpleFIN Bridge client for bank transaction pulls
|
||||
routes/
|
||||
sources.js — HTTP handlers for source management
|
||||
rules.js — HTTP handlers for rule management
|
||||
@ -62,7 +63,7 @@ ui/
|
||||
pages/
|
||||
Login.jsx — username/password form
|
||||
Sources.jsx — source CRUD, field config, view generation
|
||||
Import.jsx — CSV upload and import log
|
||||
Import.jsx — CSV upload, SimpleFIN sync, and import log
|
||||
Rules.jsx — rule CRUD with live pattern preview
|
||||
Mappings.jsx — mapping table with TSV import/export
|
||||
Records.jsx — paginated, sortable view of transformed records
|
||||
@ -112,6 +113,45 @@ CSV file → parse in Node.js → import_records(source, data)
|
||||
→ apply_transformations() runs automatically on new records
|
||||
```
|
||||
|
||||
### SimpleFIN sync (API-based bank feeds)
|
||||
```
|
||||
POST /api/sources/:name/sync → api/lib/simplefin.js
|
||||
→ GET {access_url}/accounts?account=…&start-date=… (Basic auth)
|
||||
→ drop pending, flatten transactions, fold in account context
|
||||
→ import_records(source, data) — identical path to a CSV import from here on
|
||||
```
|
||||
An alternative to CSV upload for sources that read from a bank API. Only the
|
||||
fetching differs: dedup, logging, and transformation are the same code.
|
||||
|
||||
- **Authentication.** A SimpleFIN access URL *is* the credential — it carries
|
||||
its own username and password (`https://user:pass@bridge.simplefin.org/simplefin`).
|
||||
You claim it once from a setup token (`POST /api/sources/simplefin-claim`,
|
||||
which consumes the token) and store it in `.env`, one variable per bridge. It
|
||||
is deliberately **not** stored in the database, which `manage.py` offers to reset.
|
||||
- **Source config.** A source opts in by having `simplefin` in its `config` JSONB:
|
||||
`{"simplefin": {"account_id": "ACT-…", "access_url_env": "SIMPLEFIN_ACCESS_URL",
|
||||
"days": 10}}`. Only `account_id` is required. `GET /api/sources/simplefin-accounts`
|
||||
lists the accounts behind a bridge so you can find the id.
|
||||
- **`constraint_fields` should be `['id']`.** SimpleFIN assigns each transaction a
|
||||
stable id, which makes overlapping pulls free and — unlike date + amount +
|
||||
description — keeps genuinely repeated charges as separate records.
|
||||
- **Pending transactions are skipped** (`?include_pending=true` overrides). A
|
||||
pending transaction gets a different id once it posts, so importing it would
|
||||
produce a duplicate under a different key a day or two later.
|
||||
- **Bridge errors are surfaced, not swallowed.** SimpleFIN returns HTTP 200 with
|
||||
an `errors` array when an institution is failing. Those errors ride along in
|
||||
the sync response so a broken connection doesn't read as a successful empty
|
||||
pull; the Import page shows them in orange above the counts.
|
||||
- **Refresh cadence and the 90-day wall.** The bridge polls banks roughly daily
|
||||
and transactions can take a few days to appear, so the pull asks for a rolling
|
||||
window (`days`, default 10) rather than tracking a cursor. The window has hard
|
||||
limits: SimpleFIN caps any range at 90 days and advises staying under 45, so
|
||||
`MAX_DAYS` is 89 and `days=0` or anything larger clamps to it. A `start-date`
|
||||
is **always** sent — omitting it does not mean "everything available", it
|
||||
returns only the few most recent transactions.
|
||||
- **Cron.** A daily pull is just the endpoint:
|
||||
`curl -sS -u user:pass -X POST http://localhost:3000/api/sources/NAME/sync`
|
||||
|
||||
### Transform
|
||||
```
|
||||
apply_transformations(source) — pure SQL CTE
|
||||
@ -173,6 +213,9 @@ All routes are under `/api`. Every route requires HTTP Basic Auth. The `GET /hea
|
||||
| DELETE | /api/sources/:name | Delete source and all its data |
|
||||
| POST | /api/sources/suggest | Suggest source config from an uploaded CSV |
|
||||
| POST | /api/sources/:name/import | Import CSV; transformations are applied to the new records |
|
||||
| POST | /api/sources/:name/sync | Pull transactions from SimpleFIN and import them (`?days=`, `?include_pending=`) |
|
||||
| GET | /api/sources/simplefin-accounts | List accounts behind a bridge (`?access_url_env=`) |
|
||||
| POST | /api/sources/simplefin-claim | Exchange a setup token for a permanent access URL |
|
||||
| GET | /api/sources/import-log | Import history across all sources |
|
||||
| GET | /api/sources/:name/import-log | Import history for one source |
|
||||
| DELETE | /api/sources/:name/import-log/:id | Delete an import batch and every record in it |
|
||||
@ -288,7 +331,7 @@ Built with React + Vite + Tailwind CSS. Compiled output goes to `public/`. The s
|
||||
|
||||
- **Sources** — View and edit source configuration. Shows all known field names and their origins (raw data, schema, rules, mappings). Checkboxes control which fields are constraint fields and which appear in the output view. Supports CSV upload to auto-detect fields.
|
||||
|
||||
- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation.
|
||||
- **Import** — Upload a CSV to import records into the selected source. Transformations run automatically on new records. Shows import log with inserted/duplicate counts, expandable key detail, checkbox selection, and delete with confirmation. Sources with `config.simplefin.account_id` also get a Sync panel — a window selector (10/30/90 days or everything) and a "Sync now" button that pulls from the bank API through the same import path.
|
||||
|
||||
- **Rules** — Create and manage regex rules. Live preview fires automatically (debounced 500ms) as pattern/field/flags are edited, showing match results against real records. Rules can be enabled/disabled by toggle.
|
||||
|
||||
@ -354,7 +397,9 @@ Shows current status on every screen:
|
||||
|
||||
9. **Set login credentials** — Prompts for username and password, bcrypt-hashes the password via `node -e "require('bcrypt')..."`, and writes `LOGIN_USER` and `LOGIN_PASSWORD_HASH` to `.env`. Requires Node.js and bcrypt npm package to be installed.
|
||||
|
||||
10. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place.
|
||||
10. **Claim SimpleFIN setup token** — Prompts for a setup token (hidden input), exchanges it for a permanent access URL via `api/lib/simplefin.js`, and writes `SIMPLEFIN_ACCESS_URL` to `.env`. Warns and confirms before replacing an existing URL. Setup tokens are single-use, so a failed claim generally means a new token is needed. Prints only the bridge host, never the embedded credentials.
|
||||
|
||||
11. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place.
|
||||
|
||||
**Key behaviors:**
|
||||
- All commands that will be run are printed before the user is asked to confirm.
|
||||
@ -377,6 +422,8 @@ API_PORT Port the Express server listens on (default 3020)
|
||||
NODE_ENV development | production
|
||||
LOGIN_USER Username for Basic Auth
|
||||
LOGIN_PASSWORD_HASH bcrypt hash of the password
|
||||
|
||||
SIMPLEFIN_ACCESS_URL Default SimpleFIN bridge URL; per-source override via config.simplefin.access_url_env
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
53
manage.py
53
manage.py
@ -902,6 +902,58 @@ def action_set_login_credentials(cfg):
|
||||
info('Restart the service for changes to take effect (option 7).')
|
||||
|
||||
|
||||
def action_claim_simplefin(cfg):
|
||||
header('Claim a SimpleFIN setup token')
|
||||
print(' A setup token is single-use. Claiming it returns the permanent access')
|
||||
print(' URL, which is written to .env as SIMPLEFIN_ACCESS_URL.')
|
||||
print()
|
||||
|
||||
if not ENV_FILE.exists():
|
||||
err(f'{ENV_FILE} does not exist — run the database configuration dialog first')
|
||||
return
|
||||
|
||||
if cfg and cfg.get('SIMPLEFIN_ACCESS_URL'):
|
||||
warn('SIMPLEFIN_ACCESS_URL is already set — claiming again will replace it')
|
||||
if not confirm('Replace the existing access URL?', default_yes=False):
|
||||
info('Cancelled — no changes made')
|
||||
return
|
||||
|
||||
token = prompt('Setup token', secret=True)
|
||||
if not token:
|
||||
info('Cancelled — no changes made')
|
||||
return
|
||||
|
||||
print(' Claiming token with SimpleFIN...')
|
||||
r = subprocess.run(
|
||||
['node', '-e',
|
||||
"require('./api/lib/simplefin.js').claimSetupToken(process.argv[1])"
|
||||
".then(u=>process.stdout.write(u),e=>{process.stderr.write(e.message);process.exit(1)})",
|
||||
token],
|
||||
capture_output=True, text=True, cwd=ROOT
|
||||
)
|
||||
if r.returncode != 0 or not r.stdout:
|
||||
err(f'Claim failed — the token may already have been used\n {r.stderr.strip()}')
|
||||
return
|
||||
|
||||
access_url = r.stdout.strip()
|
||||
|
||||
# Update .env
|
||||
env_text = ENV_FILE.read_text()
|
||||
key = 'SIMPLEFIN_ACCESS_URL'
|
||||
if f'{key}=' in env_text:
|
||||
import re
|
||||
env_text = re.sub(rf'^{key}=.*$', f'{key}={access_url}', env_text, flags=re.MULTILINE)
|
||||
else:
|
||||
env_text = env_text.rstrip('\n') + f'\n{key}={access_url}\n'
|
||||
ENV_FILE.write_text(env_text)
|
||||
|
||||
# Show the host only — the URL embeds its own credentials
|
||||
host = access_url.split('@')[-1] if '@' in access_url else access_url
|
||||
ok(f'{key} written to {ENV_FILE}')
|
||||
info(f'Bridge: {host}')
|
||||
info('Restart the service for changes to take effect (option 7).')
|
||||
|
||||
|
||||
# ── Main menu ─────────────────────────────────────────────────────────────────
|
||||
|
||||
MENU = [
|
||||
@ -914,6 +966,7 @@ MENU = [
|
||||
('Start / restart dataflow.service', action_restart_service),
|
||||
('Stop dataflow.service', action_stop_service),
|
||||
('Set login credentials', action_set_login_credentials),
|
||||
('Claim SimpleFIN setup token (.env)', action_claim_simplefin),
|
||||
('Uninstall (service, nginx, database, .env, build)', action_uninstall),
|
||||
]
|
||||
|
||||
|
||||
@ -66,6 +66,18 @@ export const api = {
|
||||
fd.append('file', file)
|
||||
return request('POST', `/sources/${name}/import`, fd, true)
|
||||
},
|
||||
syncSimpleFin: (name, opts = {}) => {
|
||||
const params = new URLSearchParams(opts)
|
||||
return request('POST', `/sources/${name}/sync${params.toString() ? `?${params}` : ''}`)
|
||||
},
|
||||
getSimpleFinSample: (accountId, days) => {
|
||||
const params = new URLSearchParams({ account_id: accountId })
|
||||
if (days !== undefined) params.set('days', days)
|
||||
return request('GET', `/sources/simplefin-sample?${params}`)
|
||||
},
|
||||
getSimpleFinAccounts: (accessUrlEnv) =>
|
||||
request('GET', `/sources/simplefin-accounts${accessUrlEnv ? `?access_url_env=${encodeURIComponent(accessUrlEnv)}` : ''}`),
|
||||
claimSimpleFinToken: (setup_token) => request('POST', '/sources/simplefin-claim', { setup_token }),
|
||||
transform: (name) => request('POST', `/sources/${name}/transform`),
|
||||
reprocess: (name) => request('POST', `/sources/${name}/reprocess`),
|
||||
generateView: (name) => request('POST', `/sources/${name}/view`),
|
||||
|
||||
@ -67,12 +67,15 @@ export default function Import({ source }) {
|
||||
const [error, setError] = useState('')
|
||||
const [dragOver, setDragOver] = useState(false)
|
||||
const [selected, setSelected] = useState(new Set())
|
||||
const [simplefin, setSimplefin] = useState(null)
|
||||
const [days, setDays] = useState('10')
|
||||
const fileRef = useRef()
|
||||
|
||||
useEffect(() => {
|
||||
if (!source) return
|
||||
api.getStats(source).then(setStats).catch(() => {})
|
||||
api.getImportLog(source).then(setLog).catch(() => {})
|
||||
api.getSource(source).then(s => setSimplefin(s.config?.simplefin || null)).catch(() => setSimplefin(null))
|
||||
setSelected(new Set())
|
||||
}, [source])
|
||||
|
||||
@ -93,6 +96,23 @@ export default function Import({ source }) {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSync() {
|
||||
if (!source) return
|
||||
setLoading(true)
|
||||
setError('')
|
||||
setResult(null)
|
||||
try {
|
||||
const res = await api.syncSimpleFin(source, { days })
|
||||
setResult(res)
|
||||
api.getStats(source).then(setStats)
|
||||
api.getImportLog(source).then(setLog)
|
||||
} catch (err) {
|
||||
setError(err.message)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTransform() {
|
||||
if (!source) return
|
||||
setLoading(true)
|
||||
@ -170,6 +190,30 @@ export default function Import({ source }) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* SimpleFIN sync — only for sources with a bridge account in their config */}
|
||||
{simplefin?.account_id && (
|
||||
<div className="bg-white border border-gray-200 rounded p-4 mb-4 flex items-center gap-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-medium text-gray-700">SimpleFIN</div>
|
||||
<div className="text-xs text-gray-400 font-mono truncate">{simplefin.account_id}</div>
|
||||
</div>
|
||||
<select
|
||||
value={days}
|
||||
onChange={e => setDays(e.target.value)}
|
||||
className="text-sm border border-gray-200 rounded px-2 py-1.5 bg-white text-gray-700"
|
||||
>
|
||||
<option value="10">Last 10 days</option>
|
||||
<option value="30">Last 30 days</option>
|
||||
<option value="45">Last 45 days</option>
|
||||
<option value="89">Backfill (bridge maximum)</option>
|
||||
</select>
|
||||
<button onClick={handleSync} disabled={loading}
|
||||
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
||||
{loading ? 'Syncing…' : 'Sync now'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Drop zone */}
|
||||
<div
|
||||
className={`border-2 border-dashed rounded-lg p-8 text-center mb-4 cursor-pointer transition-colors ${
|
||||
@ -215,6 +259,17 @@ export default function Import({ source }) {
|
||||
</>
|
||||
) : result.imported !== undefined ? (
|
||||
<>
|
||||
{result.errors?.length > 0 && (
|
||||
<div className="mb-2 text-xs text-orange-600">
|
||||
{result.errors.map((e, i) => <div key={i}>Bridge: {e}</div>)}
|
||||
</div>
|
||||
)}
|
||||
{result.fetched !== undefined && (
|
||||
<>
|
||||
<span className="text-gray-500">{result.fetched} fetched</span>
|
||||
<span className="text-gray-400 mx-2">·</span>
|
||||
</>
|
||||
)}
|
||||
<span className="text-green-600 font-medium">{result.imported} imported</span>
|
||||
<span className="text-gray-400 mx-2">·</span>
|
||||
<span className="text-gray-500">{result.duplicates} duplicates skipped</span>
|
||||
|
||||
@ -4,6 +4,9 @@ import { api } from '../api'
|
||||
|
||||
const FIELD_TYPES = ['text', 'numeric', 'date']
|
||||
|
||||
// Ticked into the view by default when a bank feed sample contains them
|
||||
const FEED_DEFAULT_VIEW = ['date', 'description', 'payee', 'amount']
|
||||
|
||||
function SampleTable({ rows }) {
|
||||
if (!rows || rows.length === 0) return null
|
||||
const cols = Object.keys(rows[0])
|
||||
@ -50,6 +53,10 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
const [createError, setCreateError] = useState('')
|
||||
const [createLoading, setCreateLoading] = useState(false)
|
||||
const [csvFileName, setCsvFileName] = useState('')
|
||||
const [bridgeAccounts, setBridgeAccounts] = useState(null)
|
||||
const [bridgeLoading, setBridgeLoading] = useState(false)
|
||||
const [bridgeError, setBridgeError] = useState('')
|
||||
const [sampleInfo, setSampleInfo] = useState(null)
|
||||
const fileRef = useRef()
|
||||
|
||||
const [searchParams, setSearchParams] = useSearchParams()
|
||||
@ -73,6 +80,8 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
setStats(null)
|
||||
setAvailableFields([])
|
||||
setSampleRows([])
|
||||
setBridgeAccounts(null)
|
||||
setBridgeError('')
|
||||
api.getStats(sourceObj.name).then(setStats).catch(() => {})
|
||||
api.getFields(sourceObj.name).then(setAvailableFields).catch(() => {})
|
||||
api.getRecords(sourceObj.name, 50).then(rows => setSampleRows(rows.map(r => r.data).filter(Boolean))).catch(() => {})
|
||||
@ -153,6 +162,79 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
}
|
||||
}
|
||||
|
||||
// The bridge is an external call, so accounts are fetched on demand rather
|
||||
// than on every visit to this page.
|
||||
async function loadBridgeAccounts() {
|
||||
setBridgeLoading(true)
|
||||
setBridgeError('')
|
||||
try {
|
||||
const res = await api.getSimpleFinAccounts()
|
||||
setBridgeAccounts(res.accounts || [])
|
||||
if (res.errors?.length) setBridgeError(res.errors.join('; '))
|
||||
} catch (err) {
|
||||
setBridgeError(err.message)
|
||||
} finally {
|
||||
setBridgeLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Writes only config — constraint_fields and global_picklist are left NULL so
|
||||
// update_source keeps whatever is already stored.
|
||||
async function handleLinkAccount(accountId) {
|
||||
setError('')
|
||||
setResult('')
|
||||
try {
|
||||
const config = { ...(sourceObj.config || {}) }
|
||||
if (accountId) {
|
||||
config.simplefin = { ...(config.simplefin || {}), account_id: accountId }
|
||||
} else {
|
||||
delete config.simplefin
|
||||
}
|
||||
await api.updateSource(sourceObj.name, { config })
|
||||
setSources(await api.getSources())
|
||||
setResult(accountId ? 'Account linked.' : 'Account unlinked.')
|
||||
} catch (err) {
|
||||
setError(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
// Picking a feed account samples its real transactions and fills the field
|
||||
// table from them — the same path a CSV takes through /suggest.
|
||||
async function handleSelectFeedAccount(accountId) {
|
||||
if (!accountId) {
|
||||
// Clearing the feed only resets fields we populated, not a loaded CSV
|
||||
setForm(f => csvFileName ? { ...f, simplefin_account_id: '' } : {
|
||||
...f, simplefin_account_id: '', fields: [], schema: [], constraint_fields: '', sampleRows: [],
|
||||
})
|
||||
setSampleInfo(null)
|
||||
return
|
||||
}
|
||||
|
||||
setForm(f => ({ ...f, simplefin_account_id: accountId }))
|
||||
setBridgeLoading(true)
|
||||
setBridgeError('')
|
||||
try {
|
||||
const res = await api.getSimpleFinSample(accountId)
|
||||
const names = res.fields.map(f => f.name)
|
||||
setSampleInfo({ fetched: res.fetched, fields: res.fields.length })
|
||||
if (res.errors?.length) setBridgeError(res.errors.join('; '))
|
||||
setForm(f => ({
|
||||
...f,
|
||||
fields: res.fields,
|
||||
sampleRows: res.sampleRows || [],
|
||||
schema: FEED_DEFAULT_VIEW.filter(n => names.includes(n)).map((name, i) => ({
|
||||
name, type: res.fields.find(sf => sf.name === name).type, seq: i + 1,
|
||||
})),
|
||||
// Only default the constraint if the sample actually has an id
|
||||
constraint_fields: f.constraint_fields || (names.includes('id') ? 'id' : ''),
|
||||
}))
|
||||
} catch (err) {
|
||||
setBridgeError(err.message)
|
||||
} finally {
|
||||
setBridgeLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSuggest(e) {
|
||||
const file = e.target.files[0]
|
||||
if (!file) return
|
||||
@ -182,6 +264,9 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
setCreateLoading(true)
|
||||
try {
|
||||
const config = form.schema.length > 0 ? { fields: form.schema } : {}
|
||||
if (form.simplefin_account_id) {
|
||||
config.simplefin = { account_id: form.simplefin_account_id }
|
||||
}
|
||||
await api.createSource({ name: form.name, constraint_fields: constraintArr, config, global_picklist: form.global_picklist !== false })
|
||||
if (form.schema.length > 0) {
|
||||
await api.generateView(form.name)
|
||||
@ -192,7 +277,7 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
const updated = await api.getSources()
|
||||
setSources(updated)
|
||||
setSource(form.name)
|
||||
setForm({ name: '', constraint_fields: '', fields: [], schema: [], importSample: true })
|
||||
setForm({ name: '', constraint_fields: '', fields: [], schema: [], importSample: true, simplefin_account_id: '' })
|
||||
setCreating(false)
|
||||
} catch (err) {
|
||||
setCreateError(err.message)
|
||||
@ -232,6 +317,52 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Bank feed — link this source to a SimpleFIN account */}
|
||||
<div className="pt-2 border-t border-gray-100">
|
||||
<div className="flex items-center gap-3 flex-wrap">
|
||||
<div className="text-xs font-medium text-gray-600">Bank feed</div>
|
||||
|
||||
{bridgeAccounts === null ? (
|
||||
<>
|
||||
<span className="text-xs text-gray-500 font-mono">
|
||||
{sourceObj.config?.simplefin?.account_id || 'not linked'}
|
||||
</span>
|
||||
<button
|
||||
onClick={loadBridgeAccounts}
|
||||
disabled={bridgeLoading}
|
||||
className="text-xs border border-gray-300 rounded px-2 py-1 text-gray-600 hover:bg-gray-50 hover:border-gray-400 disabled:opacity-50"
|
||||
>
|
||||
{bridgeLoading ? 'Loading…' : sourceObj.config?.simplefin?.account_id ? 'Change' : 'Link SimpleFIN account'}
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<select
|
||||
value={sourceObj.config?.simplefin?.account_id || ''}
|
||||
onChange={e => handleLinkAccount(e.target.value)}
|
||||
className="text-xs border border-gray-200 rounded px-2 py-1 bg-white text-gray-700"
|
||||
>
|
||||
<option value="">Not linked</option>
|
||||
{bridgeAccounts.map(a => (
|
||||
<option key={a.id} value={a.id}>
|
||||
{a.name}{a.organization ? ` — ${a.organization}` : ''}{a.balance ? ` (${a.balance})` : ''}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{bridgeError && <p className="text-xs text-orange-600 mt-1">{bridgeError}</p>}
|
||||
|
||||
{/* Dedupe depends on the transaction id being the constraint key */}
|
||||
{sourceObj.config?.simplefin?.account_id
|
||||
&& sourceObj.constraint_fields?.join(',') !== 'id' && (
|
||||
<p className="text-xs text-orange-600 mt-1">
|
||||
Constraint fields are “{sourceObj.constraint_fields?.join(', ') || 'none'}” — a
|
||||
bank feed should use “id” so re-syncs don’t duplicate rows.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Unified field table */}
|
||||
{availableFields.length > 0 && (
|
||||
<div className="pt-2 border-t border-gray-100 space-y-2">
|
||||
@ -451,6 +582,65 @@ export default function Sources({ source, sources, setSources, setSource }) {
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Bank feed — optional; picking an account defaults the constraint
|
||||
field to the transaction id, which is what dedupe needs */}
|
||||
<div>
|
||||
<label className="text-xs text-gray-500 block mb-1">Bank feed (optional)</label>
|
||||
<div className="flex items-center gap-3 flex-wrap">
|
||||
{bridgeAccounts === null ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={loadBridgeAccounts}
|
||||
disabled={bridgeLoading}
|
||||
className="text-sm border border-gray-300 rounded px-3 py-1.5 text-gray-600 hover:bg-gray-50 hover:border-gray-400 disabled:opacity-50"
|
||||
>
|
||||
{bridgeLoading ? 'Loading…' : 'Link SimpleFIN account…'}
|
||||
</button>
|
||||
) : (
|
||||
<select
|
||||
value={form.simplefin_account_id || ''}
|
||||
onChange={e => handleSelectFeedAccount(e.target.value)}
|
||||
className="text-sm border border-gray-200 rounded px-3 py-1.5 bg-white text-gray-700"
|
||||
>
|
||||
<option value="">No bank feed — CSV import</option>
|
||||
{bridgeAccounts.map(a => (
|
||||
<option key={a.id} value={a.id}>
|
||||
{a.name}{a.organization ? ` — ${a.organization}` : ''}{a.balance ? ` (${a.balance})` : ''}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
</div>
|
||||
{bridgeError && <p className="text-xs text-orange-600 mt-1">{bridgeError}</p>}
|
||||
|
||||
{form.simplefin_account_id && sampleInfo && (
|
||||
<div className="mt-2 bg-blue-50 border border-blue-100 rounded p-3 text-xs text-gray-600 space-y-1">
|
||||
<p>
|
||||
Read {sampleInfo.fetched} transaction{sampleInfo.fetched === 1 ? '' : 's'} from this
|
||||
account and found {sampleInfo.fields} field{sampleInfo.fields === 1 ? '' : 's'}.
|
||||
The table below lists what this account actually returns — fields it never sends
|
||||
won’t appear.
|
||||
</p>
|
||||
{form.constraint_fields === 'id' && (
|
||||
<p>
|
||||
<span className="font-mono text-gray-700">id</span> is checked as the constraint
|
||||
field because it is SimpleFIN’s own transaction identifier. Syncs pull an
|
||||
overlapping window of days, so the same transaction arrives more than once —
|
||||
matching on <span className="font-mono text-gray-700">id</span> skips the repeats
|
||||
while still keeping genuinely separate charges that share a date, amount, and
|
||||
description.
|
||||
</p>
|
||||
)}
|
||||
{sampleInfo.fetched === 0 && (
|
||||
<p className="text-orange-600">
|
||||
No transactions came back, so there was nothing to infer fields from. Sync first,
|
||||
then set the fields up here.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{form.fields.length > 0 && (
|
||||
<div className="pt-2 border-t border-gray-100 space-y-2">
|
||||
<table className="w-full text-xs">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user