From 9f164bcd34570c6f9bc3fa6b1203777b3bf3f609 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sun, 2 Aug 2026 02:50:03 -0400 Subject: [PATCH] Discover feed fields from real data; fix the sync window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field discovery now samples an account's actual transactions instead of assuming a shape. flatten() passes through every scalar the bridge sends rather than whitelisting eleven keys, so institution-specific fields turn up on their own, and inferFields() — extracted from the CSV suggest route so both paths share it — unions keys across the sample because API feeds omit optional fields entirely. Three bugs the live bridge exposed: - posted=0 on pending transactions became 1970-01-01; falsy epochs are now "no date", with date falling back to transacted_at and posted_date kept separate. - days=0 omitted start-date, which returns only the few most recent transactions rather than everything — 4 instead of 89. A start-date is always sent now, clamped to 89 days (the bridge hard-caps at 90). - Sampling asked for more than 45 days, and the bridge's advisory notice about that surfaced in the UI as an error. Samples use 44 days; the threshold is exclusive. The Sources page can now link an account: a picker in both the create dialog and the detail panel, populated on demand, which fills the field table from the sample and defaults the constraint field to the transaction id with an explanation of why. manage.py option 10 claims a setup token and writes the access URL to .env, replacing the throwaway script. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G2HFeU5neCKagTnmA6o9Tu --- api/lib/fields.js | 42 +++++++++ api/lib/simplefin.js | 79 +++++++++++----- api/routes/sources.js | 43 ++++++--- docs/spec.md | 14 ++- manage.py | 53 +++++++++++ ui/src/api.js | 5 + ui/src/pages/Import.jsx | 4 +- ui/src/pages/Sources.jsx | 192 ++++++++++++++++++++++++++++++++++++++- 8 files changed, 389 insertions(+), 43 deletions(-) create mode 100644 api/lib/fields.js diff --git a/api/lib/fields.js b/api/lib/fields.js new file mode 100644 index 0000000..4233562 --- /dev/null +++ b/api/lib/fields.js @@ -0,0 +1,42 @@ +/** + * Field inference + * Derives a field list and column types from sample records, so a source can be + * configured from real data rather than an assumed shape. Used by the CSV + * suggest endpoint and by bank-feed sampling. + */ + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/; + +/** + * @param records array of flat objects + * @param limit how many rows to look at + * @returns { fields: [{name, type}], sampleRows } + * + * Keys are unioned across the sample, not read off the first row — API feeds + * omit optional keys entirely on records that lack them. + */ +function inferFields(records, limit = 50) { + const sampleRows = records.slice(0, limit); + + const keys = []; + for (const row of sampleRows) { + for (const key of Object.keys(row)) { + if (!keys.includes(key)) keys.push(key); + } + } + + const fields = keys.map(key => { + const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null); + let type = 'text'; + if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) { + type = 'numeric'; + } else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) { + type = 'date'; + } + return { name: key, type }; + }); + + return { fields, sampleRows }; +} + +module.exports = { inferFields }; diff --git a/api/lib/simplefin.js b/api/lib/simplefin.js index deb2c82..7d1d9f4 100644 --- a/api/lib/simplefin.js +++ b/api/lib/simplefin.js @@ -15,6 +15,14 @@ const REQUEST_TIMEOUT = 30000; // days late, so pulls overlap by design and lean on constraint_key for dedupe. const DEFAULT_DAYS = 10; +// The bridge hard-caps ranges at 90 days and reports the capping in `errors`; +// 89 stays just inside it so a routine backfill doesn't look like data loss. It +// also advises staying under 45 days, which it says on anything longer — that +// notice is passed through, since it is a real hint about future behaviour. +// Omitting start-date entirely is not "everything": it returns only the most +// recent handful of transactions, so a start-date is always sent. +const MAX_DAYS = 89; + class SimpleFinError extends Error { constructor(message, status) { super(message); @@ -105,28 +113,54 @@ async function claimSetupToken(setupToken) { } // Epoch seconds → YYYY-MM-DD, so dates sort and compare as plain text the way -// CSV-imported dates already do. +// CSV-imported dates already do. Pending transactions carry posted=0 rather than +// omitting it, so falsy epochs are "no date", not 1970-01-01. function toDate(epochSeconds) { - if (epochSeconds === undefined || epochSeconds === null) return null; + if (!epochSeconds) return null; return new Date(epochSeconds * 1000).toISOString().slice(0, 10); } -// Flatten a SimpleFIN transaction into the shallow string map the rule engine -// expects. Account context is folded in so records stay self-describing. +// Flatten a SimpleFIN transaction into the shallow map the rule engine expects. +// +// Every scalar the bridge sends is passed through rather than whitelisted, so +// institution-specific fields (mcc, and whatever a given bank adds) show up in +// the data and in field discovery without a code change here. Only the epoch +// timestamps are reshaped, and account context is folded in so records stay +// self-describing. function flatten(txn, account) { - return { - id: txn.id, - date: toDate(txn.posted), - transacted_at: toDate(txn.transacted_at), - description: txn.description, - payee: txn.payee, - memo: txn.memo, - amount: txn.amount, - pending: txn.pending ? 'true' : 'false', - account_id: account.id, - account_name: account.name, - organization: account.org?.name || account.org?.domain, - }; + const out = {}; + + for (const [key, value] of Object.entries(txn)) { + if (value === null || value === undefined) continue; + if (typeof value === 'object') continue; // nested objects handled below + out[key] = value; + } + + // `extra` is free-form and institution-specific; hoist its scalars under a + // prefix so they can't collide with the documented fields + for (const [key, value] of Object.entries(txn.extra || {})) { + if (value !== null && value !== undefined && typeof value !== 'object') { + out[`extra_${key}`] = value; + } + } + + const posted = toDate(txn.posted); + const transacted = toDate(txn.transacted_at); + + delete out.posted; // replaced by the YYYY-MM-DD forms below + + // A pending transaction has no posted date yet; fall back to when it was + // transacted so every record has something usable to sort and filter on + out.date = posted || transacted; + out.posted_date = posted; + out.transacted_at = transacted; + out.pending = txn.pending ? 'true' : 'false'; + + out.account_id = account.id; + out.account_name = account.name; + out.organization = account.org?.name || account.org?.domain; + + return out; } async function listAccounts(urlEnv) { @@ -148,14 +182,17 @@ async function listAccounts(urlEnv) { /** * Fetch transactions for one account. * - * days — how far back to ask for; 0 lets the bridge return its default window + * days — how far back to ask for; 0 or more than MAX_DAYS means the full 90 * includePending — pending transactions get a new id once they post, so they are excluded by default */ async function fetchTransactions({ accountId, accessUrlEnv, days, includePending = false }) { days = Number.isFinite(days) ? days : DEFAULT_DAYS; + if (days <= 0 || days > MAX_DAYS) days = MAX_DAYS; - const params = { account: accountId }; - if (days > 0) params['start-date'] = Math.floor(Date.now() / 1000) - days * 86400; + const params = { + account: accountId, + 'start-date': Math.floor(Date.now() / 1000) - days * 86400, + }; if (includePending) params.pending = 1; const data = await get(getAccessUrl(accessUrlEnv), '/accounts', params); @@ -179,4 +216,4 @@ async function fetchTransactions({ accountId, accessUrlEnv, days, includePending }; } -module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS }; +module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS, MAX_DAYS }; diff --git a/api/routes/sources.js b/api/routes/sources.js index ebfa19c..0ccf567 100644 --- a/api/routes/sources.js +++ b/api/routes/sources.js @@ -8,6 +8,7 @@ const multer = require('multer'); const { parse } = require('csv-parse/sync'); const { lit, arr } = require('../lib/sql'); const simplefin = require('../lib/simplefin'); +const { inferFields } = require('../lib/fields'); const upload = multer({ storage: multer.memoryStorage() }); @@ -36,6 +37,32 @@ module.exports = (pool) => { } }); + // Sample an account's real transactions and infer its field list — the API + // equivalent of uploading a CSV to /suggest. Whatever the account actually + // returns is what gets offered, so investment or loan accounts describe + // themselves rather than being forced into a checking-account shape. + router.get('/simplefin-sample', async (req, res, next) => { + try { + const { account_id, access_url_env } = req.query; + if (!account_id) return res.status(400).json({ error: 'account_id is required' }); + + const { fetched, records, errors } = await simplefin.fetchTransactions({ + accountId: account_id, + accessUrlEnv: access_url_env, + // The bridge advises staying under 45 days and warns at exactly + // 45, so 44 is the largest quiet sample window + days: req.query.days !== undefined ? parseInt(req.query.days) : 44, + includePending: true, // widen the sample; pending rows can carry extra keys + }); + + const { fields, sampleRows } = inferFields(records); + res.json({ fields, sampleRows, fetched, errors }); + } catch (err) { + if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message }); + next(err); + } + }); + // Exchange a one-shot setup token for the permanent access URL to put in .env router.post('/simplefin-claim', async (req, res, next) => { try { @@ -77,21 +104,7 @@ module.exports = (pool) => { const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true }); if (records.length === 0) return res.status(400).json({ error: 'CSV file is empty' }); - const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/; - const sample = records[0]; - const sampleRows = records.slice(0, 50); - - const fields = Object.keys(sample).map(key => { - const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null); - let type = 'text'; - if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) { - type = 'numeric'; - } else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) { - type = 'date'; - } - return { name: key, type }; - }); - + const { fields, sampleRows } = inferFields(records); res.json({ name: '', constraint_fields: [], fields, sampleRows }); } catch (err) { next(err); diff --git a/docs/spec.md b/docs/spec.md index b150cd3..f7120ee 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -142,9 +142,13 @@ fetching differs: dedup, logging, and transformation are the same code. an `errors` array when an institution is failing. Those errors ride along in the sync response so a broken connection doesn't read as a successful empty pull; the Import page shows them in orange above the counts. -- **Refresh cadence.** The bridge polls banks roughly daily and transactions can - take a few days to appear, so the pull asks for a rolling window (`days`, - default 10, `0` for whatever the bridge returns) rather than tracking a cursor. +- **Refresh cadence and the 90-day wall.** The bridge polls banks roughly daily + and transactions can take a few days to appear, so the pull asks for a rolling + window (`days`, default 10) rather than tracking a cursor. The window has hard + limits: SimpleFIN caps any range at 90 days and advises staying under 45, so + `MAX_DAYS` is 89 and `days=0` or anything larger clamps to it. A `start-date` + is **always** sent — omitting it does not mean "everything available", it + returns only the few most recent transactions. - **Cron.** A daily pull is just the endpoint: `curl -sS -u user:pass -X POST http://localhost:3000/api/sources/NAME/sync` @@ -393,7 +397,9 @@ Shows current status on every screen: 9. **Set login credentials** — Prompts for username and password, bcrypt-hashes the password via `node -e "require('bcrypt')..."`, and writes `LOGIN_USER` and `LOGIN_PASSWORD_HASH` to `.env`. Requires Node.js and bcrypt npm package to be installed. -10. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place. +10. **Claim SimpleFIN setup token** — Prompts for a setup token (hidden input), exchanges it for a permanent access URL via `api/lib/simplefin.js`, and writes `SIMPLEFIN_ACCESS_URL` to `.env`. Warns and confirms before replacing an existing URL. Setup tokens are single-use, so a failed claim generally means a new token is needed. Prints only the bridge host, never the embedded credentials. + +11. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place. **Key behaviors:** - All commands that will be run are printed before the user is asked to confirm. diff --git a/manage.py b/manage.py index ccb7268..f1730f7 100755 --- a/manage.py +++ b/manage.py @@ -902,6 +902,58 @@ def action_set_login_credentials(cfg): info('Restart the service for changes to take effect (option 7).') +def action_claim_simplefin(cfg): + header('Claim a SimpleFIN setup token') + print(' A setup token is single-use. Claiming it returns the permanent access') + print(' URL, which is written to .env as SIMPLEFIN_ACCESS_URL.') + print() + + if not ENV_FILE.exists(): + err(f'{ENV_FILE} does not exist — run the database configuration dialog first') + return + + if cfg and cfg.get('SIMPLEFIN_ACCESS_URL'): + warn('SIMPLEFIN_ACCESS_URL is already set — claiming again will replace it') + if not confirm('Replace the existing access URL?', default_yes=False): + info('Cancelled — no changes made') + return + + token = prompt('Setup token', secret=True) + if not token: + info('Cancelled — no changes made') + return + + print(' Claiming token with SimpleFIN...') + r = subprocess.run( + ['node', '-e', + "require('./api/lib/simplefin.js').claimSetupToken(process.argv[1])" + ".then(u=>process.stdout.write(u),e=>{process.stderr.write(e.message);process.exit(1)})", + token], + capture_output=True, text=True, cwd=ROOT + ) + if r.returncode != 0 or not r.stdout: + err(f'Claim failed — the token may already have been used\n {r.stderr.strip()}') + return + + access_url = r.stdout.strip() + + # Update .env + env_text = ENV_FILE.read_text() + key = 'SIMPLEFIN_ACCESS_URL' + if f'{key}=' in env_text: + import re + env_text = re.sub(rf'^{key}=.*$', f'{key}={access_url}', env_text, flags=re.MULTILINE) + else: + env_text = env_text.rstrip('\n') + f'\n{key}={access_url}\n' + ENV_FILE.write_text(env_text) + + # Show the host only — the URL embeds its own credentials + host = access_url.split('@')[-1] if '@' in access_url else access_url + ok(f'{key} written to {ENV_FILE}') + info(f'Bridge: {host}') + info('Restart the service for changes to take effect (option 7).') + + # ── Main menu ───────────────────────────────────────────────────────────────── MENU = [ @@ -914,6 +966,7 @@ MENU = [ ('Start / restart dataflow.service', action_restart_service), ('Stop dataflow.service', action_stop_service), ('Set login credentials', action_set_login_credentials), + ('Claim SimpleFIN setup token (.env)', action_claim_simplefin), ('Uninstall (service, nginx, database, .env, build)', action_uninstall), ] diff --git a/ui/src/api.js b/ui/src/api.js index cf2d091..a608e14 100644 --- a/ui/src/api.js +++ b/ui/src/api.js @@ -70,6 +70,11 @@ export const api = { const params = new URLSearchParams(opts) return request('POST', `/sources/${name}/sync${params.toString() ? `?${params}` : ''}`) }, + getSimpleFinSample: (accountId, days) => { + const params = new URLSearchParams({ account_id: accountId }) + if (days !== undefined) params.set('days', days) + return request('GET', `/sources/simplefin-sample?${params}`) + }, getSimpleFinAccounts: (accessUrlEnv) => request('GET', `/sources/simplefin-accounts${accessUrlEnv ? `?access_url_env=${encodeURIComponent(accessUrlEnv)}` : ''}`), claimSimpleFinToken: (setup_token) => request('POST', '/sources/simplefin-claim', { setup_token }), diff --git a/ui/src/pages/Import.jsx b/ui/src/pages/Import.jsx index 579b1a9..651ff32 100644 --- a/ui/src/pages/Import.jsx +++ b/ui/src/pages/Import.jsx @@ -204,8 +204,8 @@ export default function Import({ source }) { > - - + + + + ) : ( + + )} + + + {bridgeError &&

{bridgeError}

} + + {/* Dedupe depends on the transaction id being the constraint key */} + {sourceObj.config?.simplefin?.account_id + && sourceObj.constraint_fields?.join(',') !== 'id' && ( +

+ Constraint fields are “{sourceObj.constraint_fields?.join(', ') || 'none'}” — a + bank feed should use “id” so re-syncs don’t duplicate rows. +

+ )} + + {/* Unified field table */} {availableFields.length > 0 && (
@@ -451,6 +582,65 @@ export default function Sources({ source, sources, setSources, setSource }) { />
+ {/* Bank feed — optional; picking an account defaults the constraint + field to the transaction id, which is what dedupe needs */} +
+ +
+ {bridgeAccounts === null ? ( + + ) : ( + + )} +
+ {bridgeError &&

{bridgeError}

} + + {form.simplefin_account_id && sampleInfo && ( +
+

+ Read {sampleInfo.fetched} transaction{sampleInfo.fetched === 1 ? '' : 's'} from this + account and found {sampleInfo.fields} field{sampleInfo.fields === 1 ? '' : 's'}. + The table below lists what this account actually returns — fields it never sends + won’t appear. +

+ {form.constraint_fields === 'id' && ( +

+ id is checked as the constraint + field because it is SimpleFIN’s own transaction identifier. Syncs pull an + overlapping window of days, so the same transaction arrives more than once — + matching on id skips the repeats + while still keeping genuinely separate charges that share a date, amount, and + description. +

+ )} + {sampleInfo.fetched === 0 && ( +

+ No transactions came back, so there was nothing to infer fields from. Sync first, + then set the fields up here. +

+ )} +
+ )} +
+ {form.fields.length > 0 && (