diff --git a/api/lib/fields.js b/api/lib/fields.js new file mode 100644 index 0000000..4233562 --- /dev/null +++ b/api/lib/fields.js @@ -0,0 +1,42 @@ +/** + * Field inference + * Derives a field list and column types from sample records, so a source can be + * configured from real data rather than an assumed shape. Used by the CSV + * suggest endpoint and by bank-feed sampling. + */ + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/; + +/** + * @param records array of flat objects + * @param limit how many rows to look at + * @returns { fields: [{name, type}], sampleRows } + * + * Keys are unioned across the sample, not read off the first row — API feeds + * omit optional keys entirely on records that lack them. + */ +function inferFields(records, limit = 50) { + const sampleRows = records.slice(0, limit); + + const keys = []; + for (const row of sampleRows) { + for (const key of Object.keys(row)) { + if (!keys.includes(key)) keys.push(key); + } + } + + const fields = keys.map(key => { + const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null); + let type = 'text'; + if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) { + type = 'numeric'; + } else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) { + type = 'date'; + } + return { name: key, type }; + }); + + return { fields, sampleRows }; +} + +module.exports = { inferFields }; diff --git a/api/lib/simplefin.js b/api/lib/simplefin.js index deb2c82..7d1d9f4 100644 --- a/api/lib/simplefin.js +++ b/api/lib/simplefin.js @@ -15,6 +15,14 @@ const REQUEST_TIMEOUT = 30000; // days late, so pulls overlap by design and lean on constraint_key for dedupe. const DEFAULT_DAYS = 10; +// The bridge hard-caps ranges at 90 days and reports the capping in `errors`; +// 89 stays just inside it so a routine backfill doesn't look like data loss. It +// also advises staying under 45 days, which it says on anything longer — that +// notice is passed through, since it is a real hint about future behaviour. +// Omitting start-date entirely is not "everything": it returns only the most +// recent handful of transactions, so a start-date is always sent. +const MAX_DAYS = 89; + class SimpleFinError extends Error { constructor(message, status) { super(message); @@ -105,28 +113,54 @@ async function claimSetupToken(setupToken) { } // Epoch seconds → YYYY-MM-DD, so dates sort and compare as plain text the way -// CSV-imported dates already do. +// CSV-imported dates already do. Pending transactions carry posted=0 rather than +// omitting it, so falsy epochs are "no date", not 1970-01-01. function toDate(epochSeconds) { - if (epochSeconds === undefined || epochSeconds === null) return null; + if (!epochSeconds) return null; return new Date(epochSeconds * 1000).toISOString().slice(0, 10); } -// Flatten a SimpleFIN transaction into the shallow string map the rule engine -// expects. Account context is folded in so records stay self-describing. +// Flatten a SimpleFIN transaction into the shallow map the rule engine expects. +// +// Every scalar the bridge sends is passed through rather than whitelisted, so +// institution-specific fields (mcc, and whatever a given bank adds) show up in +// the data and in field discovery without a code change here. Only the epoch +// timestamps are reshaped, and account context is folded in so records stay +// self-describing. function flatten(txn, account) { - return { - id: txn.id, - date: toDate(txn.posted), - transacted_at: toDate(txn.transacted_at), - description: txn.description, - payee: txn.payee, - memo: txn.memo, - amount: txn.amount, - pending: txn.pending ? 'true' : 'false', - account_id: account.id, - account_name: account.name, - organization: account.org?.name || account.org?.domain, - }; + const out = {}; + + for (const [key, value] of Object.entries(txn)) { + if (value === null || value === undefined) continue; + if (typeof value === 'object') continue; // nested objects handled below + out[key] = value; + } + + // `extra` is free-form and institution-specific; hoist its scalars under a + // prefix so they can't collide with the documented fields + for (const [key, value] of Object.entries(txn.extra || {})) { + if (value !== null && value !== undefined && typeof value !== 'object') { + out[`extra_${key}`] = value; + } + } + + const posted = toDate(txn.posted); + const transacted = toDate(txn.transacted_at); + + delete out.posted; // replaced by the YYYY-MM-DD forms below + + // A pending transaction has no posted date yet; fall back to when it was + // transacted so every record has something usable to sort and filter on + out.date = posted || transacted; + out.posted_date = posted; + out.transacted_at = transacted; + out.pending = txn.pending ? 'true' : 'false'; + + out.account_id = account.id; + out.account_name = account.name; + out.organization = account.org?.name || account.org?.domain; + + return out; } async function listAccounts(urlEnv) { @@ -148,14 +182,17 @@ async function listAccounts(urlEnv) { /** * Fetch transactions for one account. * - * days — how far back to ask for; 0 lets the bridge return its default window + * days — how far back to ask for; 0 or more than MAX_DAYS means the full 90 * includePending — pending transactions get a new id once they post, so they are excluded by default */ async function fetchTransactions({ accountId, accessUrlEnv, days, includePending = false }) { days = Number.isFinite(days) ? days : DEFAULT_DAYS; + if (days <= 0 || days > MAX_DAYS) days = MAX_DAYS; - const params = { account: accountId }; - if (days > 0) params['start-date'] = Math.floor(Date.now() / 1000) - days * 86400; + const params = { + account: accountId, + 'start-date': Math.floor(Date.now() / 1000) - days * 86400, + }; if (includePending) params.pending = 1; const data = await get(getAccessUrl(accessUrlEnv), '/accounts', params); @@ -179,4 +216,4 @@ async function fetchTransactions({ accountId, accessUrlEnv, days, includePending }; } -module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS }; +module.exports = { listAccounts, fetchTransactions, claimSetupToken, SimpleFinError, DEFAULT_DAYS, MAX_DAYS }; diff --git a/api/routes/sources.js b/api/routes/sources.js index ebfa19c..0ccf567 100644 --- a/api/routes/sources.js +++ b/api/routes/sources.js @@ -8,6 +8,7 @@ const multer = require('multer'); const { parse } = require('csv-parse/sync'); const { lit, arr } = require('../lib/sql'); const simplefin = require('../lib/simplefin'); +const { inferFields } = require('../lib/fields'); const upload = multer({ storage: multer.memoryStorage() }); @@ -36,6 +37,32 @@ module.exports = (pool) => { } }); + // Sample an account's real transactions and infer its field list — the API + // equivalent of uploading a CSV to /suggest. Whatever the account actually + // returns is what gets offered, so investment or loan accounts describe + // themselves rather than being forced into a checking-account shape. + router.get('/simplefin-sample', async (req, res, next) => { + try { + const { account_id, access_url_env } = req.query; + if (!account_id) return res.status(400).json({ error: 'account_id is required' }); + + const { fetched, records, errors } = await simplefin.fetchTransactions({ + accountId: account_id, + accessUrlEnv: access_url_env, + // The bridge advises staying under 45 days and warns at exactly + // 45, so 44 is the largest quiet sample window + days: req.query.days !== undefined ? parseInt(req.query.days) : 44, + includePending: true, // widen the sample; pending rows can carry extra keys + }); + + const { fields, sampleRows } = inferFields(records); + res.json({ fields, sampleRows, fetched, errors }); + } catch (err) { + if (err instanceof simplefin.SimpleFinError) return res.status(err.status || 502).json({ error: err.message }); + next(err); + } + }); + // Exchange a one-shot setup token for the permanent access URL to put in .env router.post('/simplefin-claim', async (req, res, next) => { try { @@ -77,21 +104,7 @@ module.exports = (pool) => { const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true }); if (records.length === 0) return res.status(400).json({ error: 'CSV file is empty' }); - const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.Z+-]+)?$/; - const sample = records[0]; - const sampleRows = records.slice(0, 50); - - const fields = Object.keys(sample).map(key => { - const vals = sampleRows.map(r => r[key]).filter(v => v !== '' && v != null); - let type = 'text'; - if (vals.length > 0 && vals.every(v => !isNaN(parseFloat(v)) && isFinite(v) && String(v).charAt(0) !== '0')) { - type = 'numeric'; - } else if (vals.length > 0 && vals.every(v => ISO_DATE_RE.test(String(v)))) { - type = 'date'; - } - return { name: key, type }; - }); - + const { fields, sampleRows } = inferFields(records); res.json({ name: '', constraint_fields: [], fields, sampleRows }); } catch (err) { next(err); diff --git a/docs/spec.md b/docs/spec.md index b150cd3..f7120ee 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -142,9 +142,13 @@ fetching differs: dedup, logging, and transformation are the same code. an `errors` array when an institution is failing. Those errors ride along in the sync response so a broken connection doesn't read as a successful empty pull; the Import page shows them in orange above the counts. -- **Refresh cadence.** The bridge polls banks roughly daily and transactions can - take a few days to appear, so the pull asks for a rolling window (`days`, - default 10, `0` for whatever the bridge returns) rather than tracking a cursor. +- **Refresh cadence and the 90-day wall.** The bridge polls banks roughly daily + and transactions can take a few days to appear, so the pull asks for a rolling + window (`days`, default 10) rather than tracking a cursor. The window has hard + limits: SimpleFIN caps any range at 90 days and advises staying under 45, so + `MAX_DAYS` is 89 and `days=0` or anything larger clamps to it. A `start-date` + is **always** sent — omitting it does not mean "everything available", it + returns only the few most recent transactions. - **Cron.** A daily pull is just the endpoint: `curl -sS -u user:pass -X POST http://localhost:3000/api/sources/NAME/sync` @@ -393,7 +397,9 @@ Shows current status on every screen: 9. **Set login credentials** — Prompts for username and password, bcrypt-hashes the password via `node -e "require('bcrypt')..."`, and writes `LOGIN_USER` and `LOGIN_PASSWORD_HASH` to `.env`. Requires Node.js and bcrypt npm package to be installed. -10. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place. +10. **Claim SimpleFIN setup token** — Prompts for a setup token (hidden input), exchanges it for a permanent access URL via `api/lib/simplefin.js`, and writes `SIMPLEFIN_ACCESS_URL` to `.env`. Warns and confirms before replacing an existing URL. Setup tokens are single-use, so a failed claim generally means a new token is needed. Prints only the bridge host, never the embedded credentials. + +11. **Uninstall** — Reverses everything the other options install, in reverse order: stops/disables/removes the systemd unit, removes the nginx site and reloads nginx, drops the database and its user (prompts for admin credentials), then deletes `.env`, `public/`, and `node_modules`. Lists exactly what it found before doing anything and requires typing `delete` to proceed. The repository itself is left in place. **Key behaviors:** - All commands that will be run are printed before the user is asked to confirm. diff --git a/manage.py b/manage.py index ccb7268..f1730f7 100755 --- a/manage.py +++ b/manage.py @@ -902,6 +902,58 @@ def action_set_login_credentials(cfg): info('Restart the service for changes to take effect (option 7).') +def action_claim_simplefin(cfg): + header('Claim a SimpleFIN setup token') + print(' A setup token is single-use. Claiming it returns the permanent access') + print(' URL, which is written to .env as SIMPLEFIN_ACCESS_URL.') + print() + + if not ENV_FILE.exists(): + err(f'{ENV_FILE} does not exist — run the database configuration dialog first') + return + + if cfg and cfg.get('SIMPLEFIN_ACCESS_URL'): + warn('SIMPLEFIN_ACCESS_URL is already set — claiming again will replace it') + if not confirm('Replace the existing access URL?', default_yes=False): + info('Cancelled — no changes made') + return + + token = prompt('Setup token', secret=True) + if not token: + info('Cancelled — no changes made') + return + + print(' Claiming token with SimpleFIN...') + r = subprocess.run( + ['node', '-e', + "require('./api/lib/simplefin.js').claimSetupToken(process.argv[1])" + ".then(u=>process.stdout.write(u),e=>{process.stderr.write(e.message);process.exit(1)})", + token], + capture_output=True, text=True, cwd=ROOT + ) + if r.returncode != 0 or not r.stdout: + err(f'Claim failed — the token may already have been used\n {r.stderr.strip()}') + return + + access_url = r.stdout.strip() + + # Update .env + env_text = ENV_FILE.read_text() + key = 'SIMPLEFIN_ACCESS_URL' + if f'{key}=' in env_text: + import re + env_text = re.sub(rf'^{key}=.*$', f'{key}={access_url}', env_text, flags=re.MULTILINE) + else: + env_text = env_text.rstrip('\n') + f'\n{key}={access_url}\n' + ENV_FILE.write_text(env_text) + + # Show the host only — the URL embeds its own credentials + host = access_url.split('@')[-1] if '@' in access_url else access_url + ok(f'{key} written to {ENV_FILE}') + info(f'Bridge: {host}') + info('Restart the service for changes to take effect (option 7).') + + # ── Main menu ───────────────────────────────────────────────────────────────── MENU = [ @@ -914,6 +966,7 @@ MENU = [ ('Start / restart dataflow.service', action_restart_service), ('Stop dataflow.service', action_stop_service), ('Set login credentials', action_set_login_credentials), + ('Claim SimpleFIN setup token (.env)', action_claim_simplefin), ('Uninstall (service, nginx, database, .env, build)', action_uninstall), ] diff --git a/ui/src/api.js b/ui/src/api.js index cf2d091..a608e14 100644 --- a/ui/src/api.js +++ b/ui/src/api.js @@ -70,6 +70,11 @@ export const api = { const params = new URLSearchParams(opts) return request('POST', `/sources/${name}/sync${params.toString() ? `?${params}` : ''}`) }, + getSimpleFinSample: (accountId, days) => { + const params = new URLSearchParams({ account_id: accountId }) + if (days !== undefined) params.set('days', days) + return request('GET', `/sources/simplefin-sample?${params}`) + }, getSimpleFinAccounts: (accessUrlEnv) => request('GET', `/sources/simplefin-accounts${accessUrlEnv ? `?access_url_env=${encodeURIComponent(accessUrlEnv)}` : ''}`), claimSimpleFinToken: (setup_token) => request('POST', '/sources/simplefin-claim', { setup_token }), diff --git a/ui/src/pages/Import.jsx b/ui/src/pages/Import.jsx index 579b1a9..651ff32 100644 --- a/ui/src/pages/Import.jsx +++ b/ui/src/pages/Import.jsx @@ -204,8 +204,8 @@ export default function Import({ source }) { > - - + + + + ) : ( + + )} + + + {bridgeError &&

{bridgeError}

} + + {/* Dedupe depends on the transaction id being the constraint key */} + {sourceObj.config?.simplefin?.account_id + && sourceObj.constraint_fields?.join(',') !== 'id' && ( +

+ Constraint fields are “{sourceObj.constraint_fields?.join(', ') || 'none'}” — a + bank feed should use “id” so re-syncs don’t duplicate rows. +

+ )} + + {/* Unified field table */} {availableFields.length > 0 && (
@@ -451,6 +582,65 @@ export default function Sources({ source, sources, setSources, setSource }) { />
+ {/* Bank feed — optional; picking an account defaults the constraint + field to the transaction id, which is what dedupe needs */} +
+ +
+ {bridgeAccounts === null ? ( + + ) : ( + + )} +
+ {bridgeError &&

{bridgeError}

} + + {form.simplefin_account_id && sampleInfo && ( +
+

+ Read {sampleInfo.fetched} transaction{sampleInfo.fetched === 1 ? '' : 's'} from this + account and found {sampleInfo.fields} field{sampleInfo.fields === 1 ? '' : 's'}. + The table below lists what this account actually returns — fields it never sends + won’t appear. +

+ {form.constraint_fields === 'id' && ( +

+ id is checked as the constraint + field because it is SimpleFIN’s own transaction identifier. Syncs pull an + overlapping window of days, so the same transaction arrives more than once — + matching on id skips the repeats + while still keeping genuinely separate charges that share a date, amount, and + description. +

+ )} + {sampleInfo.fetched === 0 && ( +

+ No transactions came back, so there was nothing to infer fields from. Sync first, + then set the fields up here. +

+ )} +
+ )} +
+ {form.fields.length > 0 && (