Paul Trowbridge
9f164bcd34
Field discovery now samples an account's actual transactions instead of assuming a shape. flatten() passes through every scalar the bridge sends rather than whitelisting eleven keys, so institution-specific fields turn up on their own, and inferFields() — extracted from the CSV suggest route so both paths share it — unions keys across the sample because API feeds omit optional fields entirely. Three bugs the live bridge exposed: - posted=0 on pending transactions became 1970-01-01; falsy epochs are now "no date", with date falling back to transacted_at and posted_date kept separate. - days=0 omitted start-date, which returns only the few most recent transactions rather than everything — 4 instead of 89. A start-date is always sent now, clamped to 89 days (the bridge hard-caps at 90). - Sampling asked for more than 45 days, and the bridge's advisory notice about that surfaced in the UI as an error. Samples use 44 days; the threshold is exclusive. The Sources page can now link an account: a picker in both the create dialog and the detail panel, populated on demand, which fills the field table from the sample and defaults the constraint field to the transaction id with an explanation of why. manage.py option 10 claims a setup token and writes the access URL to .env, replacing the throwaway script. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01G2HFeU5neCKagTnmA6o9Tu
347 lines
12 KiB
JavaScript
347 lines
12 KiB
JavaScript
import { useState, useEffect, useRef } from 'react'
|
|
import { api } from '../api'
|
|
|
|
function KeyList({ keys, label, color }) {
|
|
if (!keys || keys.length === 0) return null
|
|
return (
|
|
<div className="mb-2">
|
|
<div className={`text-xs font-medium mb-1 ${color}`}>{label} ({keys.length})</div>
|
|
<div className="max-h-32 overflow-y-auto bg-gray-50 rounded p-2 font-mono text-xs text-gray-500 space-y-0.5">
|
|
{keys.map((k, i) => (
|
|
<div key={i}>
|
|
{typeof k === 'object' && k !== null
|
|
? Object.entries(k).map(([field, val]) => `${field}: ${val}`).join(' · ')
|
|
: k}
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)
|
|
}
|
|
|
|
function LogRow({ entry, selected, onToggle }) {
|
|
const [expanded, setExpanded] = useState(false)
|
|
const info = entry.info || {}
|
|
const insertedKeys = info.inserted_keys || []
|
|
const excludedKeys = info.excluded_keys || []
|
|
const hasKeys = insertedKeys.length > 0 || excludedKeys.length > 0
|
|
|
|
return (
|
|
<>
|
|
<tr className={`border-b border-gray-50 ${selected ? 'bg-red-50' : ''}`}>
|
|
<td className="py-1.5 pr-2">
|
|
<input type="checkbox" checked={selected} onChange={onToggle} className="cursor-pointer" />
|
|
</td>
|
|
<td className="py-1.5 text-xs text-gray-400 font-mono">{entry.id}</td>
|
|
<td className="py-1.5 text-gray-500">{new Date(entry.imported_at).toLocaleString()}</td>
|
|
<td className="py-1.5 text-gray-800">{entry.records_imported}</td>
|
|
<td className="py-1.5 text-gray-400">{entry.records_duplicate}</td>
|
|
<td className="py-1.5">
|
|
{hasKeys && (
|
|
<button
|
|
onClick={() => setExpanded(e => !e)}
|
|
className="text-xs text-blue-400 hover:text-blue-600"
|
|
>
|
|
{expanded ? '▲ hide' : '▼ keys'}
|
|
</button>
|
|
)}
|
|
</td>
|
|
</tr>
|
|
{expanded && (
|
|
<tr className={selected ? 'bg-red-50' : 'bg-gray-50'}>
|
|
<td colSpan={6} className="px-4 py-3">
|
|
<KeyList keys={insertedKeys} label="Inserted" color="text-green-600" />
|
|
<KeyList keys={excludedKeys} label="Excluded" color="text-gray-500" />
|
|
</td>
|
|
</tr>
|
|
)}
|
|
</>
|
|
)
|
|
}
|
|
|
|
export default function Import({ source }) {
|
|
const [stats, setStats] = useState(null)
|
|
const [log, setLog] = useState([])
|
|
const [result, setResult] = useState(null)
|
|
const [loading, setLoading] = useState(false)
|
|
const [error, setError] = useState('')
|
|
const [dragOver, setDragOver] = useState(false)
|
|
const [selected, setSelected] = useState(new Set())
|
|
const [simplefin, setSimplefin] = useState(null)
|
|
const [days, setDays] = useState('10')
|
|
const fileRef = useRef()
|
|
|
|
useEffect(() => {
|
|
if (!source) return
|
|
api.getStats(source).then(setStats).catch(() => {})
|
|
api.getImportLog(source).then(setLog).catch(() => {})
|
|
api.getSource(source).then(s => setSimplefin(s.config?.simplefin || null)).catch(() => setSimplefin(null))
|
|
setSelected(new Set())
|
|
}, [source])
|
|
|
|
async function handleImport(file) {
|
|
if (!file || !source) return
|
|
setLoading(true)
|
|
setError('')
|
|
setResult(null)
|
|
try {
|
|
const res = await api.importCSV(source, file)
|
|
setResult(res)
|
|
api.getStats(source).then(setStats)
|
|
api.getImportLog(source).then(setLog)
|
|
} catch (err) {
|
|
setError(err.message)
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
async function handleSync() {
|
|
if (!source) return
|
|
setLoading(true)
|
|
setError('')
|
|
setResult(null)
|
|
try {
|
|
const res = await api.syncSimpleFin(source, { days })
|
|
setResult(res)
|
|
api.getStats(source).then(setStats)
|
|
api.getImportLog(source).then(setLog)
|
|
} catch (err) {
|
|
setError(err.message)
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
async function handleTransform() {
|
|
if (!source) return
|
|
setLoading(true)
|
|
try {
|
|
const res = await api.transform(source)
|
|
setResult(res)
|
|
api.getStats(source).then(setStats)
|
|
} catch (err) {
|
|
setError(err.message)
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
function toggleSelect(id) {
|
|
setSelected(prev => {
|
|
const next = new Set(prev)
|
|
next.has(id) ? next.delete(id) : next.add(id)
|
|
return next
|
|
})
|
|
}
|
|
|
|
async function handleDeleteSelected() {
|
|
if (selected.size === 0) return
|
|
const plural = selected.size === 1 ? 'import' : 'imports'
|
|
if (!confirm(`Delete ${selected.size} ${plural}? This will permanently remove all records from those batches.`)) return
|
|
setLoading(true)
|
|
try {
|
|
await Promise.all([...selected].map(id => api.deleteImport(source, id)))
|
|
const [newLog, newStats] = await Promise.all([api.getImportLog(source), api.getStats(source)])
|
|
setLog(newLog)
|
|
setStats(newStats)
|
|
setSelected(new Set())
|
|
} catch (err) {
|
|
setError(err.message)
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
async function handleReprocess() {
|
|
if (!confirm('Reprocess all records? This will clear and reapply all transformation rules.')) return
|
|
setLoading(true)
|
|
setResult(null)
|
|
try {
|
|
const res = await api.reprocess(source)
|
|
setResult(res)
|
|
api.getStats(source).then(setStats)
|
|
} catch (err) {
|
|
setError(err.message)
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
|
|
|
|
return (
|
|
<div className="p-6 max-w-2xl">
|
|
<h1 className="text-xl font-semibold text-gray-800 mb-6">Import — {source}</h1>
|
|
|
|
{/* Stats */}
|
|
{stats && (
|
|
<div className="flex gap-4 mb-6">
|
|
{[
|
|
{ label: 'Total records', value: stats.total_records },
|
|
{ label: 'Transformed', value: stats.transformed_records },
|
|
{ label: 'Pending', value: stats.pending_records },
|
|
].map(({ label, value }) => (
|
|
<div key={label} className="bg-white border border-gray-200 rounded px-4 py-3 flex-1 text-center">
|
|
<div className="text-2xl font-semibold text-gray-800">{value}</div>
|
|
<div className="text-xs text-gray-400 mt-0.5">{label}</div>
|
|
</div>
|
|
))}
|
|
</div>
|
|
)}
|
|
|
|
{/* SimpleFIN sync — only for sources with a bridge account in their config */}
|
|
{simplefin?.account_id && (
|
|
<div className="bg-white border border-gray-200 rounded p-4 mb-4 flex items-center gap-3">
|
|
<div className="flex-1 min-w-0">
|
|
<div className="text-sm font-medium text-gray-700">SimpleFIN</div>
|
|
<div className="text-xs text-gray-400 font-mono truncate">{simplefin.account_id}</div>
|
|
</div>
|
|
<select
|
|
value={days}
|
|
onChange={e => setDays(e.target.value)}
|
|
className="text-sm border border-gray-200 rounded px-2 py-1.5 bg-white text-gray-700"
|
|
>
|
|
<option value="10">Last 10 days</option>
|
|
<option value="30">Last 30 days</option>
|
|
<option value="45">Last 45 days</option>
|
|
<option value="89">Backfill (bridge maximum)</option>
|
|
</select>
|
|
<button onClick={handleSync} disabled={loading}
|
|
className="text-sm bg-blue-600 text-white px-3 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
|
{loading ? 'Syncing…' : 'Sync now'}
|
|
</button>
|
|
</div>
|
|
)}
|
|
|
|
{/* Drop zone */}
|
|
<div
|
|
className={`border-2 border-dashed rounded-lg p-8 text-center mb-4 cursor-pointer transition-colors ${
|
|
dragOver ? 'border-blue-400 bg-blue-50' : 'border-gray-200 hover:border-gray-300'
|
|
}`}
|
|
onDragOver={e => { e.preventDefault(); setDragOver(true) }}
|
|
onDragLeave={() => setDragOver(false)}
|
|
onDrop={e => { e.preventDefault(); setDragOver(false); handleImport(e.dataTransfer.files[0]) }}
|
|
onClick={() => fileRef.current?.click()}
|
|
>
|
|
<input
|
|
ref={fileRef}
|
|
type="file"
|
|
accept=".csv"
|
|
className="hidden"
|
|
onChange={e => handleImport(e.target.files[0])}
|
|
/>
|
|
{loading
|
|
? <p className="text-sm text-gray-500">Importing…</p>
|
|
: <p className="text-sm text-gray-400">Drop a CSV file here, or click to browse</p>
|
|
}
|
|
</div>
|
|
|
|
{error && <p className="text-sm text-red-500 mb-3">{error}</p>}
|
|
|
|
{result && (
|
|
<div className={`border rounded p-4 mb-4 text-sm ${result.success === false ? 'bg-red-50 border-red-200' : 'bg-white border-gray-200'}`}>
|
|
{result.success === false ? (
|
|
<>
|
|
<p className="text-red-600 font-medium mb-2">{result.error}</p>
|
|
{result.duplicate_rows && (
|
|
<div>
|
|
<p className="text-xs text-red-500 mb-1">Offending rows:</p>
|
|
<div className="max-h-48 overflow-y-auto bg-white rounded border border-red-100 p-2 font-mono text-xs text-red-700 space-y-0.5">
|
|
{result.duplicate_rows.map((row, i) => (
|
|
<div key={i}>
|
|
{Object.entries(row).map(([f, v]) => `${f}: ${v}`).join(' · ')}
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)}
|
|
</>
|
|
) : result.imported !== undefined ? (
|
|
<>
|
|
{result.errors?.length > 0 && (
|
|
<div className="mb-2 text-xs text-orange-600">
|
|
{result.errors.map((e, i) => <div key={i}>Bridge: {e}</div>)}
|
|
</div>
|
|
)}
|
|
{result.fetched !== undefined && (
|
|
<>
|
|
<span className="text-gray-500">{result.fetched} fetched</span>
|
|
<span className="text-gray-400 mx-2">·</span>
|
|
</>
|
|
)}
|
|
<span className="text-green-600 font-medium">{result.imported} imported</span>
|
|
<span className="text-gray-400 mx-2">·</span>
|
|
<span className="text-gray-500">{result.duplicates} duplicates skipped</span>
|
|
{result.transform && (
|
|
<>
|
|
<span className="text-gray-400 mx-2">·</span>
|
|
<span className="text-gray-500">{result.transform.transformed} transformed</span>
|
|
</>
|
|
)}
|
|
</>
|
|
) : (
|
|
<span className="text-green-600 font-medium">{result.transformed} records transformed</span>
|
|
)}
|
|
</div>
|
|
)}
|
|
|
|
{/* Action buttons */}
|
|
<div className="flex gap-2 mb-6">
|
|
{stats && Number(stats.pending_records) > 0 && (
|
|
<button onClick={handleTransform} disabled={loading}
|
|
className="text-sm bg-green-600 text-white px-3 py-1.5 rounded hover:bg-green-700 disabled:opacity-50">
|
|
Transform {stats.pending_records} pending records
|
|
</button>
|
|
)}
|
|
{stats && Number(stats.total_records) > 0 && (
|
|
<button onClick={handleReprocess} disabled={loading}
|
|
className="text-sm bg-orange-500 text-white px-3 py-1.5 rounded hover:bg-orange-600 disabled:opacity-50">
|
|
Reprocess all records
|
|
</button>
|
|
)}
|
|
</div>
|
|
|
|
{/* Import log */}
|
|
{log.length > 0 && (
|
|
<div>
|
|
<div className="flex items-center justify-between mb-2">
|
|
<h2 className="text-sm font-semibold text-gray-700">Import history</h2>
|
|
{selected.size > 0 && (
|
|
<button
|
|
onClick={handleDeleteSelected}
|
|
disabled={loading}
|
|
className="text-xs bg-red-500 text-white px-2.5 py-1 rounded hover:bg-red-600 disabled:opacity-50"
|
|
>
|
|
Delete {selected.size} selected
|
|
</button>
|
|
)}
|
|
</div>
|
|
<table className="w-full text-sm">
|
|
<thead>
|
|
<tr className="text-left text-xs text-gray-400 border-b border-gray-100">
|
|
<th className="pb-1 w-6"></th>
|
|
<th className="pb-1 font-medium w-12">ID</th>
|
|
<th className="pb-1 font-medium">Date</th>
|
|
<th className="pb-1 font-medium">Imported</th>
|
|
<th className="pb-1 font-medium">Duplicates</th>
|
|
<th className="pb-1 w-16"></th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{log.map(entry => (
|
|
<LogRow
|
|
key={entry.id}
|
|
entry={entry}
|
|
selected={selected.has(entry.id)}
|
|
onToggle={() => toggleSelect(entry.id)}
|
|
/>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
)}
|
|
</div>
|
|
)
|
|
}
|