dataflow/ui/src/pages/Import.jsx
Paul Trowbridge 2abcb89bcd Add import log detail, key tracking, and cascade delete
- Add import_id column to records (links each record to its import batch)
- import_records() now stores readable dedup field values (not hashes) in
  info.inserted_keys / info.excluded_keys, and stamps import_id on insert
- delete_import() simplified to delete log row; ON DELETE CASCADE removes records
- Add get_import_log() and get_all_import_logs() DB functions
- Add DELETE /api/sources/:name/import-log/:id endpoint
- Add GET /api/sources/import-log global log endpoint
- Import route now auto-applies transformations to new records after import
- Import page: show ID column, expandable key detail, checkbox delete
- New Log page: global view of all imports across sources
- Update README API reference and workflow

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-12 11:04:34 -04:00

276 lines
9.4 KiB
JavaScript

import { useState, useEffect, useRef } from 'react'
import { api } from '../api'
function KeyList({ keys, label, color }) {
if (!keys || keys.length === 0) return null
return (
<div className="mb-2">
<div className={`text-xs font-medium mb-1 ${color}`}>{label} ({keys.length})</div>
<div className="max-h-32 overflow-y-auto bg-gray-50 rounded p-2 font-mono text-xs text-gray-500 space-y-0.5">
{keys.map((k, i) => (
<div key={i}>
{typeof k === 'object' && k !== null
? Object.entries(k).map(([field, val]) => `${field}: ${val}`).join(' · ')
: k}
</div>
))}
</div>
</div>
)
}
function LogRow({ entry, selected, onToggle }) {
const [expanded, setExpanded] = useState(false)
const info = entry.info || {}
const insertedKeys = info.inserted_keys || []
const excludedKeys = info.excluded_keys || []
const hasKeys = insertedKeys.length > 0 || excludedKeys.length > 0
return (
<>
<tr className={`border-b border-gray-50 ${selected ? 'bg-red-50' : ''}`}>
<td className="py-1.5 pr-2">
<input type="checkbox" checked={selected} onChange={onToggle} className="cursor-pointer" />
</td>
<td className="py-1.5 text-xs text-gray-400 font-mono">{entry.id}</td>
<td className="py-1.5 text-gray-500">{new Date(entry.imported_at).toLocaleString()}</td>
<td className="py-1.5 text-gray-800">{entry.records_imported}</td>
<td className="py-1.5 text-gray-400">{entry.records_duplicate}</td>
<td className="py-1.5">
{hasKeys && (
<button
onClick={() => setExpanded(e => !e)}
className="text-xs text-blue-400 hover:text-blue-600"
>
{expanded ? '▲ hide' : '▼ keys'}
</button>
)}
</td>
</tr>
{expanded && (
<tr className={selected ? 'bg-red-50' : 'bg-gray-50'}>
<td colSpan={6} className="px-4 py-3">
<KeyList keys={insertedKeys} label="Inserted" color="text-green-600" />
<KeyList keys={excludedKeys} label="Excluded" color="text-gray-500" />
</td>
</tr>
)}
</>
)
}
export default function Import({ source }) {
const [stats, setStats] = useState(null)
const [log, setLog] = useState([])
const [result, setResult] = useState(null)
const [loading, setLoading] = useState(false)
const [error, setError] = useState('')
const [dragOver, setDragOver] = useState(false)
const [selected, setSelected] = useState(new Set())
const fileRef = useRef()
useEffect(() => {
if (!source) return
api.getStats(source).then(setStats).catch(() => {})
api.getImportLog(source).then(setLog).catch(() => {})
setSelected(new Set())
}, [source])
async function handleImport(file) {
if (!file || !source) return
setLoading(true)
setError('')
setResult(null)
try {
const res = await api.importCSV(source, file)
setResult(res)
api.getStats(source).then(setStats)
api.getImportLog(source).then(setLog)
} catch (err) {
setError(err.message)
} finally {
setLoading(false)
}
}
async function handleTransform() {
if (!source) return
setLoading(true)
try {
const res = await api.transform(source)
setResult(res)
api.getStats(source).then(setStats)
} catch (err) {
setError(err.message)
} finally {
setLoading(false)
}
}
function toggleSelect(id) {
setSelected(prev => {
const next = new Set(prev)
next.has(id) ? next.delete(id) : next.add(id)
return next
})
}
async function handleDeleteSelected() {
if (selected.size === 0) return
const plural = selected.size === 1 ? 'import' : 'imports'
if (!confirm(`Delete ${selected.size} ${plural}? This will permanently remove all records from those batches.`)) return
setLoading(true)
try {
await Promise.all([...selected].map(id => api.deleteImport(source, id)))
const [newLog, newStats] = await Promise.all([api.getImportLog(source), api.getStats(source)])
setLog(newLog)
setStats(newStats)
setSelected(new Set())
} catch (err) {
setError(err.message)
} finally {
setLoading(false)
}
}
async function handleReprocess() {
if (!confirm('Reprocess all records? This will clear and reapply all transformation rules.')) return
setLoading(true)
setResult(null)
try {
const res = await api.reprocess(source)
setResult(res)
api.getStats(source).then(setStats)
} catch (err) {
setError(err.message)
} finally {
setLoading(false)
}
}
if (!source) return <div className="p-6 text-sm text-gray-400">Select a source first.</div>
return (
<div className="p-6 max-w-2xl">
<h1 className="text-xl font-semibold text-gray-800 mb-6">Import {source}</h1>
{/* Stats */}
{stats && (
<div className="flex gap-4 mb-6">
{[
{ label: 'Total records', value: stats.total_records },
{ label: 'Transformed', value: stats.transformed_records },
{ label: 'Pending', value: stats.pending_records },
].map(({ label, value }) => (
<div key={label} className="bg-white border border-gray-200 rounded px-4 py-3 flex-1 text-center">
<div className="text-2xl font-semibold text-gray-800">{value}</div>
<div className="text-xs text-gray-400 mt-0.5">{label}</div>
</div>
))}
</div>
)}
{/* Drop zone */}
<div
className={`border-2 border-dashed rounded-lg p-8 text-center mb-4 cursor-pointer transition-colors ${
dragOver ? 'border-blue-400 bg-blue-50' : 'border-gray-200 hover:border-gray-300'
}`}
onDragOver={e => { e.preventDefault(); setDragOver(true) }}
onDragLeave={() => setDragOver(false)}
onDrop={e => { e.preventDefault(); setDragOver(false); handleImport(e.dataTransfer.files[0]) }}
onClick={() => fileRef.current?.click()}
>
<input
ref={fileRef}
type="file"
accept=".csv"
className="hidden"
onChange={e => handleImport(e.target.files[0])}
/>
{loading
? <p className="text-sm text-gray-500">Importing</p>
: <p className="text-sm text-gray-400">Drop a CSV file here, or click to browse</p>
}
</div>
{error && <p className="text-sm text-red-500 mb-3">{error}</p>}
{result && (
<div className="bg-white border border-gray-200 rounded p-4 mb-4 text-sm">
{result.imported !== undefined ? (
<>
<span className="text-green-600 font-medium">{result.imported} imported</span>
<span className="text-gray-400 mx-2">·</span>
<span className="text-gray-500">{result.duplicates} duplicates skipped</span>
{result.transform && (
<>
<span className="text-gray-400 mx-2">·</span>
<span className="text-gray-500">{result.transform.transformed} transformed</span>
</>
)}
</>
) : (
<span className="text-green-600 font-medium">{result.transformed} records transformed</span>
)}
</div>
)}
{/* Action buttons */}
<div className="flex gap-2 mb-6">
{stats && Number(stats.pending_records) > 0 && (
<button onClick={handleTransform} disabled={loading}
className="text-sm bg-green-600 text-white px-3 py-1.5 rounded hover:bg-green-700 disabled:opacity-50">
Transform {stats.pending_records} pending records
</button>
)}
{stats && Number(stats.total_records) > 0 && (
<button onClick={handleReprocess} disabled={loading}
className="text-sm bg-orange-500 text-white px-3 py-1.5 rounded hover:bg-orange-600 disabled:opacity-50">
Reprocess all records
</button>
)}
</div>
{/* Import log */}
{log.length > 0 && (
<div>
<div className="flex items-center justify-between mb-2">
<h2 className="text-sm font-semibold text-gray-700">Import history</h2>
{selected.size > 0 && (
<button
onClick={handleDeleteSelected}
disabled={loading}
className="text-xs bg-red-500 text-white px-2.5 py-1 rounded hover:bg-red-600 disabled:opacity-50"
>
Delete {selected.size} selected
</button>
)}
</div>
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-gray-400 border-b border-gray-100">
<th className="pb-1 w-6"></th>
<th className="pb-1 font-medium w-12">ID</th>
<th className="pb-1 font-medium">Date</th>
<th className="pb-1 font-medium">Imported</th>
<th className="pb-1 font-medium">Duplicates</th>
<th className="pb-1 w-16"></th>
</tr>
</thead>
<tbody>
{log.map(entry => (
<LogRow
key={entry.id}
entry={entry}
selected={selected.has(entry.id)}
onToggle={() => toggleSelect(entry.id)}
/>
))}
</tbody>
</table>
</div>
)}
</div>
)
}