pf_app/ui/src/views/Setup.jsx
Paul Trowbridge 07d92ccd74 Keep master data for a dim_group instead of re-deriving it from the source
Every question about a part -- what values exist, what attributes go with
one -- was answered by querying the source, and the source is the wrong
place to ask. It is a view over a transaction table, so the query is slow
(76s for one ILIKE against 6.9M rows), it describes only what was
transacted, and it cannot express intent: there is no way to say a part is
discontinued, or to name one that has not sold yet.

pf.dim_member holds the app's own list: one row per key value per group,
siblings in jsonb, keyed on (source_id, dim_group, key_value). Refresh is a
merge rather than a replace, so curation survives it -- members absent from
the source are marked source_seen = false, not deleted. Triggered from
Setup, next to Generate SQL, because it reads the whole source and the
answer only changes when the catalogue does.

A key can carry several attribute sets across history -- 11,290 parts
against 13,662 combinations on osm_skinny -- so the refresh takes the most
recent by the source's date column. That also fixes the sibling autofill,
which used to run a DISTINCT ... LIMIT 2 against the source and silently
fill nothing whenever a part came back ambiguous. A member row is one
definition by construction.

The client fetches each group's list once per source and does both
completion and autofill against it in memory, so neither costs a request.
Columns outside a group, or a group never refreshed, still fall back to the
version's values endpoint.

Run 01_schema.sql to create the table.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 00:19:39 -04:00

539 lines
25 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { useState, useEffect, useRef } from 'react'
import useTheme from '../theme.jsx'
const ROLES = ['ignore', 'dimension', 'value', 'units', 'date', 'filter']
const ROLE_STYLE = {
dimension: 'bg-blue-50 text-blue-700',
value: 'bg-green-50 text-green-700',
units: 'bg-green-50 text-green-700',
date: 'bg-purple-50 text-purple-700',
filter: 'bg-yellow-50 text-yellow-700',
ignore: 'bg-gray-100 text-gray-400',
}
export default function Setup({ refreshSources }) {
const { dark } = useTheme()
const [tables, setTables] = useState([])
const [sources, setSources] = useState([])
const [selectedSource, setSelectedSource] = useState(null)
const [cols, setCols] = useState([])
const [editedCols, setEditedCols] = useState([])
const [colsDirty, setColsDirty] = useState(false)
const [preview, setPreview] = useState(null) // { schema, tname, columns, rows }
const [previewLoading, setPreviewLoading] = useState(false)
const [sqlStatus, setSqlStatus] = useState({}) // sourceId -> bool
const [saving, setSaving] = useState(false)
const [generating, setGenerating] = useState(false)
const [refreshingDims, setRefreshingDims] = useState(false)
const [msg, setMsg] = useState(null)
const [dimPeriodCols, setDimPeriodCols] = useState([])
const [openPeriodIdx, setOpenPeriodIdx] = useState(null)
const periodDropRef = useRef(null)
useEffect(() => {
fetch('/api/tables').then(r => r.json()).then(setTables).catch(console.error)
fetch('/api/dim-period/cols').then(r => r.json()).then(setDimPeriodCols).catch(console.error)
loadSources()
}, [])
useEffect(() => {
if (openPeriodIdx === null) return
function handleClick(e) {
if (periodDropRef.current && !periodDropRef.current.contains(e.target))
setOpenPeriodIdx(null)
}
document.addEventListener('mousedown', handleClick)
return () => document.removeEventListener('mousedown', handleClick)
}, [openPeriodIdx])
function loadSources() {
fetch('/api/sources').then(r => r.json()).then(data => {
setSources(data)
// check sql status for each source
data.forEach(s => {
fetch(`/api/sources/${s.id}/sql`).then(r => r.json()).then(sqls => {
setSqlStatus(prev => ({ ...prev, [s.id]: sqls.length > 0 }))
})
})
}).catch(console.error)
refreshSources?.()
}
function selectSource(source) {
setSelectedSource(source)
setColsDirty(false)
fetch(`/api/sources/${source.id}/cols`).then(r => r.json()).then(data => {
setCols(data)
setEditedCols(data.map(c => ({ ...c })))
})
}
async function openPreview(schema, tname, e) {
e.stopPropagation()
setPreviewLoading(true)
setPreview({ schema, tname, loading: true })
try {
const data = await fetch(`/api/tables/${schema}/${tname}/preview`).then(r => r.json())
setPreview({ schema, tname, ...data })
} catch {
setPreview(null)
} finally {
setPreviewLoading(false)
}
}
async function registerSource(schema, tname) {
try {
const res = await fetch('/api/sources', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ schema, tname, created_by: 'admin' })
})
if (!res.ok) {
const err = await res.json()
flash(err.error, 'error')
return
}
const source = await res.json()
loadSources()
flash(`Registered ${schema}.${tname}`)
// auto-select new source and load its cols
fetch(`/api/sources/${source.id}/cols`).then(r => r.json()).then(data => {
setSelectedSource(source)
setCols(data)
setEditedCols(data.map(c => ({ ...c })))
setColsDirty(false)
})
} catch (err) {
flash(err.message, 'error')
}
}
function updateCol(idx, field, value) {
setEditedCols(prev => {
const next = prev.map((c, i) => i === idx ? { ...c, [field]: value } : c)
return next
})
setColsDirty(true)
}
async function saveCols() {
setSaving(true)
try {
const res = await fetch(`/api/sources/${selectedSource.id}/cols`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(editedCols)
})
if (!res.ok) { const e = await res.json(); flash(e.error, 'error'); return }
const saved = await res.json()
setCols(saved)
setEditedCols(saved.map(c => ({ ...c })))
setColsDirty(false)
flash('Saved')
} catch (err) {
flash(err.message, 'error')
} finally {
setSaving(false)
}
}
async function generateSQL() {
setGenerating(true)
try {
const res = await fetch(`/api/sources/${selectedSource.id}/generate-sql`, { method: 'POST' })
const data = await res.json()
if (!res.ok) { flash(data.error, 'error'); return }
setSqlStatus(prev => ({ ...prev, [selectedSource.id]: true }))
flash(`SQL generated: ${data.operations.join(', ')}`)
} catch (err) {
flash(err.message, 'error')
} finally {
setGenerating(false)
}
}
// Rebuild every keyed dim_group's member list from the source. Deliberate rather
// than automatic: it reads the whole source, which for a view over a transaction
// table is millions of rows, and the answer only changes when the catalogue does.
async function refreshDimMembers() {
const groups = [...new Set(cols.filter(c => c.dim_group && c.is_key).map(c => c.dim_group))]
if (groups.length === 0) {
flash('No dim_group has an is_key column, so there is nothing to build a list from', 'error')
return
}
setRefreshingDims(true)
try {
const done = []
for (const g of groups) {
const res = await fetch(`/api/sources/${selectedSource.id}/dim/${encodeURIComponent(g)}/refresh`,
{ method: 'POST' })
const data = await res.json()
if (!res.ok) { flash(`${g}: ${data.error}`, 'error'); return }
done.push(`${g}: ${data.members.toLocaleString()} members`
+ (data.no_longer_in_source ? `, ${data.no_longer_in_source} no longer in source` : '')
+ ` (${(data.ms / 1000).toFixed(1)}s)`)
}
flash(done.join(' · '))
} catch (err) {
flash(err.message, 'error')
} finally {
setRefreshingDims(false)
}
}
async function deleteSource(id, e) {
e.stopPropagation()
if (!confirm('Deregister this source? Existing forecast tables are not affected.')) return
const res = await fetch(`/api/sources/${id}`, { method: 'DELETE' })
if (!res.ok) {
const data = await res.json().catch(() => ({}))
flash(data.error || 'Delete failed', 'err')
return
}
if (selectedSource?.id === id) { setSelectedSource(null); setCols([]); setEditedCols([]) }
loadSources()
}
function flash(text, type = 'ok') {
setMsg({ text, type })
setTimeout(() => setMsg(null), 3000)
}
const registeredKeys = new Set(sources.map(s => `${s.schema}.${s.tname}`))
// display grain — must match grainOf() in lib/sql_generator.js
const grainCols = editedCols
.filter(c => c.in_grain && (c.role === 'dimension' || c.role === 'date'))
.map(c => c.cname)
return (
<div className="h-full flex overflow-hidden text-sm">
{/* All Tables */}
<div className="w-64 bg-white border-r border-gray-200 flex flex-col shrink-0">
<div className="px-3 py-2 border-b border-gray-100 text-xs font-medium text-gray-500 uppercase tracking-wide">
All Tables
</div>
<div className="overflow-y-auto flex-1">
<table className="w-full text-xs">
<thead className="sticky top-0 bg-gray-50">
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="px-3 py-1.5 font-medium">schema</th>
<th className="px-3 py-1.5 font-medium">table</th>
<th className="px-3 py-1.5 font-medium text-right">rows</th>
</tr>
</thead>
<tbody>
{tables.map(t => {
const key = `${t.schema}.${t.tname}`
const registered = registeredKeys.has(key)
return (
<tr
key={key}
onClick={e => openPreview(t.schema, t.tname, e)}
className="border-t border-gray-50 hover:bg-blue-50 cursor-pointer group"
>
<td className="px-3 py-1.5 text-gray-400">{t.schema}</td>
<td className="px-3 py-1.5 font-medium">
<div className="flex items-center gap-1">
<span className={registered ? 'text-green-600' : ''}>{t.tname}</span>
{registered && <span className="text-green-400 text-xs"></span>}
</div>
</td>
<td className="px-3 py-1.5 text-right text-gray-500">
{Number(t.row_estimate).toLocaleString()}
</td>
</tr>
)
})}
</tbody>
</table>
</div>
</div>
{/* Right side */}
<div className="flex-1 flex flex-col overflow-hidden min-w-0">
{/* Flash message */}
{msg && (
<div className={`px-4 py-2 text-xs font-medium ${msg.type === 'error' ? 'bg-red-50 text-red-700' : 'bg-green-50 text-green-700'}`}>
{msg.text}
</div>
)}
<div className="flex-1 flex flex-col gap-0 overflow-hidden">
{/* Registered Sources */}
<div className="bg-white border-b border-gray-200 shrink-0">
<div className="px-3 py-2 border-b border-gray-100 text-xs font-medium text-gray-500 uppercase tracking-wide">
Registered Sources
</div>
<table className="w-full text-xs">
<thead className="bg-gray-50">
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="px-3 py-1.5 font-medium">source</th>
<th className="px-3 py-1.5 font-medium">schema</th>
<th className="px-3 py-1.5 font-medium">sql</th>
<th className="px-3 py-1.5 font-medium">created</th>
<th className="px-3 py-1.5"></th>
</tr>
</thead>
<tbody>
{sources.length === 0 && (
<tr><td colSpan={5} className="px-3 py-3 text-gray-300 italic">No sources registered click a table to preview, then register it</td></tr>
)}
{sources.map(s => (
<tr
key={s.id}
onClick={() => selectSource(s)}
className={`border-t border-gray-50 cursor-pointer hover:bg-gray-50 ${selectedSource?.id === s.id ? 'bg-blue-50' : ''}`}
>
<td className={`px-3 py-2 font-medium ${selectedSource?.id === s.id ? 'text-blue-700' : ''}`}>{s.tname}</td>
<td className="px-3 py-2 text-gray-400">{s.schema}</td>
<td className="px-3 py-2">
{sqlStatus[s.id]
? <span className="text-green-600 font-medium"> ready</span>
: <span className="text-gray-300"></span>}
</td>
<td className="px-3 py-2 text-gray-400">{s.created_by || '—'}</td>
<td className="px-3 py-2 text-right">
<button onClick={e => deleteSource(s.id, e)} className="text-gray-300 hover:text-red-500 text-xs"></button>
</td>
</tr>
))}
</tbody>
</table>
</div>
{/* Col Meta Editor */}
{selectedSource ? (
<div className="flex-1 flex flex-col overflow-hidden bg-white">
<div className="px-3 py-2 border-b border-gray-100 flex items-center justify-between shrink-0">
<span className="text-xs font-medium text-gray-500 uppercase tracking-wide">
Col Meta <span className="text-gray-700 normal-case">{selectedSource.schema}.{selectedSource.tname}</span>
<span className="ml-3 normal-case font-normal text-gray-400" title="Columns the forecast load is pre-aggregated to">
grain: {grainCols.length
? <span className="font-mono text-gray-600">{grainCols.join(' × ')}</span>
: <span className="italic">none raw rows</span>}
</span>
</span>
<div className="flex items-center gap-2">
{colsDirty && (
<button onClick={saveCols} disabled={saving} className="text-xs border border-gray-200 px-3 py-1 rounded hover:bg-gray-50 disabled:opacity-50">
{saving ? 'Saving…' : 'Save'}
</button>
)}
{cols.some(c => c.dim_group && c.is_key) && (
<button
onClick={refreshDimMembers}
disabled={refreshingDims || colsDirty}
className="text-xs border border-gray-200 px-3 py-1 rounded hover:bg-gray-50 disabled:opacity-50"
title={colsDirty ? 'Save col meta first'
: 'Rebuild the member list for each keyed dim_group from the source. Reads the whole source, so it takes a while.'}
>
{refreshingDims ? 'Refreshing…' : 'Refresh master data'}
</button>
)}
<button
onClick={generateSQL}
disabled={generating || colsDirty}
className="text-xs bg-blue-600 text-white px-3 py-1 rounded hover:bg-blue-700 disabled:opacity-50"
title={colsDirty ? 'Save col meta first' : ''}
>
{generating ? 'Generating…' : 'Generate SQL'}
</button>
</div>
</div>
<div className="overflow-y-auto flex-1">
<table className="w-full text-xs">
<thead className="sticky top-0 bg-gray-50">
<tr className="text-left text-gray-400 border-b border-gray-100">
<th className="px-3 py-1.5 font-medium">column</th>
<th className="px-3 py-1.5 font-medium">role</th>
<th className="px-3 py-1.5 font-medium text-center">key</th>
<th className="px-3 py-1.5 font-medium text-center" title="Include this column in the display grain — the load is pre-aggregated to the flagged columns">grain</th>
<th className="px-3 py-1.5 font-medium">group</th>
<th className="px-3 py-1.5 font-medium">period col</th>
<th className="px-3 py-1.5 font-medium">label</th>
</tr>
</thead>
<tbody>
{editedCols.map((col, i) => (
<tr key={col.cname} className="border-t border-gray-50 hover:bg-gray-50">
<td className="px-3 py-1.5 font-mono text-gray-700">{col.cname}</td>
<td className="px-3 py-1.5">
<select
value={col.role}
onChange={e => updateCol(i, 'role', e.target.value)}
className={`text-xs px-1.5 py-0.5 rounded border-0 font-medium cursor-pointer ${ROLE_STYLE[col.role] || ''}`}
>
{ROLES.map(r => <option key={r} value={r}>{r}</option>)}
</select>
</td>
<td className="px-3 py-1.5 text-center">
<input
type="checkbox"
checked={!!col.is_key}
onChange={e => updateCol(i, 'is_key', e.target.checked)}
disabled={col.role !== 'dimension' && col.role !== 'date'}
className="cursor-pointer disabled:opacity-20"
/>
</td>
<td className="px-3 py-1.5 text-center">
<input
type="checkbox"
checked={!!col.in_grain}
onChange={e => updateCol(i, 'in_grain', e.target.checked)}
disabled={col.role !== 'dimension' && col.role !== 'date'}
className="cursor-pointer disabled:opacity-20"
/>
</td>
<td className="px-3 py-1.5">
<input
type="text"
value={col.dim_group || ''}
onChange={e => updateCol(i, 'dim_group', e.target.value || null)}
placeholder="—"
disabled={col.role !== 'dimension' && col.role !== 'date'}
className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent disabled:opacity-20 disabled:cursor-default"
/>
</td>
<td className="px-3 py-1.5 relative">
{(() => {
const disabled = col.role !== 'dimension' || !col.dim_group
const isOpen = openPeriodIdx === i
return (
<div ref={isOpen ? periodDropRef : null} className="relative">
<button
type="button"
disabled={disabled}
onClick={() => setOpenPeriodIdx(isOpen ? null : i)}
className="text-xs px-1.5 py-0.5 rounded border border-transparent hover:border-gray-200 outline-none bg-white text-gray-700 disabled:opacity-20 disabled:cursor-default font-mono w-full text-left"
>
{col.dim_period_col || '—'}
</button>
{isOpen && (
<ul style={{ backgroundColor: dark ? '#2a2c2f' : '#ffffff', color: dark ? '#c5c9d0' : '#374151', borderColor: dark ? '#3b3f46' : '#e5e7eb' }} className="absolute z-50 left-0 top-full mt-0.5 min-w-full border rounded shadow-lg text-xs font-mono max-h-48 overflow-y-auto">
<li
className="px-2 py-1 cursor-pointer hover:bg-gray-100"
onMouseDown={() => { updateCol(i, 'dim_period_col', null); setOpenPeriodIdx(null) }}
></li>
{dimPeriodCols.map(c => (
<li
key={c}
className={`px-2 py-1 cursor-pointer hover:bg-gray-100 ${col.dim_period_col === c ? 'bg-blue-50 text-blue-700' : ''}`}
onMouseDown={() => { updateCol(i, 'dim_period_col', c); setOpenPeriodIdx(null) }}
>{c}</li>
))}
</ul>
)}
</div>
)
})()}
</td>
<td className="px-3 py-1.5">
<input
type="text"
value={col.label || ''}
onChange={e => updateCol(i, 'label', e.target.value)}
placeholder={col.cname}
className="border border-transparent hover:border-gray-200 focus:border-gray-300 rounded px-1.5 py-0.5 w-full outline-none bg-transparent"
/>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
) : (
<div className="flex-1 flex items-center justify-center text-gray-300 text-xs italic">
Select a source to edit col meta
</div>
)}
</div>
</div>
{/* Table preview modal */}
{preview && (
<div className="fixed inset-0 z-50 flex items-center justify-center">
<div className="absolute inset-0 bg-black/30" onClick={() => setPreview(null)} />
<div className="relative bg-white rounded-lg shadow-2xl flex flex-col z-10 text-xs" style={{ width: 720, maxWidth: '90vw', maxHeight: '80vh' }}>
<div className="flex items-center justify-between px-4 py-3 border-b border-gray-200 shrink-0">
<div className="flex items-center gap-3">
<span className="text-sm font-semibold text-gray-800">{preview.schema}.{preview.tname}</span>
{preview.columns && (
<span className="text-gray-400">{preview.columns.length} columns</span>
)}
{!registeredKeys.has(`${preview.schema}.${preview.tname}`) && (
<button
onClick={() => { registerSource(preview.schema, preview.tname); setPreview(null) }}
className="bg-blue-600 text-white text-xs px-3 py-1 rounded hover:bg-blue-700"
>
+ Register source
</button>
)}
</div>
<button onClick={() => setPreview(null)} className="text-gray-300 hover:text-gray-600 text-lg leading-none ml-4"></button>
</div>
{preview.loading ? (
<div className="p-8 text-center text-gray-400">Loading</div>
) : (
<div className="overflow-y-auto flex-1">
{/* Columns */}
<div className="px-4 pt-3 pb-1 text-gray-400 uppercase tracking-wide font-medium text-xs">Columns</div>
<table className="w-full mb-2">
<thead>
<tr className="text-left text-gray-400 border-b border-gray-100 bg-gray-50">
<th className="px-4 py-1 font-medium">name</th>
<th className="px-4 py-1 font-medium">type</th>
<th className="px-4 py-1 font-medium">nullable</th>
</tr>
</thead>
<tbody>
{(preview.columns || []).map(c => (
<tr key={c.column_name} className="border-t border-gray-50">
<td className="px-4 py-1 font-mono text-gray-700">{c.column_name}</td>
<td className="px-4 py-1 text-gray-400">{c.data_type}</td>
<td className="px-4 py-1 text-gray-400">{c.is_nullable}</td>
</tr>
))}
</tbody>
</table>
{/* Sample rows */}
<div className="px-4 py-1 text-gray-400 uppercase tracking-wide font-medium text-xs border-t border-gray-100">Sample rows</div>
<div className="overflow-x-auto">
<table className="text-xs" style={{ minWidth: '100%' }}>
<thead>
<tr className="text-left text-gray-400 bg-gray-50 border-b border-gray-100">
{(preview.columns || []).map(c => (
<th key={c.column_name} className="px-4 py-1 font-medium whitespace-nowrap">{c.column_name}</th>
))}
</tr>
</thead>
<tbody>
{(preview.rows || []).map((row, i) => (
<tr key={i} className="border-t border-gray-50">
{(preview.columns || []).map(c => (
<td key={c.column_name} className={`px-4 py-1 font-mono whitespace-nowrap ${row[c.column_name] == null ? 'text-gray-300' : 'text-gray-600'}`}>
{row[c.column_name] == null ? 'null' : String(row[c.column_name])}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
</div>
)}
</div>
)
}