diff --git a/CLAUDE.md b/CLAUDE.md index e5faf78..30db039 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -353,6 +353,22 @@ effects call the API immediately. `pf_user: 'admin'`, which any client could have set to anything. The audit log now names the account that made the change. +## Source columns come from pg_catalog + +`information_schema.columns` omits **materialized views** — they are not in the +SQL standard — and `gs.osm_skinny` is one. So the source the whole app is built +on looked like it had no columns: registering it seeded nothing, and creating a +version failed with "No usable columns in col_meta" while col_meta plainly held +thirty-six. + +`RELATION_COLUMNS_SQL` in `lib/utils.js` is the replacement, used by version +creation, source registration and the table preview. It returns the same shape +information_schema did, so `mapType` and the callers were unchanged: +`data_type` is `format_type` with the modifier stripped, which gives the same +spelling (`character varying`, `numeric`), and precision and scale are unpacked +from `atttypmod`. The table browser lists from `pg_class` by `relkind` for the +same reason. + ## Territory scoping An account sees and changes only its own territory. The list lives on diff --git a/lib/utils.js b/lib/utils.js index c1245e8..a485a8f 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -3,7 +3,41 @@ function fcTable(tname, versionId) { return `pf.fc_${tname}_${versionId}`; } -// map information_schema data_type to a clean postgres column type +// The columns of a relation, from pg_catalog rather than information_schema. +// +// information_schema.columns omits materialized views -- they are not in the +// SQL standard -- so a source built on one looked like it had no columns at +// all: registering it seeded nothing, and creating a version failed with "No +// usable columns in col_meta" while col_meta plainly had thirty-six. +// +// The shape matches what information_schema returned, so mapType and the +// callers did not have to change. data_type is format_type with the modifier +// stripped, which gives the same spelling information_schema uses ('character +// varying', 'numeric'), and the numeric precision and scale are unpacked from +// atttypmod the way information_schema does internally. +// +// Takes $1 = schema, $2 = relation name. +const RELATION_COLUMNS_SQL = ` + SELECT a.attname AS column_name + ,regexp_replace(format_type(a.atttypid, a.atttypmod), '\\(.*\\)$', '') AS data_type + ,a.attnum AS ordinal_position + ,CASE WHEN a.attnotnull THEN 'NO' ELSE 'YES' END AS is_nullable + ,CASE WHEN a.atttypid = 'numeric'::regtype AND a.atttypmod > 4 + THEN ((a.atttypmod - 4) >> 16) & 65535 END AS numeric_precision + ,CASE WHEN a.atttypid = 'numeric'::regtype AND a.atttypmod > 4 + THEN (a.atttypmod - 4) & 65535 END AS numeric_scale + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE TRUE + AND n.nspname = $1 + AND c.relname = $2 + AND c.relkind IN ('r', 'v', 'm', 'f', 'p') + AND a.attnum > 0 + AND NOT a.attisdropped +`; + +// map a data_type name to a clean postgres column type function mapType(dataType, numericPrecision, numericScale) { switch (dataType) { case 'character varying': @@ -36,4 +70,4 @@ function mapType(dataType, numericPrecision, numericScale) { } } -module.exports = { fcTable, mapType }; +module.exports = { fcTable, mapType, RELATION_COLUMNS_SQL }; diff --git a/routes/sources.js b/routes/sources.js index 67eefcd..b01865b 100644 --- a/routes/sources.js +++ b/routes/sources.js @@ -1,5 +1,6 @@ const express = require('express'); const { generateSQL, buildTerritoryClause } = require('../lib/sql_generator'); +const { RELATION_COLUMNS_SQL } = require('../lib/utils'); const { sessionTerritory } = require('../lib/auth'); const { sessionUser } = require('../lib/auth'); @@ -42,15 +43,14 @@ module.exports = function(pool) { ); const source = src.rows[0]; - // seed col_meta from information_schema + // seed col_meta from the source's real columns await client.query(` INSERT INTO pf.col_meta (source_id, cname, role, opos) - SELECT $1, column_name, 'dimension', ordinal_position - FROM information_schema.columns - WHERE table_schema = $2 AND table_name = $3 + SELECT $3, column_name, 'dimension', ordinal_position + FROM (${RELATION_COLUMNS_SQL}) c ORDER BY ordinal_position ON CONFLICT (source_id, cname) DO NOTHING - `, [source.id, schema, tname]); + `, [schema, tname, source.id]); await client.query('COMMIT'); res.status(201).json(source); diff --git a/routes/tables.js b/routes/tables.js index 7c9eca3..3dd4ee4 100644 --- a/routes/tables.js +++ b/routes/tables.js @@ -1,4 +1,5 @@ const express = require('express'); +const { RELATION_COLUMNS_SQL } = require('../lib/utils'); module.exports = function(pool) { const router = express.Router(); @@ -7,15 +8,18 @@ module.exports = function(pool) { router.get('/tables', async (req, res) => { try { const result = await pool.query(` + -- pg_class, not information_schema.tables, which omits + -- materialized views: gs.osm_skinny is one, and the browser + -- could not offer what the app is already built on. SELECT - t.table_schema AS schema, - t.table_name AS tname, + n.nspname AS schema, + c.relname AS tname, c.reltuples::bigint AS row_estimate - FROM information_schema.tables t - LEFT JOIN pg_namespace n ON n.nspname = t.table_schema - LEFT JOIN pg_class c ON c.relname = t.table_name AND c.relnamespace = n.oid - WHERE t.table_schema NOT IN ('pg_catalog', 'information_schema', 'pf') - ORDER BY t.table_schema, t.table_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'v', 'm', 'f', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pf') + ORDER BY n.nspname, c.relname `); res.json(result.rows); } catch (err) { @@ -31,12 +35,8 @@ module.exports = function(pool) { return res.status(400).json({ error: 'Invalid schema or table name' }); } try { - const cols = await pool.query(` - SELECT column_name, data_type, is_nullable, ordinal_position - FROM information_schema.columns - WHERE table_schema = $1 AND table_name = $2 - ORDER BY ordinal_position - `, [schema, tname]); + const cols = await pool.query( + `${RELATION_COLUMNS_SQL} ORDER BY ordinal_position`, [schema, tname]); const rows = await pool.query( `SELECT * FROM ${schema}.${tname} LIMIT 5` diff --git a/routes/versions.js b/routes/versions.js index f62f0c3..11e2652 100644 --- a/routes/versions.js +++ b/routes/versions.js @@ -1,5 +1,5 @@ const express = require('express'); -const { fcTable, mapType } = require('../lib/utils'); +const { fcTable, mapType, RELATION_COLUMNS_SQL } = require('../lib/utils'); const { sessionUser, sessionTerritory } = require('../lib/auth'); const { buildTerritoryClause } = require('../lib/sql_generator'); @@ -39,8 +39,9 @@ module.exports = function(pool) { } const source = srcResult.rows[0]; - // fetch col_meta joined to information_schema for data types + // col_meta joined to the source's real columns, for the data types const colResult = await client.query(` + WITH cols AS (${RELATION_COLUMNS_SQL}) SELECT m.cname, m.role, @@ -49,14 +50,11 @@ module.exports = function(pool) { i.numeric_precision, i.numeric_scale FROM pf.col_meta m - JOIN information_schema.columns i - ON i.table_schema = $2 - AND i.table_name = $3 - AND i.column_name = m.cname - WHERE m.source_id = $1 + JOIN cols i ON i.column_name = m.cname + WHERE m.source_id = $3 AND m.role NOT IN ('ignore') ORDER BY m.opos - `, [sourceId, source.schema, source.tname]); + `, [source.schema, source.tname, sourceId]); if (colResult.rows.length === 0) { return res.status(400).json({