pt/pf_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Read source columns from pg_catalog, not information_schema

Browse Source 
information_schema.columns omits materialized views -- they are not in the
SQL standard -- and gs.osm_skinny is one. So the source this app runs on
looked like it had no columns at all: creating a version failed with "No
usable columns in col_meta" while col_meta plainly held thirty-six, and
registering such a source would have seeded nothing.

RELATION_COLUMNS_SQL returns the same shape information_schema did, so
mapType and every caller are unchanged: data_type is format_type with the
modifier stripped, which spells things the same way ('character varying',
'numeric'), and precision and scale are unpacked from atttypmod as
information_schema does internally. Verified against the live matview -- 36
usable columns, and the types map to exactly what fc_osm_skinny_29 already
has.

The table browser had the same blind spot from information_schema.tables and
now lists from pg_class by relkind, so a materialized view can be registered
rather than merely used by a source registered when it was still a table.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-09-18 16:12:05 -04:00
parent 469e4cc957
commit 23032a0658
5 changed files with 76 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CLAUDE.md
View File

@ -353,6 +353,22 @@ effects call the API immediately.
`pf_user: 'admin'`, which any client could have set to anything. The audit log
now names the account that made the change.
## Source columns come from pg_catalog
`information_schema.columns` omits **materialized views** — they are not in the
SQL standard — and `gs.osm_skinny` is one. So the source the whole app is built
on looked like it had no columns: registering it seeded nothing, and creating a
version failed with "No usable columns in col_meta" while col_meta plainly held
thirty-six.
`RELATION_COLUMNS_SQL` in `lib/utils.js` is the replacement, used by version
creation, source registration and the table preview. It returns the same shape
information_schema did, so `mapType` and the callers were unchanged:
`data_type` is `format_type` with the modifier stripped, which gives the same
spelling (`character varying`, `numeric`), and precision and scale are unpacked
from `atttypmod`. The table browser lists from `pg_class` by `relkind` for the
same reason.
## Territory scoping
An account sees and changes only its own territory. The list lives on

38
lib/utils.js
View File

@ -3,7 +3,41 @@ function fcTable(tname, versionId) {
return `pf.fc_${tname}_${versionId}`;
}
// map information_schema data_type to a clean postgres column type
// The columns of a relation, from pg_catalog rather than information_schema.
//
// information_schema.columns omits materialized views -- they are not in the
// SQL standard -- so a source built on one looked like it had no columns at
// all: registering it seeded nothing, and creating a version failed with "No
// usable columns in col_meta" while col_meta plainly had thirty-six.
//
// The shape matches what information_schema returned, so mapType and the
// callers did not have to change. data_type is format_type with the modifier
// stripped, which gives the same spelling information_schema uses ('character
// varying', 'numeric'), and the numeric precision and scale are unpacked from
// atttypmod the way information_schema does internally.
//
// Takes $1 = schema, $2 = relation name.
const RELATION_COLUMNS_SQL = `
SELECT a.attname AS column_name
,regexp_replace(format_type(a.atttypid, a.atttypmod), '\\(.*\\)$', '') AS data_type
,a.attnum AS ordinal_position
,CASE WHEN a.attnotnull THEN 'NO' ELSE 'YES' END AS is_nullable
,CASE WHEN a.atttypid = 'numeric'::regtype AND a.atttypmod > 4
THEN ((a.atttypmod - 4) >> 16) & 65535 END AS numeric_precision
,CASE WHEN a.atttypid = 'numeric'::regtype AND a.atttypmod > 4
THEN (a.atttypmod - 4) & 65535 END AS numeric_scale
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE TRUE
AND n.nspname = $1
AND c.relname = $2
AND c.relkind IN ('r', 'v', 'm', 'f', 'p')
AND a.attnum > 0
AND NOT a.attisdropped
`;
// map a data_type name to a clean postgres column type
function mapType(dataType, numericPrecision, numericScale) {
switch (dataType) {
case 'character varying':
@ -36,4 +70,4 @@ function mapType(dataType, numericPrecision, numericScale) {
}
}
module.exports = { fcTable, mapType };
module.exports = { fcTable, mapType, RELATION_COLUMNS_SQL };

10
routes/sources.js
View File

@ -1,5 +1,6 @@
const express = require('express');
const { generateSQL, buildTerritoryClause } = require('../lib/sql_generator');
const { RELATION_COLUMNS_SQL } = require('../lib/utils');
const { sessionTerritory } = require('../lib/auth');
const { sessionUser } = require('../lib/auth');
@ -42,15 +43,14 @@ module.exports = function(pool) {
);
const source = src.rows[0];
// seed col_meta from information_schema
// seed col_meta from the source's real columns
await client.query(`
INSERT INTO pf.col_meta (source_id, cname, role, opos)
SELECT $1, column_name, 'dimension', ordinal_position
FROM information_schema.columns
WHERE table_schema = $2 AND table_name = $3
SELECT $3, column_name, 'dimension', ordinal_position
FROM (${RELATION_COLUMNS_SQL}) c
ORDER BY ordinal_position
ON CONFLICT (source_id, cname) DO NOTHING
`, [source.id, schema, tname]);
`, [schema, tname, source.id]);
await client.query('COMMIT');
res.status(201).json(source);

26
routes/tables.js
View File

@ -1,4 +1,5 @@
const express = require('express');
const { RELATION_COLUMNS_SQL } = require('../lib/utils');
module.exports = function(pool) {
const router = express.Router();
@ -7,15 +8,18 @@ module.exports = function(pool) {
router.get('/tables', async (req, res) => {
try {
const result = await pool.query(`
-- pg_class, not information_schema.tables, which omits
-- materialized views: gs.osm_skinny is one, and the browser
-- could not offer what the app is already built on.
SELECT
t.table_schema AS schema,
t.table_name AS tname,
n.nspname AS schema,
c.relname AS tname,
c.reltuples::bigint AS row_estimate
FROM information_schema.tables t
LEFT JOIN pg_namespace n ON n.nspname = t.table_schema
LEFT JOIN pg_class c ON c.relname = t.table_name AND c.relnamespace = n.oid
WHERE t.table_schema NOT IN ('pg_catalog', 'information_schema', 'pf')
ORDER BY t.table_schema, t.table_name
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r', 'v', 'm', 'f', 'p')
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pf')
ORDER BY n.nspname, c.relname
`);
res.json(result.rows);
} catch (err) {
@ -31,12 +35,8 @@ module.exports = function(pool) {
return res.status(400).json({ error: 'Invalid schema or table name' });
}
try {
const cols = await pool.query(`
SELECT column_name, data_type, is_nullable, ordinal_position
FROM information_schema.columns
WHERE table_schema = $1 AND table_name = $2
ORDER BY ordinal_position
`, [schema, tname]);
const cols = await pool.query(
`${RELATION_COLUMNS_SQL} ORDER BY ordinal_position`, [schema, tname]);
const rows = await pool.query(
`SELECT * FROM ${schema}.${tname} LIMIT 5`

14
routes/versions.js
View File

@ -1,5 +1,5 @@
const express = require('express');
const { fcTable, mapType } = require('../lib/utils');
const { fcTable, mapType, RELATION_COLUMNS_SQL } = require('../lib/utils');
const { sessionUser, sessionTerritory } = require('../lib/auth');
const { buildTerritoryClause } = require('../lib/sql_generator');
@ -39,8 +39,9 @@ module.exports = function(pool) {
}
const source = srcResult.rows[0];
// fetch col_meta joined to information_schema for data types
// col_meta joined to the source's real columns, for the data types
const colResult = await client.query(`
WITH cols AS (${RELATION_COLUMNS_SQL})
SELECT
m.cname,
m.role,
@ -49,14 +50,11 @@ module.exports = function(pool) {
i.numeric_precision,
i.numeric_scale
FROM pf.col_meta m
JOIN information_schema.columns i
ON i.table_schema = $2
AND i.table_name = $3
AND i.column_name = m.cname
WHERE m.source_id = $1
JOIN cols i ON i.column_name = m.cname
WHERE m.source_id = $3
AND m.role NOT IN ('ignore')
ORDER BY m.opos
`, [sourceId, source.schema, source.tname]);
`, [source.schema, source.tname, sourceId]);
if (colResult.rows.length === 0) {
return res.status(400).json({