From 18070143232d7ee87c80906c67c8a2117e14d06f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 16 Sep 2026 22:07:29 -0400 Subject: [PATCH] Hash pf_gkey instead of shipping the concatenated grain tuple pf_gkey is the Perspective table index -- an opaque handle read only by table.update() and table.remove(). It was the raw concat_ws of every grain column, which on the 24-column grain of fc_osm_skinny_29 averaged 233 characters. Being unique per row by construction, it also defeated Arrow's dictionary encoding, so it alone accounted for 65.6 MB of a 109 MB payload -- more than the other 31 columns combined, all of which do dictionary cleanly. md5 of the same string keeps the determinism undo depends on (routes/log.js recomputes the key through this same grainOf) and keeps collisions out of reach at 128 bits, while fixing the width at 32 characters. Measured on the live 285,685-row aggregate: payload 109.0 MB -> 54.2 MB, all 285,685 keys still distinct. Co-Authored-By: Claude Opus 5 (1M context) --- lib/sql_generator.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 63a082a..a6b0076 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -34,11 +34,17 @@ function grainOf(colMeta) { // parts and chr(30) stands in for NULL, so ('a', NULL) cannot collide with // (NULL, 'a') and a NULL stays distinct from an empty string — a collision // would silently merge two groups into one indexed row. - const key = (pfx = '') => `concat_ws(chr(31), ${[ + // + // md5 of that, rather than the concatenation itself, because the key is an + // opaque handle -- nothing reads it but table.update() and table.remove(). + // The raw form averaged 233 chars on a 24-column grain and, being unique per + // row, defeated Arrow's dictionary encoding: 65.6 MB of a 109 MB payload, + // more than every other column combined. 128 bits keeps collisions unreachable. + const key = (pfx = '') => `md5(concat_ws(chr(31), ${[ ...cols.map(c => `COALESCE(${pfx}${q(c)}::text, chr(30))`), `${pfx}pf_iter`, `${pfx}pf_logid::text` - ].join(', ')})`; + ].join(', ')}))`; const groupCols = (pfx = '') => [...cols.map(c => `${pfx}${q(c)}`), `${pfx}pf_iter`, `${pfx}pf_logid`];