diff --git a/lib/sql_generator.js b/lib/sql_generator.js index 63a082a..a6b0076 100644 --- a/lib/sql_generator.js +++ b/lib/sql_generator.js @@ -34,11 +34,17 @@ function grainOf(colMeta) { // parts and chr(30) stands in for NULL, so ('a', NULL) cannot collide with // (NULL, 'a') and a NULL stays distinct from an empty string — a collision // would silently merge two groups into one indexed row. - const key = (pfx = '') => `concat_ws(chr(31), ${[ + // + // md5 of that, rather than the concatenation itself, because the key is an + // opaque handle -- nothing reads it but table.update() and table.remove(). + // The raw form averaged 233 chars on a 24-column grain and, being unique per + // row, defeated Arrow's dictionary encoding: 65.6 MB of a 109 MB payload, + // more than every other column combined. 128 bits keeps collisions unreachable. + const key = (pfx = '') => `md5(concat_ws(chr(31), ${[ ...cols.map(c => `COALESCE(${pfx}${q(c)}::text, chr(30))`), `${pfx}pf_iter`, `${pfx}pf_logid::text` - ].join(', ')})`; + ].join(', ')}))`; const groupCols = (pfx = '') => [...cols.map(c => `${pfx}${q(c)}`), `${pfx}pf_iter`, `${pfx}pf_logid`];