Compare commits

..

2 Commits

Author SHA1 Message Date
78c832eb1f fix: quote json/jsonb/bpchar/uuid values in migration INSERTs
The migration INSERT builder's switch quoted varchar/text/char/clob/date/time
but let everything else fall to a default that emits rs.getString() unquoted
(correct for numerics, broken for strings). A pg->SQL Server pull of a jsonb
column failed with "Incorrect syntax near 'volume_bucket'" — the JSON text's
embedded double-quotes were read as a SQL identifier. Quote json/jsonb, plus
bpchar (PG char(n)) and uuid, like varchar.

Note: the default case still emits unquoted; other unhandled string types
(e.g. bool->'t'/'f') would need similar handling or a quote-by-default flip.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 16:31:45 -04:00
a1c9ea26ce fix: stream PostgreSQL source (migration mode) instead of buffering it all
setFetchSize(10000) is a no-op on the PostgreSQL JDBC driver while autoCommit
is true — the driver loads the entire ResultSet into memory, OOM/GC-thrashing
on large source tables (a pg->SQL Server pull pinned the box: 4GB heap, swap
full, 0 rows written). PG only uses a server-side cursor when autoCommit is
false AND fetchSize > 0.

Set the source connection to manual commit ONLY in migration mode: the
migration source is read-only so never committing is harmless. Query mode is
excluded on purpose — callers (pipekit's run_dest_sql) run committed DDL/DML
through query mode, and autoCommit=false would roll those back on close.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 15:51:57 -04:00

View File

@ -208,6 +208,15 @@ public class jrunner {
} }
try { try {
scon = DriverManager.getConnection(scu, scn, scp); scon = DriverManager.getConnection(scu, scn, scp);
// Migration mode only: PostgreSQL ignores setFetchSize unless autoCommit
// is false without this it buffers the ENTIRE result set into memory
// (OOM on big tables). The migration source is read-only, so never
// committing is harmless. Do NOT do this in query mode: callers run
// committed DDL/DML through query mode, and autoCommit=false would roll
// those statements back on connection close.
if (!queryMode) {
scon.setAutoCommit(false);
}
} catch (SQLException e) { } catch (SQLException e) {
System.out.println("issue connecting to source:"); System.out.println("issue connecting to source:");
e.printStackTrace(); e.printStackTrace();
@ -302,7 +311,16 @@ public class jrunner {
nr = ""; nr = "";
for (int i = 1; i <= cols; i++){ for (int i = 1; i <= cols; i++){
switch (dtn[i].toUpperCase()){ switch (dtn[i].toUpperCase()){
// PG string-ish types that otherwise fall to the default
// case and get emitted UNQUOTED (breaking the INSERT):
// jsonb/json carry embedded quotes, bpchar is PG's char(n),
// uuid is a quoted literal. Quote them like varchar.
case "JSON":
case "JSONB":
case "BPCHAR":
case "UUID":
case "VARCHAR": case "VARCHAR":
case "NVARCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -313,6 +331,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "TEXT": case "TEXT":
case "NTEXT":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -323,6 +342,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CHAR": case "CHAR":
case "NCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -333,6 +353,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CLOB": case "CLOB":
case "NCLOB":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";