pt/jrunner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 5 Wiki Activity

fix: stream PostgreSQL source (migration mode) instead of buffering it all

Browse Source 
setFetchSize(10000) is a no-op on the PostgreSQL JDBC driver while autoCommit
is true — the driver loads the entire ResultSet into memory, OOM/GC-thrashing
on large source tables (a pg->SQL Server pull pinned the box: 4GB heap, swap
full, 0 rows written). PG only uses a server-side cursor when autoCommit is
false AND fetchSize > 0.

Set the source connection to manual commit ONLY in migration mode: the
migration source is read-only so never committing is harmless. Query mode is
excluded on purpose — callers (pipekit's run_dest_sql) run committed DDL/DML
through query mode, and autoCommit=false would roll those back on close.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-06-17 15:37:47 -04:00
parent f632a77e8e
commit a1c9ea26ce
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
jrunner/src/main/java/jrunner/jrunner.java
View File

@ -208,6 +208,15 @@ public class jrunner {
} }
try { try {
scon = DriverManager.getConnection(scu, scn, scp); scon = DriverManager.getConnection(scu, scn, scp);
// Migration mode only: PostgreSQL ignores setFetchSize unless autoCommit
// is false — without this it buffers the ENTIRE result set into memory
// (OOM on big tables). The migration source is read-only, so never
// committing is harmless. Do NOT do this in query mode: callers run
// committed DDL/DML through query mode, and autoCommit=false would roll
// those statements back on connection close.
if (!queryMode) {
scon.setAutoCommit(false);
}
} catch (SQLException e) { } catch (SQLException e) {
System.out.println("issue connecting to source:"); System.out.println("issue connecting to source:");
e.printStackTrace(); e.printStackTrace();
@ -303,6 +312,7 @@ public class jrunner {
for (int i = 1; i <= cols; i++){ for (int i = 1; i <= cols; i++){
switch (dtn[i].toUpperCase()){ switch (dtn[i].toUpperCase()){
case "VARCHAR": case "VARCHAR":
case "NVARCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -313,6 +323,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "TEXT": case "TEXT":
case "NTEXT":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -323,6 +334,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CHAR": case "CHAR":
case "NCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -333,6 +345,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CLOB": case "CLOB":
case "NCLOB":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";