Compare commits

...

4 Commits

Author SHA1 Message Date
78c832eb1f fix: quote json/jsonb/bpchar/uuid values in migration INSERTs
The migration INSERT builder's switch quoted varchar/text/char/clob/date/time
but let everything else fall to a default that emits rs.getString() unquoted
(correct for numerics, broken for strings). A pg->SQL Server pull of a jsonb
column failed with "Incorrect syntax near 'volume_bucket'" — the JSON text's
embedded double-quotes were read as a SQL identifier. Quote json/jsonb, plus
bpchar (PG char(n)) and uuid, like varchar.

Note: the default case still emits unquoted; other unhandled string types
(e.g. bool->'t'/'f') would need similar handling or a quote-by-default flip.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 16:31:45 -04:00
a1c9ea26ce fix: stream PostgreSQL source (migration mode) instead of buffering it all
setFetchSize(10000) is a no-op on the PostgreSQL JDBC driver while autoCommit
is true — the driver loads the entire ResultSet into memory, OOM/GC-thrashing
on large source tables (a pg->SQL Server pull pinned the box: 4GB heap, swap
full, 0 rows written). PG only uses a server-side cursor when autoCommit is
false AND fetchSize > 0.

Set the source connection to manual commit ONLY in migration mode: the
migration source is read-only so never committing is harmless. Query mode is
excluded on purpose — callers (pipekit's run_dest_sql) run committed DDL/DML
through query mode, and autoCommit=false would roll those back on close.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 15:51:57 -04:00
f632a77e8e Add SQL Server datetime type variants to type handling
Adds DATETIME, DATETIME2, SMALLDATETIME, and DATETIMEOFFSET cases to
the TIMESTAMP branch so SQL Server datetime columns are handled correctly.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-24 02:23:07 -04:00
ff4cf25585 Add ~/.jrunnerpass named connection profile support
Implements .pgpass-style credential file for jrunner. Named aliases can
be used with -sc and -dc flags instead of spelling out -scu/-scn/-scp
for each invocation. Explicit flags still take priority over the file.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-03 15:59:04 -05:00

View File

@ -5,6 +5,7 @@ import java.nio.file.Files;
import java.nio.file.Path ; import java.nio.file.Path ;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.time.*; import java.time.*;
import java.io.IOException;
public class jrunner { public class jrunner {
//static final String QUERY = "SELECT * from rlarp.osm LIMIT 100"; //static final String QUERY = "SELECT * from rlarp.osm LIMIT 100";
@ -14,9 +15,11 @@ public class jrunner {
String scu = ""; String scu = "";
String scn = ""; String scn = "";
String scp = ""; String scp = "";
String scAlias = "";
String dcu = ""; String dcu = "";
String dcn = ""; String dcn = "";
String dcp = ""; String dcp = "";
String dcAlias = "";
String sq = ""; String sq = "";
String dt = ""; String dt = "";
Boolean trim = true; Boolean trim = true;
@ -41,24 +44,36 @@ public class jrunner {
Timestamp tsStart = null; Timestamp tsStart = null;
Timestamp tsEnd = null; Timestamp tsEnd = null;
msg = "jrunner version 1.1"; msg = "jrunner version 1.2";
msg = msg + nl + "-sc source connection alias (from ~/.jrunnerpass)";
msg = msg + nl + "-scu source jdbc url"; msg = msg + nl + "-scu source jdbc url";
msg = msg + nl + "-scn source username"; msg = msg + nl + "-scn source username";
msg = msg + nl + "-scp source passowrd"; msg = msg + nl + "-scp source password";
msg = msg + nl + "-dc destination connection alias (from ~/.jrunnerpass)";
msg = msg + nl + "-dcu destination jdbc url"; msg = msg + nl + "-dcu destination jdbc url";
msg = msg + nl + "-dcn destination username"; msg = msg + nl + "-dcn destination username";
msg = msg + nl + "-dcp destination passowrd"; msg = msg + nl + "-dcp destination password";
msg = msg + nl + "-sq path to source query"; msg = msg + nl + "-sq path to source query";
msg = msg + nl + "-dt fully qualified name of destination table"; msg = msg + nl + "-dt fully qualified name of destination table";
msg = msg + nl + "-t trim text"; msg = msg + nl + "-t trim text";
msg = msg + nl + "-c clear target table"; msg = msg + nl + "-c clear target table";
msg = msg + nl + "-f output format (csv, tsv, table, json) - default: csv"; msg = msg + nl + "-f output format (csv, tsv, table, json) - default: csv";
msg = msg + nl + "--help info"; msg = msg + nl + "--help info";
msg = msg + nl + "";
msg = msg + nl + "~/.jrunnerpass format:";
msg = msg + nl + " [alias]";
msg = msg + nl + " url=jdbc:...";
msg = msg + nl + " user=username";
msg = msg + nl + " pass=password";
//---------------------------------------parse args into variables------------------------------------------------- //---------------------------------------parse args into variables-------------------------------------------------
for (int i = 0; i < args.length; i = i +1 ){ for (int i = 0; i < args.length; i = i +1 ){
switch (args[i]) { switch (args[i]) {
//source connection alias
case "-sc":
scAlias = args[i+1];
break;
//source connection string //source connection string
case "-scu": case "-scu":
scu = args[i+1]; scu = args[i+1];
@ -68,10 +83,13 @@ public class jrunner {
scn = args[i+1]; scn = args[i+1];
break; break;
//source password //source password
//import java.time.*;
case "-scp": case "-scp":
scp = args[i+1]; scp = args[i+1];
break; break;
//destination connection alias
case "-dc":
dcAlias = args[i+1];
break;
//destination connection string //destination connection string
case "-dcu": case "-dcu":
dcu = args[i+1]; dcu = args[i+1];
@ -133,6 +151,31 @@ public class jrunner {
} }
} }
// Resolve connection aliases from ~/.jrunnerpass
if (!scAlias.isEmpty() || !dcAlias.isEmpty()) {
Map<String, String[]> connections = loadPassFile();
if (!scAlias.isEmpty()) {
String[] sc = connections.get(scAlias);
if (sc == null) {
System.err.println("Error: source alias '" + scAlias + "' not found in ~/.jrunnerpass");
System.exit(1);
}
if (scu.isEmpty()) scu = sc[0];
if (scn.isEmpty()) scn = sc[1];
if (scp.isEmpty()) scp = sc[2];
}
if (!dcAlias.isEmpty()) {
String[] dc = connections.get(dcAlias);
if (dc == null) {
System.err.println("Error: destination alias '" + dcAlias + "' not found in ~/.jrunnerpass");
System.exit(1);
}
if (dcu.isEmpty()) dcu = dc[0];
if (dcn.isEmpty()) dcn = dc[1];
if (dcp.isEmpty()) dcp = dc[2];
}
}
// Detect query mode when destination flags are not provided // Detect query mode when destination flags are not provided
queryMode = dcu.isEmpty() && dcn.isEmpty() && dcp.isEmpty() && dt.isEmpty(); queryMode = dcu.isEmpty() && dcn.isEmpty() && dcp.isEmpty() && dt.isEmpty();
@ -165,6 +208,15 @@ public class jrunner {
} }
try { try {
scon = DriverManager.getConnection(scu, scn, scp); scon = DriverManager.getConnection(scu, scn, scp);
// Migration mode only: PostgreSQL ignores setFetchSize unless autoCommit
// is false without this it buffers the ENTIRE result set into memory
// (OOM on big tables). The migration source is read-only, so never
// committing is harmless. Do NOT do this in query mode: callers run
// committed DDL/DML through query mode, and autoCommit=false would roll
// those statements back on connection close.
if (!queryMode) {
scon.setAutoCommit(false);
}
} catch (SQLException e) { } catch (SQLException e) {
System.out.println("issue connecting to source:"); System.out.println("issue connecting to source:");
e.printStackTrace(); e.printStackTrace();
@ -259,7 +311,16 @@ public class jrunner {
nr = ""; nr = "";
for (int i = 1; i <= cols; i++){ for (int i = 1; i <= cols; i++){
switch (dtn[i].toUpperCase()){ switch (dtn[i].toUpperCase()){
// PG string-ish types that otherwise fall to the default
// case and get emitted UNQUOTED (breaking the INSERT):
// jsonb/json carry embedded quotes, bpchar is PG's char(n),
// uuid is a quoted literal. Quote them like varchar.
case "JSON":
case "JSONB":
case "BPCHAR":
case "UUID":
case "VARCHAR": case "VARCHAR":
case "NVARCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -270,6 +331,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "TEXT": case "TEXT":
case "NTEXT":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -280,6 +342,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CHAR": case "CHAR":
case "NCHAR":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -290,6 +353,7 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "CLOB": case "CLOB":
case "NCLOB":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -320,6 +384,10 @@ public class jrunner {
nc = "'" + nc + "'"; nc = "'" + nc + "'";
break; break;
case "TIMESTAMP": case "TIMESTAMP":
case "DATETIME":
case "DATETIME2":
case "SMALLDATETIME":
case "DATETIMEOFFSET":
nc = rs.getString(i); nc = rs.getString(i);
if (rs.wasNull() || nc == null) { if (rs.wasNull() || nc == null) {
nc = "NULL"; nc = "NULL";
@ -482,4 +550,47 @@ public class jrunner {
private static String escapeTSV(String value) { private static String escapeTSV(String value) {
return value.replaceAll("\t", " ").replaceAll("\n", " ").replaceAll("\r", " "); return value.replaceAll("\t", " ").replaceAll("\n", " ").replaceAll("\r", " ");
} }
// Loads ~/.jrunnerpass and returns a map of alias -> {url, user, pass}
// Format:
// [alias]
// url=jdbc:...
// user=username
// pass=password
private static Map<String, String[]> loadPassFile() {
Map<String, String[]> connections = new LinkedHashMap<>();
Path passFile = Paths.get(System.getProperty("user.home"), ".jrunnerpass");
if (!Files.exists(passFile)) {
System.err.println("Error: ~/.jrunnerpass not found");
System.exit(1);
}
try {
String currentAlias = null;
String url = "", user = "", pass = "";
for (String line : Files.readAllLines(passFile)) {
line = line.trim();
if (line.isEmpty() || line.startsWith("#")) continue;
if (line.startsWith("[") && line.endsWith("]")) {
if (currentAlias != null) {
connections.put(currentAlias, new String[]{url, user, pass});
}
currentAlias = line.substring(1, line.length() - 1).trim();
url = ""; user = ""; pass = "";
} else if (line.startsWith("url=")) {
url = line.substring(4);
} else if (line.startsWith("user=")) {
user = line.substring(5);
} else if (line.startsWith("pass=")) {
pass = line.substring(5);
}
}
if (currentAlias != null) {
connections.put(currentAlias, new String[]{url, user, pass});
}
} catch (IOException e) {
System.err.println("Error reading ~/.jrunnerpass: " + e.getMessage());
System.exit(1);
}
return connections;
}
} }