diff --git a/jrunner/src/main/java/jrunner/jrunner.java b/jrunner/src/main/java/jrunner/jrunner.java index e4533a3..aef92aa 100644 --- a/jrunner/src/main/java/jrunner/jrunner.java +++ b/jrunner/src/main/java/jrunner/jrunner.java @@ -4,6 +4,7 @@ import java.util.*; import java.nio.file.Files; import java.nio.file.Path ; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; import java.time.*; import java.io.IOException; import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy; @@ -40,6 +41,7 @@ public class jrunner { String nl = "\n"; Boolean queryMode = false; String outputFormat = "csv"; + String passFilePath = ""; String msg = ""; Connection scon = null; Connection dcon = null; @@ -66,10 +68,15 @@ public class jrunner { msg = msg + nl + "-t trim text"; msg = msg + nl + "-c clear target table"; msg = msg + nl + "-b bulk copy into destination (SQL Server dest only)"; - msg = msg + nl + "-f output format (csv, tsv, table, json) - default: csv"; + msg = msg + nl + "-f output format (csv, tsv, json) - default: csv"; + msg = msg + nl + "--passfile path to the connection alias file - default: ~/.jrunnerpass"; + msg = msg + nl + "--strict exit 1 on failure (default exits 0, for legacy callers)"; msg = msg + nl + "--help info"; msg = msg + nl + ""; - msg = msg + nl + "~/.jrunnerpass format:"; + msg = msg + nl + "Prefer -sc/-dc aliases over -scp/-dcp: a password given on the"; + msg = msg + nl + "command line is visible to any local user via 'ps'."; + msg = msg + nl + ""; + msg = msg + nl + "passfile format:"; msg = msg + nl + " [alias]"; msg = msg + nl + " url=jdbc:..."; msg = msg + nl + " user=username"; @@ -77,6 +84,12 @@ public class jrunner { //---------------------------------------parse args into variables------------------------------------------------- + // Pre-scan: -sq can fail while the loop is still running, and it must + // honour --strict regardless of where the flag sits in argv. + for (String a : args) { + if (a.equals("--strict")) strictExit = true; + } + for (int i = 0; i < args.length; i = i +1 ){ switch (args[i]) { //source connection alias @@ -120,7 +133,7 @@ public class jrunner { catch (Exception e) { //System.out.println(nl + "error reasing source sql file: " + printStackTrace()); e.printStackTrace(); - System.exit(0); + die(); return; } break; @@ -140,6 +153,13 @@ public class jrunner { case "-f": outputFormat = args[i+1].toLowerCase(); break; + //alias file location (the service user has no home directory) + case "--passfile": + passFilePath = args[i+1]; + break; + //exit non-zero on failure (handled in the pre-scan above) + case "--strict": + break; case "-v": System.out.println(msg); return; @@ -163,13 +183,16 @@ public class jrunner { } } - // Resolve connection aliases from ~/.jrunnerpass + // Resolve connection aliases from the passfile if (!scAlias.isEmpty() || !dcAlias.isEmpty()) { - Map connections = loadPassFile(); + Path resolvedPassFile = passFilePath.isEmpty() + ? Paths.get(System.getProperty("user.home"), ".jrunnerpass") + : Paths.get(passFilePath); + Map connections = loadPassFile(resolvedPassFile); if (!scAlias.isEmpty()) { String[] sc = connections.get(scAlias); if (sc == null) { - System.err.println("Error: source alias '" + scAlias + "' not found in ~/.jrunnerpass"); + System.err.println("Error: source alias '" + scAlias + "' not found in " + resolvedPassFile); System.exit(1); } if (scu.isEmpty()) scu = sc[0]; @@ -179,7 +202,7 @@ public class jrunner { if (!dcAlias.isEmpty()) { String[] dc = connections.get(dcAlias); if (dc == null) { - System.err.println("Error: destination alias '" + dcAlias + "' not found in ~/.jrunnerpass"); + System.err.println("Error: destination alias '" + dcAlias + "' not found in " + resolvedPassFile); System.exit(1); } if (dcu.isEmpty()) dcu = dc[0]; @@ -210,7 +233,7 @@ public class jrunner { Class.forName("com.ibm.as400.access.AS400JDBCDriver"); } catch (ClassNotFoundException cnf) { System.out.println("The AS400 JDBC driver did not load"); - System.exit(0); + die(); } //-------------------------------------------establish connections------------------------------------------------- @@ -232,7 +255,7 @@ public class jrunner { } catch (SQLException e) { System.out.println("issue connecting to source:"); e.printStackTrace(); - System.exit(0); + die(); } if (!queryMode) { System.out.println(" ✅ source database"); @@ -244,7 +267,7 @@ public class jrunner { } catch (SQLException e) { System.out.println("issue connecting to destination:"); e.printStackTrace(); - System.exit(0); + die(); } System.out.println(" ✅ destination database"); } @@ -253,18 +276,46 @@ public class jrunner { stmt = scon.createStatement(); stmt.setFetchSize(10000); tsStart = Timestamp.from(Instant.now()); + summaryStart = tsStart; if (!queryMode) { System.out.println(tsStart); } - rs = stmt.executeQuery(sq); - //while (rs.next()) { - // System.out.println(rs.getString("x")); - //} + // execute() rather than executeQuery(): a statement that returns no + // result set (DDL, INSERT, DELETE, TRUNCATE, MERGE) is a normal, + // successful outcome, but executeQuery() raises for it — Postgres + // "No results were returned by the query", SQL Server "The + // statement did not return a result set". Callers had to allowlist + // those messages to tell a real failure from a working TRUNCATE, + // which made a non-zero exit code impossible to adopt. + boolean hasResultSet = stmt.execute(sq); + // A stored procedure may report update counts before opening its + // cursor, and /opt/sync uses CALL statements as migration sources + // (rlarp.QUOTE_REBUILD and friends). Walk past any update counts to + // the first real result set instead of concluding there is none — + // executeQuery() would simply have thrown here. The last count is + // retained so a plain DML statement can still report rows affected. + int updated = -1; + while (!hasResultSet) { + int uc = stmt.getUpdateCount(); + if (uc == -1) break; // no further results of any kind + updated = uc; + hasResultSet = stmt.getMoreResults(); + } + if (!hasResultSet) { + if (!queryMode) { + System.out.println("------------no result set---------------------------------"); + System.out.println("rows affected: " + updated); + } + emitSummary("ok", updated >= 0 ? updated : -1, summaryElapsedMs()); + closeQuietly(scon, dcon, queryMode); + return; + } + rs = stmt.getResultSet(); } catch (SQLException e) { System.out.println("issue retrieving rows from source:"); e.printStackTrace(); - System.exit(0); - } + die(); + } //---------------------------------------build meta--------------------------------------------------------------- try { @@ -277,7 +328,7 @@ public class jrunner { dtn = new String[cols + 1]; } catch (SQLException e) { e.printStackTrace(); - System.exit(0); + die(); } try { for (int i = 1; i <= cols; i++){ @@ -288,7 +339,7 @@ public class jrunner { } } catch (SQLException e) { e.printStackTrace(); - System.exit(0); + die(); } //-------------------------clear the target table if requeted---------------------------------------------------- if (!queryMode && clear) { @@ -301,7 +352,7 @@ public class jrunner { } catch (SQLException e) { e.printStackTrace(); System.out.println(sql); - System.exit(0); + die(); } } if (queryMode) { @@ -310,7 +361,7 @@ public class jrunner { outputQueryResults(rs, cols, dtn, outputFormat); } catch (SQLException e) { e.printStackTrace(); - System.exit(0); + die(); } } else if (bulk && dcu.toLowerCase().startsWith("jdbc:sqlserver:")) { //-------------------------------bulk copy (SQL Server dest)------------------------------------------------- @@ -332,9 +383,10 @@ public class jrunner { // onto the last progress tick; the trailing " rows written" makes // it parseable. System.out.print("\r" + src.rowsWritten()); + summaryRows = src.rowsWritten(); } catch (Exception e) { e.printStackTrace(); - System.exit(0); + die(); } } else if (bulk && dcu.toLowerCase().startsWith("jdbc:postgresql:")) { //-------------------------------bulk copy (COPY, Postgres dest)-------------------------------------------- @@ -374,9 +426,10 @@ public class jrunner { } cin.endCopy(); System.out.print("\r" + rows); + summaryRows = rows; } catch (Exception e) { e.printStackTrace(); - System.exit(0); + die(); } } else { System.out.println("------------row count-------------------------------------"); @@ -510,7 +563,7 @@ public class jrunner { } catch (SQLException e) { e.printStackTrace(); System.out.println(sql); - System.exit(0); + die(); } sql = ""; } @@ -522,15 +575,16 @@ public class jrunner { stmtd = dcon.createStatement(); stmtd.executeUpdate(sql); System.out.print("\r" + t); + summaryRows = t; } catch (SQLException e) { e.printStackTrace(); System.out.println(sql); - System.exit(0); + die(); } } } catch (SQLException e) { e.printStackTrace(); - System.exit(0); + die(); } } //System.out.println(sql); @@ -550,6 +604,7 @@ public class jrunner { tsEnd = Timestamp.from(Instant.now()); System.out.println(tsStart); System.out.println(tsEnd); + emitSummary("ok", summaryRows, summaryElapsedMs()); } //long time = Duration.between(tsStart, tsEnd).toMillis(); //System.out.println("time elapsed: " + time); @@ -693,14 +748,85 @@ public class jrunner { case "tsv": outputTSV(rs, cols); break; - case "table": case "json": + outputJSON(rs, cols); + break; default: outputCSV(rs, cols); break; } } + // JSON output. Two things CSV cannot express, both of which callers need: + // + // * NULL vs empty string. outputCSV writes an empty field for both, so + // everything downstream — the merge, reconcile, the wizard — sees them + // as identical. Here NULL is JSON null and '' is "". + // * Column types. Without them the wizard's SQL-entry mode has to default + // every destination column to text, because a CSV stream carries no + // type metadata for the caller to map. + // + // Values are emitted as JSON strings rather than JSON numbers: a DECIMAL + // rendered as a JSON number would go through a float and lose exactness, + // and callers already treat query output as text. null is the one non-string. + // + // Written incrementally rather than assembled in memory, so a large result + // set streams the same way CSV does while still being one valid document. + private static void outputJSON(ResultSet rs, int cols) throws SQLException { + ResultSetMetaData md = rs.getMetaData(); + StringBuilder head = new StringBuilder("{\"columns\":["); + for (int i = 1; i <= cols; i++) { + if (i > 1) head.append(","); + head.append("{\"name\":\"").append(escapeJSON(md.getColumnName(i))).append("\""); + head.append(",\"type\":\"").append(escapeJSON(md.getColumnTypeName(i))).append("\""); + head.append(",\"precision\":").append(md.getPrecision(i)); + head.append(",\"scale\":").append(md.getScale(i)); + head.append("}"); + } + head.append("],\"rows\":["); + System.out.print(head); + + boolean firstRow = true; + while (rs.next()) { + if (!firstRow) System.out.print(","); + firstRow = false; + System.out.print("["); + for (int i = 1; i <= cols; i++) { + if (i > 1) System.out.print(","); + String value = rs.getString(i); + if (rs.wasNull() || value == null) { + System.out.print("null"); + } else { + System.out.print("\"" + escapeJSON(value) + "\""); + } + } + System.out.print("]"); + } + System.out.println("]}"); + } + + private static String escapeJSON(String value) { + if (value == null) return ""; + StringBuilder sb = new StringBuilder(value.length() + 16); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': sb.append("\\\""); break; + case '\\': sb.append("\\\\"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + case '\b': sb.append("\\b"); break; + case '\f': sb.append("\\f"); break; + default: + // Other control characters are illegal raw in JSON strings. + if (c < 0x20) sb.append(String.format("\\u%04x", (int) c)); + else sb.append(c); + } + } + return sb.toString(); + } + private static void outputCSV(ResultSet rs, int cols) throws SQLException { // Print header row for (int i = 1; i <= cols; i++) { @@ -754,23 +880,80 @@ public class jrunner { return value; } + // Historically every error path did printStackTrace() then System.exit(0), + // so a failed run was indistinguishable from a successful one to any + // caller checking $?. 112 /opt/sync scripts run under `set -e` against + // that behaviour, so flipping it unconditionally would change them all at + // once; --strict makes the correct exit code opt-in until they're audited. + private static boolean strictExit = false; + + private static void die() { + emitSummary("error", summaryRows, summaryElapsedMs()); + System.exit(strictExit ? 1 : 0); + } + + // Row count for the summary. Migration has three write paths (SQLServer + // bulk copy, Postgres COPY, batched INSERT) each with its own counter, so + // whichever runs records the total here. + private static long summaryRows = -1; + private static Timestamp summaryStart = null; + + private static long summaryElapsedMs() { + if (summaryStart == null) return -1; + return Duration.between(summaryStart.toInstant(), Instant.now()).toMillis(); + } + + // A single machine-readable line for the calling process. + // + // Deliberately on stderr: migration-mode stdout carries human-readable + // progress that pipekit streams to its live log, and query-mode stdout must + // stay pure CSV, so anything structured on stdout would corrupt one or the + // other. stderr is already captured in full by the caller. + // + // Replaces having to regex a row count out of prose — the count previously + // reached the caller only because a \r progress tick and a trailing + // " rows written" happened to render on the same line. + private static void emitSummary(String status, long rows, long ms) { + StringBuilder sb = new StringBuilder("@summary {\"status\":\""); + sb.append(status).append("\""); + if (rows >= 0) sb.append(",\"rows\":").append(rows); + if (ms >= 0) sb.append(",\"ms\":").append(ms); + sb.append("}"); + System.err.println(sb); + } + + // Best-effort connection close for the early return taken when a statement + // produced no result set. A failure here cannot invalidate work that the + // server already committed, so it is reported but not fatal. + private static void closeQuietly(Connection scon, Connection dcon, boolean queryMode) { + try { + if (scon != null) scon.close(); + if (!queryMode && dcon != null) dcon.close(); + } catch (SQLException e) { + System.err.println("issue closing connections: " + e.getMessage()); + } + } + private static String escapeTSV(String value) { return value.replaceAll("\t", " ").replaceAll("\n", " ").replaceAll("\r", " "); } - // Loads ~/.jrunnerpass and returns a map of alias -> {url, user, pass} + // Loads a passfile and returns a map of alias -> {url, user, pass} // Format: // [alias] // url=jdbc:... // user=username // pass=password - private static Map loadPassFile() { + // + // Defaults to ~/.jrunnerpass; --passfile overrides it, which is what lets a + // service account with no home directory keep its passwords off argv. + private static Map loadPassFile(Path passFile) { Map connections = new LinkedHashMap<>(); - Path passFile = Paths.get(System.getProperty("user.home"), ".jrunnerpass"); if (!Files.exists(passFile)) { - System.err.println("Error: ~/.jrunnerpass not found"); + System.err.println("Error: passfile not found: " + passFile); System.exit(1); } + warnIfGroupOrWorldReadable(passFile); try { String currentAlias = null; String url = "", user = "", pass = ""; @@ -795,9 +978,23 @@ public class jrunner { connections.put(currentAlias, new String[]{url, user, pass}); } } catch (IOException e) { - System.err.println("Error reading ~/.jrunnerpass: " + e.getMessage()); + System.err.println("Error reading passfile " + passFile + ": " + e.getMessage()); System.exit(1); } return connections; } + + // The passfile holds plaintext passwords, so a too-permissive mode defeats + // the point of moving them off argv. Warn rather than refuse — the file may + // be deliberately group-readable by a service group. + private static void warnIfGroupOrWorldReadable(Path passFile) { + try { + Set perms = Files.getPosixFilePermissions(passFile); + if (perms.contains(PosixFilePermission.OTHERS_READ)) { + System.err.println("Warning: " + passFile + " is world-readable; chmod 600 it"); + } + } catch (Exception e) { + // Non-POSIX filesystem, or permissions unreadable — not worth failing over. + } + } }