web: collapse jrunner progress ticks in live_log to a single line

jrunner prints an in-place progress counter (a bare number) every batch;
read with universal newlines, each tick became its own live_log line, so a
large load stacked thousands of numbers (a 1.27M-row INSERT left 5,073
progress lines). append_run_live_log now overwrites a trailing bare-number
line with the new tick instead of appending, keeping a single current count.
Real lines (headers, "N rows written", timestamps) aren't bare numbers and
are preserved. No jrunner change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-06-18 07:52:34 -04:00
parent ad14d0f5c9
commit e9ff37a3a0

View File

@ -473,6 +473,24 @@ def log_run_output(run_id: int, *, jrunner_stdout: str | None = None,
def append_run_live_log(run_id: int, text: str) -> None: def append_run_live_log(run_id: int, text: str) -> None:
# jrunner emits an in-place progress counter (a bare number) every batch.
# stdout is read with universal newlines, so each tick arrives as its own
# line — appending them all stacks thousands of numbers. Collapse them:
# a bare-number tick overwrites a trailing bare-number line instead of
# appending, so the log keeps a single, current count. Real log lines
# (headers, "N rows written", timestamps) aren't bare numbers — untouched.
if text.strip().isdigit():
with db.connect() as c:
row = c.execute(
"SELECT live_log FROM run_log WHERE id=?", (run_id,)).fetchone()
cur = (row[0] if row and row[0] else "")
body = cur[:-1] if cur.endswith("\n") else cur
lines = body.split("\n") if body else []
if lines and lines[-1].strip().isdigit():
lines[-1] = text
c.execute("UPDATE run_log SET live_log=? WHERE id=?",
("\n".join(lines) + "\n", run_id))
return
with db.connect() as c: with db.connect() as c:
c.execute( c.execute(
"UPDATE run_log SET live_log = COALESCE(live_log, '') || ? WHERE id=?", "UPDATE run_log SET live_log = COALESCE(live_log, '') || ? WHERE id=?",