From e9ff37a3a0ba8b115ad47b181b99e020fc1a5b36 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 18 Jun 2026 07:52:34 -0400 Subject: [PATCH] web: collapse jrunner progress ticks in live_log to a single line jrunner prints an in-place progress counter (a bare number) every batch; read with universal newlines, each tick became its own live_log line, so a large load stacked thousands of numbers (a 1.27M-row INSERT left 5,073 progress lines). append_run_live_log now overwrites a trailing bare-number line with the new tick instead of appending, keeping a single current count. Real lines (headers, "N rows written", timestamps) aren't bare numbers and are preserved. No jrunner change. Co-Authored-By: Claude Opus 4.8 --- pipekit/repo.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pipekit/repo.py b/pipekit/repo.py index 5b32d66..c9071b6 100644 --- a/pipekit/repo.py +++ b/pipekit/repo.py @@ -473,6 +473,24 @@ def log_run_output(run_id: int, *, jrunner_stdout: str | None = None, def append_run_live_log(run_id: int, text: str) -> None: + # jrunner emits an in-place progress counter (a bare number) every batch. + # stdout is read with universal newlines, so each tick arrives as its own + # line — appending them all stacks thousands of numbers. Collapse them: + # a bare-number tick overwrites a trailing bare-number line instead of + # appending, so the log keeps a single, current count. Real log lines + # (headers, "N rows written", timestamps) aren't bare numbers — untouched. + if text.strip().isdigit(): + with db.connect() as c: + row = c.execute( + "SELECT live_log FROM run_log WHERE id=?", (run_id,)).fetchone() + cur = (row[0] if row and row[0] else "") + body = cur[:-1] if cur.endswith("\n") else cur + lines = body.split("\n") if body else [] + if lines and lines[-1].strip().isdigit(): + lines[-1] = text + c.execute("UPDATE run_log SET live_log=? WHERE id=?", + ("\n".join(lines) + "\n", run_id)) + return with db.connect() as c: c.execute( "UPDATE run_log SET live_log = COALESCE(live_log, '') || ? WHERE id=?",