From a66488d1f2c96bfbf20d923bcf960a5f1ae77bf5 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 3 Jun 2026 23:21:05 -0400 Subject: [PATCH 1/3] Convert all timestamps to local time for display and scheduling Scheduler now evaluates cron expressions against local time instead of UTC, so schedules fire at the user's local clock time. All timestamp displays in templates use a new `localtime` Jinja filter that converts UTC strings from SQLite to the server's local timezone. Updated CLAUDE.md to reflect the systemd service setup. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- CLAUDE.md | 13 +++++++++++-- pipekit/scheduler.py | 14 +++++++------- pipekit/web/app.py | 17 +++++++++++++++-- pipekit/web/templates/_group_live.html | 2 +- pipekit/web/templates/_group_run_live.html | 2 +- pipekit/web/templates/_module_live.html | 2 +- pipekit/web/templates/group_detail.html | 4 ++-- pipekit/web/templates/group_form.html | 21 ++++++++++++++++++++- pipekit/web/templates/group_run_detail.html | 8 ++++---- pipekit/web/templates/groups.html | 2 +- pipekit/web/templates/modules_index.html | 2 +- pipekit/web/templates/run_detail.html | 6 +++--- pipekit/web/templates/runs.html | 2 +- 13 files changed, 68 insertions(+), 27 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 24797fc..6f52788 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,13 +4,22 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Running the Server +Pipekit runs as a systemd service (`pipekit.service`). Common commands: + +```bash +sudo systemctl status pipekit # check status +sudo systemctl restart pipekit # restart after changes +sudo systemctl stop pipekit # stop +sudo journalctl -u pipekit -f # follow logs +``` + +For dev/testing outside the service — never nohup or background it: + ```bash pipekit serve # default host/port from config.yaml pipekit serve --host 0.0.0.0 --port 8080 --reload # dev mode with auto-reload ``` -The user runs the server themselves in their own terminal — do not nohup or background it. - ## Other CLI Commands ```bash diff --git a/pipekit/scheduler.py b/pipekit/scheduler.py index 64b9727..a29f941 100644 --- a/pipekit/scheduler.py +++ b/pipekit/scheduler.py @@ -17,8 +17,8 @@ from datetime import datetime, timedelta, timezone log = logging.getLogger(__name__) -def _utcnow() -> datetime: - return datetime.now(timezone.utc) +def _now() -> datetime: + return datetime.now() def _check_and_fire() -> None: @@ -29,7 +29,7 @@ def _check_and_fire() -> None: log.error("scheduler: failed to load schedules: %s", e) return - now = _utcnow() + now = _now() for sched in schedules: try: _maybe_fire(sched, now) @@ -43,15 +43,15 @@ def _maybe_fire(sched: dict, now: datetime) -> None: last_str = sched["last_fired_at"] if last_str: - last_dt = datetime.fromisoformat(last_str).replace(tzinfo=timezone.utc) + # SQLite stores UTC; convert to local for cron evaluation. + last_dt = datetime.fromisoformat(last_str).replace( + tzinfo=timezone.utc).astimezone().replace(tzinfo=None) else: - # Never fired — look back 2 minutes so we pick up the current tick - # without replaying indefinitely old missed ticks. last_dt = now - timedelta(seconds=120) try: cron = croniter(sched["cron_expr"], last_dt) - next_dt = cron.get_next(datetime).replace(tzinfo=timezone.utc) + next_dt = cron.get_next(datetime) except CroniterBadCronError: log.warning("scheduler: invalid cron_expr %r for schedule id=%s — skipping", sched["cron_expr"], sched["id"]) diff --git a/pipekit/web/app.py b/pipekit/web/app.py index 19f4962..2132c65 100644 --- a/pipekit/web/app.py +++ b/pipekit/web/app.py @@ -38,7 +38,20 @@ def _fmt_duration(seconds) -> str: return f"{s // 60}m {s % 60:02d}s" +def _fmt_localtime(utc_str) -> str: + """Convert a UTC datetime string from SQLite to local time for display.""" + if not utc_str: + return "—" + from datetime import datetime, timezone + try: + dt = datetime.fromisoformat(str(utc_str)).replace(tzinfo=timezone.utc) + return dt.astimezone().strftime("%Y-%m-%d %H:%M %Z") + except (ValueError, TypeError): + return str(utc_str) + + _templates.env.filters["duration"] = _fmt_duration +_templates.env.filters["localtime"] = _fmt_localtime def mount_web(app: FastAPI) -> None: @@ -656,13 +669,13 @@ def _schedules_with_next(schedules: list[dict]) -> list[dict]: """Attach a 'next_fire_at' string to each schedule dict.""" from croniter import croniter, CroniterBadCronError from datetime import datetime, timezone - now = datetime.now(timezone.utc) + now = datetime.now() result = [] for s in schedules: s = dict(s) try: cron = croniter(s["cron_expr"], now) - s["next_fire_at"] = cron.get_next(datetime).strftime("%Y-%m-%d %H:%M UTC") + s["next_fire_at"] = cron.get_next(datetime).strftime("%Y-%m-%d %H:%M %Z") except CroniterBadCronError: s["next_fire_at"] = "invalid expression" result.append(s) diff --git a/pipekit/web/templates/_group_live.html b/pipekit/web/templates/_group_live.html index c95aab5..3216c49 100644 --- a/pipekit/web/templates/_group_live.html +++ b/pipekit/web/templates/_group_live.html @@ -28,7 +28,7 @@ {% for r in recent_runs %} #{{ r.id }} - {{ r.started_at or "—" }} + {{ r.started_at | localtime }} {{ r.duration_s | duration }} {{ r.triggered_by or "—" }} {{ r.status }} diff --git a/pipekit/web/templates/_group_run_live.html b/pipekit/web/templates/_group_run_live.html index 42768b9..3eb1105 100644 --- a/pipekit/web/templates/_group_run_live.html +++ b/pipekit/web/templates/_group_run_live.html @@ -31,7 +31,7 @@ #{{ r.id }} {{ r.module_name }} - {{ r.started_at or "—" }} + {{ r.started_at | localtime }} {{ r.duration_s | duration }} {{ r.status }} {{ r.row_count if r.row_count is not none else "—" }} diff --git a/pipekit/web/templates/_module_live.html b/pipekit/web/templates/_module_live.html index 47bd24e..821ce36 100644 --- a/pipekit/web/templates/_module_live.html +++ b/pipekit/web/templates/_module_live.html @@ -21,7 +21,7 @@ {% for r in recent_runs %} #{{ r.id }} - {{ r.started_at or '—' }} + {{ r.started_at | localtime }} {{ r.duration_s | duration }} {{ r.status }} {{ r.row_count if r.row_count is not none else "—" }} diff --git a/pipekit/web/templates/group_detail.html b/pipekit/web/templates/group_detail.html index f91482b..777c0bf 100644 --- a/pipekit/web/templates/group_detail.html +++ b/pipekit/web/templates/group_detail.html @@ -67,7 +67,7 @@ cron expression - next fire (UTC) + next fire last fired enabled @@ -77,7 +77,7 @@ {{ s.cron_expr }} {% if s.enabled %}{{ s.next_fire_at }}{% else %}—{% endif %} - {{ s.last_fired_at or "never" }} + {{ s.last_fired_at | localtime }} {% if s.enabled %} yes diff --git a/pipekit/web/templates/group_form.html b/pipekit/web/templates/group_form.html index 123f8ed..3cd1b87 100644 --- a/pipekit/web/templates/group_form.html +++ b/pipekit/web/templates/group_form.html @@ -66,7 +66,7 @@
Schedules - cron expressions in UTC — e.g. 0 4 * * * for daily at 04:00 + cron expressions in local time — e.g. 0 4 * * * for daily at 04:00
@@ -106,6 +106,25 @@ {% if not schedules %}
No schedules — group runs manually only.
{% endif %} +
+ cron expression reference + + + + + + + + + + + + +
expressionmeaning
0 * * * *every hour (on the hour)
*/15 * * * *every 15 minutes
0 4 * * *daily at 04:00
0 4 * * 1-5weekdays at 04:00
0 4 * * 6,0weekends at 04:00
0 0 * * 1every Monday at midnight
0 6,18 * * *twice daily at 06:00 and 18:00
0 0 1 * *first day of each month
+
+ Format: minute hour day-of-month month day-of-week  ·  all times local +
+
diff --git a/pipekit/web/templates/group_run_detail.html b/pipekit/web/templates/group_run_detail.html index 1abe76d..a758abd 100644 --- a/pipekit/web/templates/group_run_detail.html +++ b/pipekit/web/templates/group_run_detail.html @@ -8,7 +8,7 @@ Group run #{{ group_run.id }} {{ group_run.group_name }} · - started {{ group_run.started_at }} + started {{ group_run.started_at | localtime }} {{ group_run.status }} @@ -16,8 +16,8 @@
-
started
{{ group_run.started_at }}
-
finished
{{ group_run.finished_at or "—" }}
+
started
{{ group_run.started_at | localtime }}
+
finished
{{ group_run.finished_at | localtime }}
duration
{{ group_run.duration_s | duration }}
triggered by
{{ group_run.triggered_by or "—" }}
@@ -50,7 +50,7 @@ #{{ r.id }} {{ r.module_name }} - {{ r.started_at or "—" }} + {{ r.started_at | localtime }} {{ r.duration_s | duration }} {{ r.status }} {{ r.row_count if r.row_count is not none else "—" }} diff --git a/pipekit/web/templates/groups.html b/pipekit/web/templates/groups.html index 995bbdd..007d4a6 100644 --- a/pipekit/web/templates/groups.html +++ b/pipekit/web/templates/groups.html @@ -28,7 +28,7 @@ {{ g.name }} {{ g.member_count }} - {{ g.last_run_at or "—" }} + {{ g.last_run_at | localtime }} {% if g.last_status %} {{ g.last_status }} diff --git a/pipekit/web/templates/modules_index.html b/pipekit/web/templates/modules_index.html index 9d03972..3435fa6 100644 --- a/pipekit/web/templates/modules_index.html +++ b/pipekit/web/templates/modules_index.html @@ -39,7 +39,7 @@ {{ g.group_name }} {% endfor %} - {{ m.last_run_at or "—" }} + {{ m.last_run_at | localtime }} {% with module=m %}{% include "_module_status_pill.html" %}{% endwith %} {{ m.last_row_count if m.last_row_count is not none else "—" }} diff --git a/pipekit/web/templates/run_detail.html b/pipekit/web/templates/run_detail.html index 7c901d0..5c64b1f 100644 --- a/pipekit/web/templates/run_detail.html +++ b/pipekit/web/templates/run_detail.html @@ -8,14 +8,14 @@ Run #{{ run.id }} {{ run.module_name }} · - started {{ run.started_at }} + started {{ run.started_at | localtime }} {{ run.status }}
-
started
{{ run.started_at }}
-
finished
{{ run.finished_at or '—' }}
+
started
{{ run.started_at | localtime }}
+
finished
{{ run.finished_at | localtime }}
rows
{{ run.row_count if run.row_count is not none else '—' }}
watermarks
{{ run.watermark_values_json or '—' }}
{% if run.error %}
error
{{ run.error }}
{% endif %} diff --git a/pipekit/web/templates/runs.html b/pipekit/web/templates/runs.html index 102e840..c3ad514 100644 --- a/pipekit/web/templates/runs.html +++ b/pipekit/web/templates/runs.html @@ -33,7 +33,7 @@ #{{ r.id }} {{ r.module_name }} - {{ r.started_at or '—' }} + {{ r.started_at | localtime }} {{ r.duration_s | duration }} {{ r.status }} {{ r.row_count if r.row_count is not none else "—" }} From ed3653f4101ba4edb0ec799ab3426e5e6e60f471 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 3 Jun 2026 23:34:27 -0400 Subject: [PATCH 2/3] deploy.sh: detect JAVA_HOME and inject into systemd unit at deploy time The pipekit system user has no PATH to java. deploy.sh now detects JAVA_HOME by searching common locations and injects Environment= lines into the installed unit file, making deploys portable across machines. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- deploy.sh | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index d94df9b..2e1005a 100755 --- a/deploy.sh +++ b/deploy.sh @@ -182,8 +182,31 @@ step "Systemd unit" if [ ! -f "$UNIT_SRC" ]; then echo " WARNING: $UNIT_SRC not found — skipping" else + # Detect JAVA_HOME — check PATH first, then search each location individually + JAVA_BIN="$(command -v java 2>/dev/null || true)" + if [ -z "$JAVA_BIN" ]; then + for _dir in /opt /usr/lib/jvm /usr/local/lib/jvm; do + [ -d "$_dir" ] || continue + JAVA_BIN="$(find "$_dir" -maxdepth 3 -name java -type f 2>/dev/null | head -1 || true)" + [ -n "$JAVA_BIN" ] && break + done + fi + if [ -z "$JAVA_BIN" ]; then + echo " WARNING: java not found — JAVA_HOME will not be set in unit" + JAVA_HOME_DETECTED="" + else + JAVA_HOME_DETECTED="$(dirname "$(dirname "$(readlink -f "$JAVA_BIN")")")" + echo " Detected JAVA_HOME: $JAVA_HOME_DETECTED" + fi + echo " Installing $UNIT_DST" - cp "$UNIT_SRC" "$UNIT_DST" + if [ -n "$JAVA_HOME_DETECTED" ]; then + # Inject Environment lines after WorkingDirectory= using printf to handle newlines portably + ENV_BLOCK="$(printf 'Environment=JAVA_HOME=%s\nEnvironment=PATH=%s/bin:/usr/local/bin:/usr/bin:/bin' "$JAVA_HOME_DETECTED" "$JAVA_HOME_DETECTED")" + awk -v env="$ENV_BLOCK" '/^WorkingDirectory=/{print; print env; next}1' "$UNIT_SRC" > "$UNIT_DST" + else + cp "$UNIT_SRC" "$UNIT_DST" + fi echo " Running systemctl daemon-reload" systemctl daemon-reload echo " Enabling $SERVICE_NAME service" From c8b507cdc3cbb0e80caee1cc8f2edd839493f3e3 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 3 Jun 2026 23:48:45 -0400 Subject: [PATCH 3/3] repo: add duration_s to get_group_run query The group run detail page was crashing because get_group_run returned no duration_s field, unlike the list queries. Fixes 500 on /group-runs/{id}. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- pipekit/repo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipekit/repo.py b/pipekit/repo.py index b2ef192..da54df7 100644 --- a/pipekit/repo.py +++ b/pipekit/repo.py @@ -662,7 +662,10 @@ def create_group_run(group_id: int, *, triggered_by: str | None = None) -> int: def get_group_run(group_run_id: int) -> dict | None: with db.connect() as c: return _row(c.execute( - "SELECT gr.*, g.name AS group_name " + "SELECT gr.*, g.name AS group_name, " + "CASE WHEN gr.started_at IS NOT NULL AND gr.finished_at IS NOT NULL " + "THEN CAST(ROUND((julianday(gr.finished_at) - julianday(gr.started_at)) * 86400) AS INTEGER) " + "ELSE NULL END AS duration_s " "FROM group_run gr " "JOIN grp g ON gr.group_id=g.id " "WHERE gr.id=?",