pt/pipekit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: module descriptions, modules-list redesign, abandoned-run sweep

Browse Source 
Modules:
- add `description` column (schema + migration); create/update/wizard/edit
  forms and module detail all carry it

Modules list page:
- columns: name, description, groups, last run, rows, duration, status
  (dropped strategy/dest and the run/dry-run buttons)
- every column click-to-sort; description truncates single-line with ellipsis
- compact grid; restore main max-width (now 1400px)
- status pill partial is now a <span> so it sits in its own cell

Startup reliability:
- reconcile_abandoned_runs(): on boot, mark run_log/group_run rows stuck in
  `running` (left by a hard kill mid-run) as `error`, complementing the
  existing stale module-lock sweep

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Paul Trowbridge 2026-07-22 15:06:42 -04:00
parent dd9f89f4b5
commit 3b3fe6f865
11 changed files with 142 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pipekit/api/app.py
View File

@ -124,6 +124,18 @@ async def _lifespan(app: FastAPI):
_log.warning("cleared %d stale module lock(s) at startup", n)
except Exception: # noqa: BLE001
_log.exception("stale-lock cleanup failed")
# Finalise run_log / group_run rows abandoned by a process that died
# mid-run — in a fresh process nothing legitimate is running yet, so any
# `running` row is a leftover that would otherwise poll forever.
try:
runs, group_runs = repo.reconcile_abandoned_runs(
"run abandoned — pipekit process exited before completion "
"(reconciled at startup)")
if runs or group_runs:
_log.warning("reconciled %d abandoned run(s) and %d group run(s) "
"at startup", runs, group_runs)
except Exception: # noqa: BLE001
_log.exception("abandoned-run reconciliation failed")
from ..scheduler import start_scheduler
start_scheduler()
yield

2
pipekit/db.py
View File

@ -37,6 +37,8 @@ def _apply_migrations(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE module ADD COLUMN columns_json TEXT")
if "dest_description" not in cols:
conn.execute("ALTER TABLE module ADD COLUMN dest_description TEXT")
if "description" not in cols:
conn.execute("ALTER TABLE module ADD COLUMN description TEXT")
rl_cols = {r[1] for r in conn.execute("PRAGMA table_info(run_log)")}
if "live_log" not in rl_cols:

36
pipekit/repo.py
View File

@ -164,18 +164,19 @@ def create_module(*, name: str, source_connection_id: int,
merge_strategy: str = "full", merge_key: str | None = None,
staging_table: str | None = None,
columns: list[dict] | None = None,
dest_description: str | None = None) -> dict:
dest_description: str | None = None,
description: str | None = None) -> dict:
staging = staging_table or f"pipekit_staging.{name}"
cols_json = json.dumps(columns) if columns else None
with db.connect() as c:
cur = c.execute(
"INSERT INTO module (name, source_connection_id, dest_connection_id, "
"dest_table, staging_table, source_query, merge_strategy, merge_key, "
"columns_json, dest_description) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"columns_json, dest_description, description) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(name, source_connection_id, dest_connection_id, dest_table,
staging, source_query, merge_strategy, merge_key, cols_json,
dest_description),
dest_description, description),
)
return _row(c.execute(
"SELECT * FROM module WHERE id=?", (cur.lastrowid,)).fetchone())
@ -207,6 +208,7 @@ def update_module(module_id: int, *, name: str | None = None,
merge_strategy: str | None = None,
merge_key: str | None = None,
dest_description: str | None = None,
description: str | None = None,
enabled: int | None = None) -> dict | None:
fields: list[str] = []
values: list = []
@ -219,6 +221,7 @@ def update_module(module_id: int, *, name: str | None = None,
("merge_strategy", merge_strategy),
("merge_key", merge_key),
("dest_description", dest_description),
("description", description),
("enabled", enabled)):
if val is not None:
fields.append(f"{col}=?")
@ -426,6 +429,31 @@ def clear_stale_locks(max_age_hours: int = 24, live_pids: set[int] | None = None
return cleared
def reconcile_abandoned_runs(reason: str) -> tuple[int, int]:
"""Mark run_log / group_run rows still ``running`` as ``error``.
Meant to be called once at process startup, before any run can begin: a
hard kill (or ``systemctl stop`` mid-run) skips the engine's finally, so
the run_log row (and any owning group_run) stays ``running`` forever — the
live-log poller would spin and the runs list would show a phantom run. In a
fresh process no run has started yet, so every ``running`` row is a leftover
and is safe to finalise. Returns (runs_reconciled, group_runs_reconciled).
"""
with db.connect() as c:
runs = c.execute(
"UPDATE run_log SET finished_at=datetime('now'), status='error', "
"error=CASE WHEN error IS NULL OR error='' THEN ? "
"ELSE error || char(10) || ? END "
"WHERE status='running'",
(reason, reason),
).rowcount
group_runs = c.execute(
"UPDATE group_run SET finished_at=datetime('now'), status='error' "
"WHERE status='running'",
).rowcount
return runs, group_runs
# ---------------------------------------------------------------------------
# Run log
# ---------------------------------------------------------------------------

1
pipekit/schema.sql
View File

@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS connection (
CREATE TABLE IF NOT EXISTS module (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
description TEXT, -- free-text note about the module, shown in the UI
source_connection_id INTEGER NOT NULL REFERENCES connection(id),
dest_connection_id INTEGER NOT NULL REFERENCES connection(id),
dest_table TEXT NOT NULL,

5
pipekit/web/app.py
View File

@ -147,10 +147,12 @@ def home(request: Request):
m["last_run_at"] = last["started_at"]
m["last_status"] = last["status"]
m["last_row_count"] = last["row_count"]
m["last_duration_s"] = last["duration_s"]
else:
m["last_run_at"] = None
m["last_status"] = None
m["last_row_count"] = None
m["last_duration_s"] = None
m["groups"] = groups_by_module.get(m["id"], [])
# group by source connection
@ -287,6 +289,7 @@ async def module_update(request: Request, module_id: int):
merge_strategy=merge_strategy,
merge_key=(form.get("merge_key") or "").strip() or None,
dest_description=new_description,
description=(form.get("description") or "").strip(),
enabled=1 if form.get("enabled") == "1" else 0,
)
_save_inline_watermarks(form, module_id)
@ -647,6 +650,7 @@ async def wizard_create(request: Request):
merge_key = (form.get("merge_key") or "").strip() or None
staging_table = (form.get("staging_table") or "").strip() or None
dest_description = (form.get("dest_description") or "").strip() or None
description = (form.get("description") or "").strip() or None
picked = form.getlist("col")
if repo.get_module_by_name(module_name) is not None:
@ -809,6 +813,7 @@ async def wizard_create(request: Request):
staging_table=staging_table,
columns=chosen,
dest_description=dest_description,
description=description,
)
_save_inline_watermarks(form, module["id"])
return RedirectResponse(url=f"/modules/{module['id']}", status_code=303)

1
pipekit/web/static/style.css
View File

@ -66,6 +66,7 @@ header.topbar nav a:hover {
header.topbar .right { margin-left: auto; color: var(--text-muted); font-size: 12px; }
main {
max-width: 1400px;
margin: 1rem auto;
padding: 0 1.2rem;
}

9
pipekit/web/templates/_module_status_pill.html
View File

@ -1,6 +1,7 @@
{# Partial: status cell for one module row on the index page.
Swaps itself (outerHTML) every 3s while running; stops when idle. #}
<td id="module-status-{{ module.id }}"
{# Partial: status indicator for one module on the index page.
Renders a <span> (lives inside the compact "last run" cell). Swaps itself
(outerHTML) every 3s while running; stops when idle. #}
<span id="module-status-{{ module.id }}"
{% if module.running or force_poll %}
hx-get="/modules/{{ module.id }}/status-pill"
hx-trigger="every 3s"
@ -15,4 +16,4 @@
{% else %}
<span class="pill">never ran</span>
{% endif %}
</td>
</span>

3
pipekit/web/templates/module_detail.html
View File

@ -36,6 +36,9 @@
</header>
<div class="body">
<dl class="keyval">
{% if module.description %}
<dt>description</dt> <dd>{{ module.description }}</dd>
{% endif %}
<dt>source</dt> <dd>{{ source_conn.name }} <span style="opacity:.6" class="mono">({{ source_conn.jdbc_url }})</span></dd>
<dt>destination</dt> <dd>{{ dest_conn.name }} <span style="opacity:.6" class="mono">({{ dest_conn.jdbc_url }})</span></dd>
<dt>dest table</dt> <dd class="mono">{{ module.dest_table }}</dd>

6
pipekit/web/templates/module_form.html
View File

@ -26,6 +26,12 @@
<span class="help">must be unique; also used as the default staging table suffix</span>
</label>
<label class="field">
<span>description</span>
<textarea name="description" rows="2">{{ module.description or '' }}</textarea>
<span class="help">free-text note about what this module syncs; shown on the modules list</span>
</label>
<div class="two-col" style="gap:1rem">
<label class="field">
<span>source connection</span>

100
pipekit/web/templates/modules_index.html
View File

@ -15,48 +15,32 @@
{% if grouped %}
{% for conn_name, driver_label, modules in grouped %}
<div class="group-head">{{ conn_name }} <span style="opacity:.7">({{ driver_label }})</span></div>
<table class="grid">
<table class="grid sortable compact">
<thead>
<tr>
<th style="width:12em">name</th>
<th style="width:7em">strategy</th>
<th style="width:14em">dest</th>
<th style="width:13em">groups</th>
<th style="width:11em;white-space:nowrap">last run</th>
<th style="width:8em">status</th>
<th style="width:6em">rows</th>
<th style="width:12em"></th>
<th style="width:12em" onclick="sortTable(this)">name</th>
<th onclick="sortTable(this)">description</th>
<th style="width:11em" onclick="sortTable(this)">groups</th>
<th style="width:11em;white-space:nowrap" onclick="sortTable(this)">last run</th>
<th style="width:6em;text-align:right" data-type="num" onclick="sortTable(this)">rows</th>
<th style="width:6em;text-align:right" data-type="num" onclick="sortTable(this)">duration</th>
<th style="width:7em" onclick="sortTable(this)">status</th>
</tr>
</thead>
<tbody>
{% for m in modules %}
<tr>
<td><a href="/modules/{{ m.id }}"><strong>{{ m.name }}</strong></a></td>
<td class="mono">{{ m.merge_strategy }}</td>
<td class="mono">{{ m.dest_table }}</td>
<td>
<td class="namecell"><a href="/modules/{{ m.id }}"><strong>{{ m.name }}</strong></a></td>
<td class="desccell">{% if m.description %}<span class="help modesc" title="{{ m.description }}">{{ m.description }}</span>{% endif %}</td>
<td style="white-space:nowrap" data-sort="{{ m.groups | map(attribute='group_name') | join(' ') }}">
{% for g in m.groups %}
<a href="/groups/{{ g.group_id }}" class="tag">{{ g.group_name }}</a>
{% endfor %}
</td>
<td class="mono" style="white-space:nowrap">{{ m.last_run_at | localtime }}</td>
{% with module=m %}{% include "_module_status_pill.html" %}{% endwith %}
<td class="mono">{{ m.last_row_count if m.last_row_count is not none else "—" }}</td>
<td style="text-align:right">
<form class="inline"
hx-post="/modules/{{ m.id }}/run"
hx-target="#module-status-{{ m.id }}"
hx-swap="outerHTML">
<button type="submit">Run</button>
</form>
<form class="inline"
hx-post="/modules/{{ m.id }}/run"
hx-target="#module-status-{{ m.id }}"
hx-swap="outerHTML">
<input type="hidden" name="dry_run" value="1">
<button type="submit" class="ghost">Dry run</button>
</form>
</td>
<td class="mono" style="white-space:nowrap" data-sort="{{ m.last_run_at or '' }}">{{ m.last_run_at | localtime }}</td>
<td class="mono" style="text-align:right" data-sort="{{ m.last_row_count if m.last_row_count is not none else -1 }}">{{ m.last_row_count if m.last_row_count is not none else "—" }}</td>
<td class="mono" style="text-align:right" data-sort="{{ m.last_duration_s if m.last_duration_s is not none else -1 }}">{{ m.last_duration_s | duration }}</td>
<td>{% with module=m %}{% include "_module_status_pill.html" %}{% endwith %}</td>
</tr>
{% endfor %}
</tbody>
@ -70,4 +54,58 @@
{% endif %}
</div>
</div>
<style>
table.sortable th[onclick] { cursor: pointer; user-select: none; }
table.sortable th[onclick]:hover { text-decoration: underline; }
table.sortable th[data-dir]::after { content: " \25B4"; opacity: .6; }
table.sortable th[data-dir="desc"]::after { content: " \25BE"; }
/* compact grid */
table.grid.compact th, table.grid.compact td { padding: 0.3rem 0.6rem; }
td.namecell { white-space: nowrap; }
/* description column: single line, truncate with ellipsis (full text on hover) */
td.desccell .modesc {
display: inline-block;
max-width: 42vw;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
vertical-align: bottom;
}
</style>
<script>
function sortTable(th) {
const table = th.closest('table');
const tbody = table.tBodies[0];
if (!tbody) return;
const idx = Array.prototype.indexOf.call(th.parentNode.children, th);
const type = th.dataset.type || 'text';
const asc = th.dataset.dir !== 'asc'; // toggle; default first click ascending
// clear direction indicator on sibling headers
Array.prototype.forEach.call(th.parentNode.children, h => {
if (h !== th) delete h.dataset.dir;
});
th.dataset.dir = asc ? 'asc' : 'desc';
const key = (row) => {
const cell = row.cells[idx];
if (!cell) return type === 'num' ? -Infinity : '';
const raw = cell.dataset.sort !== undefined ? cell.dataset.sort : cell.textContent.trim();
if (type === 'num') { const n = parseFloat(raw); return isNaN(n) ? -Infinity : n; }
return raw.toLowerCase();
};
Array.from(tbody.rows)
.sort((a, b) => {
const x = key(a), y = key(b);
if (x < y) return asc ? -1 : 1;
if (x > y) return asc ? 1 : -1;
return 0;
})
.forEach(r => tbody.appendChild(r));
}
</script>
{% endblock %}

6
pipekit/web/templates/wizard_step3.html
View File

@ -153,6 +153,12 @@
value="{{ default_module_name }}">
<span class="help">used in the URL and as the default staging table name</span>
</label>
<label class="field">
<span>description</span>
<textarea name="description" rows="2"
style="width:100%">{{ module_description or '' }}</textarea>
<span class="help">optional free-text note; shown on the modules list</span>
</label>
</div>
</div>