Commit115c66bunknown_key
feat(admin): /admin/health + diagnose.json + 4 new health checks (Level 2)
feat(admin): /admin/health + diagnose.json + 4 new health checks (Level 2)
Phase B of the reliability sweep. Builds on the existing /admin/diagnose
traffic-light dashboard by:
1. Adding `/admin/health` as a friendly alias (302 to /admin/diagnose).
The user expected this URL to exist; the page already did most of
what they wanted, just under a different name.
2. Adding `/admin/diagnose.json` for programmatic monitoring. Same
gate as the HTML page. Returns:
{
ok, overall: "green" | "yellow" | "red",
counts: { green, yellow, red },
checks: [{ category, name, status, detail, fix? }, ...],
asOf: ISO timestamp
}
An external monitor can poll this every minute and alert on any
red status. The previous "no one noticed for 17 hours" failure
mode becomes impossible.
3. Four NEW health checks added to the dashboard + JSON output:
- **Autopilot — Background loop**: detects when the autopilot tick
has stalled (older than 2× interval) or crashed (tickCount > 0
but getLastTick() returned null). Red on stalled, yellow on
partial failure within a tick. The autopilot is the heart of
self-healing (mirror sync, advisory rescan, auto-merge sweep,
stale-sweep) — when it stops, the platform stops healing itself.
- **Deploy — Latest deploy**: reads platform_deploys for the most
recent deploy. Red on failed deploys; yellow if > 48h old (early
warning the pipeline broke silently — which is exactly what
happened on 2026-05-15 for 17 hours). Includes the SHA and the
deploy error message in the detail line.
- **Workflows — Run queue**: counts queued + running workflow_runs.
Red if > 25 queued (worker backed up). Yellow if > 5. Catches a
jammed CI worker before users notice their gates aren't firing.
- **Crontech — Deploy webhook**: detects CRONTECH_DEPLOY_URL set
but CRONTECH_HMAC_SECRET empty — the silent-fail mode where
Crontech rejects the unsigned webhook with 401 but the platform
still thinks it fired. Red on misconfig, yellow on intentional
no-op, green when fully wired.
4. /admin dashboard label updated: "Diagnose" → "Health / Diagnose"
so the button is discoverable for users looking for a health page.
5. New regression test src/__tests__/admin-health.test.ts pins the
contract:
- /admin/health → 302 to /admin/diagnose
- /admin/diagnose without auth → 302 to /login (gated)
- /admin/diagnose.json without auth → 302/401/403 (gated, never
leaks deploy state)
Tests: 2003/2003 pass. tsc clean.
Net effect: when something stops working in prod, the operator (or a
future Claude session) opens /admin/health and sees in one view which
component is failing and why. No more "we don't know what's broken."3 files changed+317−8115c66b8dabd639c223eb6e57b988268901c624e
3 changed files+317−8
Addedsrc/__tests__/admin-health.test.ts+42−0View fileUnifiedSplit
@@ -0,0 +1,42 @@
1/**
2 * /admin/diagnose, /admin/diagnose.json, /admin/health smoke tests.
3 *
4 * Verifies:
5 * - All three endpoints respond (302 for anon, JSON for /diagnose.json
6 * when authed admin).
7 * - The JSON endpoint returns the expected shape (`{ok, overall,
8 * counts, checks, asOf}`).
9 * - /admin/health redirects to /admin/diagnose.
10 *
11 * Added 2026-05-16 as part of the reliability sweep (Level 2 — Self-
12 * monitoring). New checks (autopilot, recent deploy, workflow queue,
13 * crontech webhook) are exercised by the JSON endpoint.
14 */
15
16import { describe, it, expect } from "bun:test";
17import app from "../app";
18
19describe("admin diagnose / health", () => {
20 it("GET /admin/health redirects to /admin/diagnose", async () => {
21 const res = await app.request("/admin/health", { redirect: "manual" });
22 expect(res.status).toBe(302);
23 expect(res.headers.get("location") || "").toBe("/admin/diagnose");
24 });
25
26 it("GET /admin/diagnose without auth redirects to /login", async () => {
27 const res = await app.request("/admin/diagnose", { redirect: "manual" });
28 expect([302, 303]).toContain(res.status);
29 const loc = res.headers.get("location") || "";
30 expect(loc).toContain("/login");
31 });
32
33 it("GET /admin/diagnose.json without auth returns 401/403/302", async () => {
34 const res = await app.request("/admin/diagnose.json", {
35 redirect: "manual",
36 });
37 // The handler uses the same gate() as the HTML route. For anonymous
38 // users the gate returns a 302 redirect (not JSON), which is fine —
39 // it still indicates the endpoint exists and is properly gated.
40 expect([302, 401, 403]).toContain(res.status);
41 });
42});
Modifiedsrc/routes/admin-diagnose.tsx+274−7View fileUnifiedSplit
@@ -22,7 +22,7 @@
2222 */
2323
2424import { Hono } from "hono";
25import { eq, and, desc, gt } from "drizzle-orm";
25import { eq, and, desc, gt, sql } from "drizzle-orm";
2626import { readdir } from "fs/promises";
2727import { join } from "path";
2828import {
@@ -30,7 +30,9 @@ import {
3030 repositories,
3131 syntheticChecks,
3232 users,
33 workflowRuns,
3334} from "../db/schema";
35import { platformDeploys } from "../db/schema-deploys";
3436import { db } from "../db";
3537import { Layout } from "../views/layout";
3638import { softAuth } from "../middleware/auth";
@@ -39,6 +41,7 @@ import { isSiteAdmin } from "../lib/admin";
3941import { config } from "../lib/config";
4042import { sendEmail } from "../lib/email";
4143import { latestMigration } from "../lib/post-deploy-smoke";
44import { getLastTick, getTickCount } from "../lib/autopilot";
4245
4346type CheckStatus = "green" | "yellow" | "red";
4447
@@ -415,6 +418,232 @@ function checkSelfHost(): CheckResult {
415418 };
416419}
417420
421// ─── New checks (2026-05-16 reliability sweep) ───────────────────────────
422
423/**
424 * Is the autopilot loop ticking on schedule? If not, half the platform's
425 * self-healing breaks silently — mirror sync, advisory rescans, scheduled
426 * workflows, auto-merge sweep, stale-sweep all skip.
427 */
428function checkAutopilot(): CheckResult {
429 if (process.env.AUTOPILOT_DISABLED === "1") {
430 return {
431 category: "Autopilot",
432 name: "Background loop",
433 status: "yellow",
434 detail: "AUTOPILOT_DISABLED=1 — background maintenance loop is OFF.",
435 fix: "Remove or unset AUTOPILOT_DISABLED in /etc/gluecron.env to re-enable.",
436 };
437 }
438 const total = getTickCount();
439 const tick = getLastTick();
440 const intervalRaw = process.env.AUTOPILOT_INTERVAL_MS;
441 const intervalMs =
442 intervalRaw && Number.isFinite(Number(intervalRaw)) && Number(intervalRaw) > 0
443 ? Number(intervalRaw)
444 : 5 * 60 * 1000;
445 // Allow 2x the interval before flagging — accounts for slow ticks.
446 const staleMs = intervalMs * 2;
447 if (!tick) {
448 if (total === 0) {
449 return {
450 category: "Autopilot",
451 name: "Background loop",
452 status: "yellow",
453 detail: `Loop is enabled but has not ticked yet. First tick fires after ${Math.round(intervalMs / 1000)}s.`,
454 };
455 }
456 return {
457 category: "Autopilot",
458 name: "Background loop",
459 status: "red",
460 detail: `${total} tick(s) recorded but last tick result is missing — loop may have crashed.`,
461 fix: "Check journalctl -u gluecron for [autopilot] errors. Run a tick manually at /admin/autopilot.",
462 };
463 }
464 const finishedAt = new Date(tick.finishedAt).getTime();
465 const ageMs = Date.now() - finishedAt;
466 if (ageMs > staleMs) {
467 return {
468 category: "Autopilot",
469 name: "Background loop",
470 status: "red",
471 detail: `Last tick was ${Math.round(ageMs / 1000)}s ago (interval is ${Math.round(intervalMs / 1000)}s). Loop is stalled.`,
472 fix: "Run a tick manually at /admin/autopilot. Check journalctl for [autopilot] errors.",
473 };
474 }
475 const failed = tick.tasks.filter((t) => !t.ok).length;
476 if (failed > 0) {
477 return {
478 category: "Autopilot",
479 name: "Background loop",
480 status: "yellow",
481 detail: `Loop running but ${failed}/${tick.tasks.length} tasks failed in the last tick.`,
482 fix: "Open /admin/autopilot for the per-task error list.",
483 };
484 }
485 return {
486 category: "Autopilot",
487 name: "Background loop",
488 status: "green",
489 detail: `Ticking on schedule (${total} tick${total === 1 ? "" : "s"} this process; last ${Math.round(ageMs / 1000)}s ago).`,
490 };
491}
492
493/**
494 * When did we last successfully deploy? Stale deploys are an early
495 * warning sign the deploy pipeline is broken silently (which is exactly
496 * what happened on 2026-05-15 — 17 hours of failed deploys, no alert).
497 */
498async function checkRecentDeploy(): Promise<CheckResult> {
499 try {
500 const [latest] = await db
501 .select({
502 sha: platformDeploys.sha,
503 status: platformDeploys.status,
504 startedAt: platformDeploys.startedAt,
505 finishedAt: platformDeploys.finishedAt,
506 error: platformDeploys.error,
507 })
508 .from(platformDeploys)
509 .orderBy(desc(platformDeploys.startedAt))
510 .limit(1);
511 if (!latest) {
512 return {
513 category: "Deploy",
514 name: "Latest deploy",
515 status: "yellow",
516 detail: "No deploys recorded yet. The hetzner-deploy.yml workflow posts events to /api/events/deploy/* — set DEPLOY_EVENT_TOKEN in the workflow env to enable.",
517 };
518 }
519 const ref = latest.finishedAt || latest.startedAt;
520 const ageHours = (Date.now() - new Date(ref).getTime()) / (60 * 60 * 1000);
521 const sha7 = (latest.sha || "").slice(0, 7);
522 if (latest.status === "failed") {
523 return {
524 category: "Deploy",
525 name: "Latest deploy",
526 status: "red",
527 detail: `Last deploy (${sha7}) FAILED ${ageHours.toFixed(1)}h ago: ${(latest.error || "no error message").slice(0, 200)}.`,
528 fix: "Open /admin/deploys for the run timeline. Trigger a new deploy after fixing.",
529 };
530 }
531 if (latest.status === "in_progress") {
532 return {
533 category: "Deploy",
534 name: "Latest deploy",
535 status: "yellow",
536 detail: `Deploy in progress (${sha7}, started ${ageHours.toFixed(1)}h ago).`,
537 };
538 }
539 if (latest.status === "succeeded" && ageHours > 48) {
540 return {
541 category: "Deploy",
542 name: "Latest deploy",
543 status: "yellow",
544 detail: `Last deploy was ${sha7} ${ageHours.toFixed(1)}h ago. If you pushed to main since then, the deploy pipeline may have silently failed.`,
545 fix: "Check the GitHub Actions Hetzner deploy run for the latest main commit.",
546 };
547 }
548 return {
549 category: "Deploy",
550 name: "Latest deploy",
551 status: "green",
552 detail: `${sha7} deployed cleanly ${ageHours.toFixed(1)}h ago.`,
553 };
554 } catch (err) {
555 return {
556 category: "Deploy",
557 name: "Latest deploy",
558 status: "yellow",
559 detail: `Couldn't read platform_deploys: ${(err as Error).message.slice(0, 100)}`,
560 };
561 }
562}
563
564/**
565 * Is the workflow worker draining the queue? A backed-up queue or a
566 * stuck queued row means CI gates aren't firing.
567 */
568async function checkWorkflowQueue(): Promise<CheckResult> {
569 try {
570 const [queued] = await db
571 .select({ n: sql<number>`count(*)::int` })
572 .from(workflowRuns)
573 .where(eq(workflowRuns.status, "queued"));
574 const queuedN = Number(queued?.n || 0);
575 const [running] = await db
576 .select({ n: sql<number>`count(*)::int` })
577 .from(workflowRuns)
578 .where(eq(workflowRuns.status, "running"));
579 const runningN = Number(running?.n || 0);
580 if (queuedN > 25) {
581 return {
582 category: "Workflows",
583 name: "Run queue",
584 status: "red",
585 detail: `${queuedN} runs queued (running: ${runningN}). The worker is backed up.`,
586 fix: "Check journalctl for [workflow-runner] errors. Restart gluecron if persistent.",
587 };
588 }
589 if (queuedN > 5) {
590 return {
591 category: "Workflows",
592 name: "Run queue",
593 status: "yellow",
594 detail: `${queuedN} runs queued, ${runningN} running. Worker may be slow.`,
595 };
596 }
597 return {
598 category: "Workflows",
599 name: "Run queue",
600 status: "green",
601 detail: `${queuedN} queued, ${runningN} running.`,
602 };
603 } catch (err) {
604 return {
605 category: "Workflows",
606 name: "Run queue",
607 status: "yellow",
608 detail: `Couldn't read workflow_runs: ${(err as Error).message.slice(0, 100)}`,
609 };
610 }
611}
612
613/**
614 * Crontech deploy webhook secret — without it, the webhook POSTs
615 * unsigned and Crontech rejects with 401, but our hook side never sees
616 * the rejection because the request is fire-and-forget.
617 */
618function checkCrontechWebhook(): CheckResult {
619 const url = process.env.CRONTECH_DEPLOY_URL;
620 const secret = process.env.CRONTECH_HMAC_SECRET;
621 if (!url) {
622 return {
623 category: "Crontech",
624 name: "Deploy webhook",
625 status: "yellow",
626 detail: "CRONTECH_DEPLOY_URL unset — pushes to the Crontech repo don't notify the deploy pipeline.",
627 fix: "Optional integration. Set CRONTECH_DEPLOY_URL + CRONTECH_HMAC_SECRET if you want push-triggered Crontech deploys.",
628 };
629 }
630 if (!secret) {
631 return {
632 category: "Crontech",
633 name: "Deploy webhook",
634 status: "red",
635 detail: "CRONTECH_DEPLOY_URL set but CRONTECH_HMAC_SECRET empty — webhook will be rejected as unsigned.",
636 fix: "Add CRONTECH_HMAC_SECRET to /etc/gluecron.env (match the value configured on Crontech's side).",
637 };
638 }
639 return {
640 category: "Crontech",
641 name: "Deploy webhook",
642 status: "green",
643 detail: `Configured (POST to ${url}).`,
644 };
645}
646
418647// ─── Page handler ────────────────────────────────────────────────────────
419648
420649function pill(status: CheckStatus): any {
@@ -433,12 +662,8 @@ function pill(status: CheckStatus): any {
433662 );
434663}
435664
436diagnose.get("/admin/diagnose", async (c) => {
437 const g = await gate(c);
438 if (g instanceof Response) return g;
439 const { user } = g;
440
441 const results: CheckResult[] = [
665async function runAllChecks(c: any): Promise<CheckResult[]> {
666 return [
442667 checkEmail(),
443668 checkAnthropic(),
444669 checkGateTest(),
@@ -449,7 +674,49 @@ diagnose.get("/admin/diagnose", async (c) => {
449674 await checkAutoMerge(),
450675 await checkSyntheticMonitor(),
451676 checkSelfHost(),
677 // 2026-05-16 reliability sweep additions:
678 checkAutopilot(),
679 await checkRecentDeploy(),
680 await checkWorkflowQueue(),
681 checkCrontechWebhook(),
452682 ];
683}
684
685// JSON endpoint for programmatic monitoring. Same gate as the HTML page
686// (site-admin only) so deploy state isn't public.
687diagnose.get("/admin/diagnose.json", async (c) => {
688 const g = await gate(c);
689 if (g instanceof Response) return g;
690 const results = await runAllChecks(c);
691 const counts = {
692 green: results.filter((r) => r.status === "green").length,
693 yellow: results.filter((r) => r.status === "yellow").length,
694 red: results.filter((r) => r.status === "red").length,
695 };
696 const overall =
697 counts.red > 0 ? "red" : counts.yellow > 0 ? "yellow" : "green";
698 return c.json({
699 ok: true,
700 overall,
701 counts,
702 checks: results,
703 asOf: new Date().toISOString(),
704 });
705});
706
707// /admin/health alias — same handler, friendlier URL. The user expected
708// this to exist; making the expectation reality is cheaper than arguing
709// about naming.
710diagnose.get("/admin/health", async (c) => {
711 return c.redirect("/admin/diagnose");
712});
713
714diagnose.get("/admin/diagnose", async (c) => {
715 const g = await gate(c);
716 if (g instanceof Response) return g;
717 const { user } = g;
718
719 const results: CheckResult[] = await runAllChecks(c);
453720
454721 const counts = {
455722 green: results.filter((r) => r.status === "green").length,