Blame · Line-by-line history
admin-diagnose.tsx
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 826eccf | 1 | /** |
| 2 | * /admin/diagnose — comprehensive AI health scan. | |
| 3 | * | |
| 4 | * Single page the site admin opens to see, at a glance, every config knob | |
| 5 | * the platform depends on and whether it is wired up. Each row is one | |
| 6 | * check; status is green / yellow / red with a one-line "what to do". | |
| 7 | * | |
| 8 | * Categories covered: | |
| 9 | * - Email delivery (EMAIL_PROVIDER, RESEND_API_KEY) | |
| 10 | * - AI (ANTHROPIC_API_KEY presence) | |
| 11 | * - GateTest integration (URL + API key) | |
| 12 | * - Service worker SHA (BUILD_SHA pinned vs dev-stable fallback) | |
| 13 | * - Database (DATABASE_URL well-formed, latest migration applied) | |
| 14 | * - Canonical URL (APP_BASE_URL matches request host) | |
| 15 | * - Self-host (SELF_HOST_REPO declared + post-receive hook present) | |
| 16 | * - Auto-merge (branch_protection.enable_auto_merge for main) | |
| 17 | * - Synthetic monitor (any RED checks in last hour) | |
| 18 | * - Email smoke (POST /admin/diagnose/test-email fires a test) | |
| 19 | * | |
| 20 | * Gating: requireAuth + isSiteAdmin via the same gate() pattern as | |
| 21 | * /admin/ops + /admin/status. No data leaks to non-admins. | |
| 22 | */ | |
| 23 | ||
| 24 | import { Hono } from "hono"; | |
| 115c66b | 25 | import { eq, and, desc, gt, sql } from "drizzle-orm"; |
| 826eccf | 26 | import { readdir } from "fs/promises"; |
| 27 | import { join } from "path"; | |
| 28 | import { | |
| 29 | branchProtection, | |
| 30 | repositories, | |
| 31 | syntheticChecks, | |
| 32 | users, | |
| 115c66b | 33 | workflowRuns, |
| 826eccf | 34 | } from "../db/schema"; |
| 115c66b | 35 | import { platformDeploys } from "../db/schema-deploys"; |
| 826eccf | 36 | import { db } from "../db"; |
| 37 | import { Layout } from "../views/layout"; | |
| f4a1547 | 38 | import { AdminShell } from "../views/admin-shell"; |
| 826eccf | 39 | import { softAuth } from "../middleware/auth"; |
| 40 | import type { AuthEnv } from "../middleware/auth"; | |
| 41 | import { isSiteAdmin } from "../lib/admin"; | |
| 42 | import { config } from "../lib/config"; | |
| 43 | import { sendEmail } from "../lib/email"; | |
| 44 | import { latestMigration } from "../lib/post-deploy-smoke"; | |
| 115c66b | 45 | import { getLastTick, getTickCount } from "../lib/autopilot"; |
| 826eccf | 46 | |
| 47 | type CheckStatus = "green" | "yellow" | "red"; | |
| 48 | ||
| 49 | interface CheckResult { | |
| 50 | category: string; | |
| 51 | name: string; | |
| 52 | status: CheckStatus; | |
| 53 | detail: string; | |
| 54 | fix?: string; | |
| 55 | } | |
| 56 | ||
| 57 | const diagnose = new Hono<AuthEnv>(); | |
| 58 | diagnose.use("*", softAuth); | |
| 59 | ||
| 60 | async function gate(c: any): Promise<{ user: any } | Response> { | |
| 61 | const user = c.get("user"); | |
| 62 | if (!user) return c.redirect("/login?next=/admin/diagnose"); | |
| 63 | if (!(await isSiteAdmin(user.id))) { | |
| 64 | return c.html( | |
| 65 | <Layout title="Forbidden" user={user}> | |
| 66 | <div class="empty-state"> | |
| 67 | <h2>403 — Not a site admin</h2> | |
| 68 | <p>You don't have permission to view this page.</p> | |
| 69 | </div> | |
| 70 | </Layout>, | |
| 71 | 403 | |
| 72 | ); | |
| 73 | } | |
| 74 | return { user }; | |
| 75 | } | |
| 76 | ||
| 77 | // ─── Individual checks ─────────────────────────────────────────────────── | |
| 78 | ||
| 79 | function checkEmail(): CheckResult { | |
| 80 | if (config.emailProvider !== "resend") { | |
| 81 | return { | |
| 82 | category: "Email", | |
| 83 | name: "Provider", | |
| 84 | status: "red", | |
| 85 | detail: `EMAIL_PROVIDER=${config.emailProvider} — verification + magic-link emails go to stderr, not inboxes.`, | |
| 86 | fix: "Set EMAIL_PROVIDER=resend in /etc/gluecron.env, then `systemctl restart gluecron`.", | |
| 87 | }; | |
| 88 | } | |
| 89 | if (!config.resendApiKey) { | |
| 90 | return { | |
| 91 | category: "Email", | |
| 92 | name: "Provider", | |
| 93 | status: "red", | |
| 94 | detail: "EMAIL_PROVIDER=resend but RESEND_API_KEY is empty — every send will fail.", | |
| 95 | fix: "Add RESEND_API_KEY=re_xxx to /etc/gluecron.env, then restart.", | |
| 96 | }; | |
| 97 | } | |
| 98 | return { | |
| 99 | category: "Email", | |
| 100 | name: "Provider", | |
| 101 | status: "green", | |
| 102 | detail: `Resend wired (from: ${config.emailFrom}). Use the test button below to confirm.`, | |
| 103 | }; | |
| 104 | } | |
| 105 | ||
| 106 | function checkAnthropic(): CheckResult { | |
| 107 | if (!config.anthropicApiKey) { | |
| 108 | return { | |
| 109 | category: "AI", | |
| 110 | name: "Anthropic API key", | |
| 111 | status: "yellow", | |
| 112 | detail: "ANTHROPIC_API_KEY unset — AI PR review + AI deploy-failure analysis disabled.", | |
| 113 | fix: "Add ANTHROPIC_API_KEY=sk-ant-xxx to /etc/gluecron.env.", | |
| 114 | }; | |
| 115 | } | |
| 116 | return { | |
| 117 | category: "AI", | |
| 118 | name: "Anthropic API key", | |
| 119 | status: "green", | |
| 120 | detail: `Key present (length ${config.anthropicApiKey.length}).`, | |
| 121 | }; | |
| 122 | } | |
| 123 | ||
| 124 | function checkGateTest(): CheckResult { | |
| 125 | const hasUrl = !!process.env.GATETEST_URL; | |
| 126 | const hasKey = !!process.env.GATETEST_API_KEY; | |
| 127 | if (!hasUrl && !hasKey) { | |
| 128 | return { | |
| 129 | category: "GateTest", | |
| 130 | name: "Scanner integration", | |
| 131 | status: "yellow", | |
| 132 | detail: "Unconfigured — push-time GateTest scans skip silently.", | |
| 133 | fix: "Set GATETEST_URL + GATETEST_API_KEY in /etc/gluecron.env to enable per-push scans.", | |
| 134 | }; | |
| 135 | } | |
| 136 | if (hasUrl && !hasKey) { | |
| 137 | return { | |
| 138 | category: "GateTest", | |
| 139 | name: "Scanner integration", | |
| 140 | status: "red", | |
| 141 | detail: "GATETEST_URL set but GATETEST_API_KEY empty — calls will 401.", | |
| 142 | fix: "Add GATETEST_API_KEY to /etc/gluecron.env.", | |
| 143 | }; | |
| 144 | } | |
| 145 | return { | |
| 146 | category: "GateTest", | |
| 147 | name: "Scanner integration", | |
| 148 | status: "green", | |
| 149 | detail: `Configured — pushes POST to ${process.env.GATETEST_URL}.`, | |
| 150 | }; | |
| 151 | } | |
| 152 | ||
| 153 | function checkBuildSha(): CheckResult { | |
| 154 | const sha = process.env.BUILD_SHA?.trim(); | |
| 155 | if (sha) { | |
| 156 | return { | |
| 157 | category: "Deploy", | |
| 158 | name: "BUILD_SHA pinned", | |
| 159 | status: "green", | |
| 160 | detail: `${sha.slice(0, 12)} — service worker rotates per deploy.`, | |
| 161 | }; | |
| 162 | } | |
| 163 | return { | |
| 164 | category: "Deploy", | |
| 165 | name: "BUILD_SHA pinned", | |
| 166 | status: "yellow", | |
| 167 | detail: "BUILD_SHA unset — falling back to dev-stable. Browsers won't see new deploys reflected in the SW cache.", | |
| 168 | fix: "Latest scripts/self-deploy.sh + hetzner-deploy.yml pin this automatically; trigger a deploy.", | |
| 169 | }; | |
| 170 | } | |
| 171 | ||
| 172 | function checkAppBaseUrl(c: any): CheckResult { | |
| 173 | const expected = config.appBaseUrl; | |
| 174 | const host = c.req.header("host") || ""; | |
| 175 | const proto = | |
| 176 | c.req.header("x-forwarded-proto") || | |
| 177 | (c.req.url.startsWith("https://") ? "https" : "http"); | |
| 178 | const actual = `${proto}://${host}`; | |
| 179 | if (!expected || expected === "http://localhost:3000") { | |
| 180 | return { | |
| 181 | category: "Config", | |
| 182 | name: "APP_BASE_URL canonical", | |
| 183 | status: "yellow", | |
| 184 | detail: `APP_BASE_URL is "${expected}" — outbound email links + WebAuthn origin will be wrong.`, | |
| 185 | fix: "Set APP_BASE_URL=https://gluecron.com in /etc/gluecron.env.", | |
| 186 | }; | |
| 187 | } | |
| 188 | if (host && !expected.endsWith(host)) { | |
| 189 | return { | |
| 190 | category: "Config", | |
| 191 | name: "APP_BASE_URL canonical", | |
| 192 | status: "yellow", | |
| 193 | detail: `APP_BASE_URL=${expected} but request arrived at ${actual}. WebAuthn passkeys issued for one host can't be used at the other.`, | |
| 194 | fix: "Align APP_BASE_URL with the host you actually serve from.", | |
| 195 | }; | |
| 196 | } | |
| 197 | return { | |
| 198 | category: "Config", | |
| 199 | name: "APP_BASE_URL canonical", | |
| 200 | status: "green", | |
| 201 | detail: expected, | |
| 202 | }; | |
| 203 | } | |
| 204 | ||
| 205 | function checkDatabase(): CheckResult { | |
| 206 | const url = config.databaseUrl; | |
| 207 | if (!url) { | |
| 208 | return { | |
| 209 | category: "Database", | |
| 210 | name: "Connection string", | |
| 211 | status: "red", | |
| 212 | detail: "DATABASE_URL unset — every page that queries the DB will 500.", | |
| 213 | fix: "Set DATABASE_URL in /etc/gluecron.env.", | |
| 214 | }; | |
| 215 | } | |
| 216 | let masked = url; | |
| 217 | try { | |
| 218 | const u = new URL(url); | |
| 219 | masked = `${u.protocol}//${u.username ? "***" : ""}@${u.host}${u.pathname}`; | |
| 220 | } catch { | |
| 221 | // unparseable | |
| 222 | return { | |
| 223 | category: "Database", | |
| 224 | name: "Connection string", | |
| 225 | status: "red", | |
| 226 | detail: "DATABASE_URL is not a valid URL.", | |
| b5dd694 | 227 | fix: "Fix the URL format: postgres://user:pass@host:port/dbname", // secrets-ok: placeholder example URL, not a real credential |
| 826eccf | 228 | }; |
| 229 | } | |
| 230 | return { | |
| 231 | category: "Database", | |
| 232 | name: "Connection string", | |
| 233 | status: "green", | |
| 234 | detail: masked, | |
| 235 | }; | |
| 236 | } | |
| 237 | ||
| 238 | async function checkMigrations(): Promise<CheckResult> { | |
| 239 | try { | |
| 240 | const drizzleDir = join(process.cwd(), "drizzle"); | |
| 241 | const files = (await readdir(drizzleDir)).filter((f) => f.endsWith(".sql")); | |
| 242 | const latest = latestMigration(files); | |
| 243 | if (!latest) { | |
| 244 | return { | |
| 245 | category: "Database", | |
| 246 | name: "Migrations applied", | |
| 247 | status: "yellow", | |
| 248 | detail: "No migration files found in drizzle/.", | |
| 249 | }; | |
| 250 | } | |
| 251 | const rows = (await db.execute( | |
| 252 | `SELECT name FROM _migrations ORDER BY name DESC LIMIT 1` as never | |
| 253 | )) as any; | |
| 254 | const list = rows?.rows ?? (Array.isArray(rows) ? rows : []); | |
| 255 | const applied: string | undefined = list[0]?.name; | |
| 256 | if (!applied) { | |
| 257 | return { | |
| 258 | category: "Database", | |
| 259 | name: "Migrations applied", | |
| 260 | status: "red", | |
| 261 | detail: "_migrations table is empty.", | |
| 262 | fix: "Run `bun run db:migrate` on the box.", | |
| 263 | }; | |
| 264 | } | |
| 265 | if (applied !== latest) { | |
| 266 | return { | |
| 267 | category: "Database", | |
| 268 | name: "Migrations applied", | |
| 269 | status: "red", | |
| 270 | detail: `DB at ${applied}, drizzle/ has ${latest}.`, | |
| 271 | fix: "Run `bun run db:migrate` on the box, or redeploy (the workflow runs it).", | |
| 272 | }; | |
| 273 | } | |
| 274 | return { | |
| 275 | category: "Database", | |
| 276 | name: "Migrations applied", | |
| 277 | status: "green", | |
| 278 | detail: `Latest: ${applied}`, | |
| 279 | }; | |
| 280 | } catch (err) { | |
| 281 | return { | |
| 282 | category: "Database", | |
| 283 | name: "Migrations applied", | |
| 284 | status: "yellow", | |
| 285 | detail: `Couldn't read migration state: ${(err as Error).message.slice(0, 100)}`, | |
| 286 | }; | |
| 287 | } | |
| 288 | } | |
| 289 | ||
| 290 | async function checkAutoMerge(): Promise<CheckResult> { | |
| a3b6378 | 291 | // Resolve which repo to check. SELF_HOST_REPO is the canonical |
| 292 | // "this is the platform's own repo" pointer — falling back to | |
| 293 | // `ccantynz/Gluecron.com` keeps the legacy default behaviour. | |
| 294 | // Without this, the check used to hardcode `ccantynz` and report | |
| 295 | // "Owner not found" for installs where the canonical owner is | |
| 296 | // `ccantynz-alt` or anything else. | |
| 297 | const selfRepo = process.env.SELF_HOST_REPO || "ccantynz/Gluecron.com"; | |
| 298 | const [ownerName, repoName] = selfRepo.includes("/") | |
| 299 | ? selfRepo.split("/") | |
| 300 | : [selfRepo, "Gluecron.com"]; | |
| 826eccf | 301 | try { |
| 302 | const [owner] = await db | |
| 303 | .select({ id: users.id }) | |
| 304 | .from(users) | |
| a3b6378 | 305 | .where(eq(users.username, ownerName)) |
| 826eccf | 306 | .limit(1); |
| 307 | if (!owner) { | |
| 308 | return { | |
| 309 | category: "Auto-merge", | |
| 310 | name: "main protection", | |
| 311 | status: "yellow", | |
| a3b6378 | 312 | detail: `Owner user '${ownerName}' not found in users table (looked up via SELF_HOST_REPO).`, |
| 313 | fix: "Set SELF_HOST_REPO=<actual-owner>/<repo> in /etc/gluecron.env, or register the owner.", | |
| 826eccf | 314 | }; |
| 315 | } | |
| 316 | const [repo] = await db | |
| 317 | .select({ id: repositories.id }) | |
| 318 | .from(repositories) | |
| 319 | .where( | |
| 320 | and( | |
| 321 | eq(repositories.ownerId, owner.id), | |
| a3b6378 | 322 | eq(repositories.name, repoName) |
| 826eccf | 323 | ) |
| 324 | ) | |
| 325 | .limit(1); | |
| 326 | if (!repo) { | |
| 327 | return { | |
| 328 | category: "Auto-merge", | |
| 329 | name: "main protection", | |
| 330 | status: "yellow", | |
| a3b6378 | 331 | detail: `Repository row for ${ownerName}/${repoName} not found. Either the platform repo isn't registered in its own DB, or SELF_HOST_REPO points at the wrong owner/name.`, |
| 332 | fix: `Create the repo at /new (owner=${ownerName}, name=${repoName}), or correct SELF_HOST_REPO in /etc/gluecron.env.`, | |
| 826eccf | 333 | }; |
| 334 | } | |
| 335 | const [bp] = await db | |
| 336 | .select({ enableAutoMerge: branchProtection.enableAutoMerge }) | |
| 337 | .from(branchProtection) | |
| 338 | .where( | |
| 339 | and( | |
| 340 | eq(branchProtection.repositoryId, repo.id), | |
| 341 | eq(branchProtection.pattern, "main") | |
| 342 | ) | |
| 343 | ) | |
| 344 | .limit(1); | |
| 345 | if (!bp) { | |
| 346 | return { | |
| 347 | category: "Auto-merge", | |
| 348 | name: "main protection", | |
| 349 | status: "yellow", | |
| 350 | detail: "No branch_protection row for main yet.", | |
| a3b6378 | 351 | fix: `Visit /${ownerName}/${repoName}/gates/protection to configure.`, |
| 826eccf | 352 | }; |
| 353 | } | |
| 354 | return { | |
| 355 | category: "Auto-merge", | |
| 356 | name: "main protection", | |
| 357 | status: bp.enableAutoMerge ? "green" : "yellow", | |
| 358 | detail: bp.enableAutoMerge | |
| 359 | ? "Auto-merge ENABLED on main." | |
| 360 | : "Auto-merge DISABLED on main.", | |
| 361 | fix: bp.enableAutoMerge | |
| 362 | ? undefined | |
| 363 | : "Visit /admin/ops to enable.", | |
| 364 | }; | |
| 365 | } catch (err) { | |
| 366 | return { | |
| 367 | category: "Auto-merge", | |
| 368 | name: "main protection", | |
| 369 | status: "yellow", | |
| 370 | detail: `Couldn't read: ${(err as Error).message.slice(0, 100)}`, | |
| 371 | }; | |
| 372 | } | |
| 373 | } | |
| 374 | ||
| 375 | async function checkSyntheticMonitor(): Promise<CheckResult> { | |
| 376 | try { | |
| 377 | const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); | |
| 378 | const reds = await db | |
| 379 | .select({ name: syntheticChecks.checkName }) | |
| 380 | .from(syntheticChecks) | |
| 381 | .where( | |
| 382 | and( | |
| 383 | eq(syntheticChecks.status, "red"), | |
| 384 | gt(syntheticChecks.checkedAt, oneHourAgo) | |
| 385 | ) | |
| 386 | ) | |
| 387 | .orderBy(desc(syntheticChecks.checkedAt)) | |
| 388 | .limit(10); | |
| 389 | if (reds.length === 0) { | |
| 390 | return { | |
| 391 | category: "Monitor", | |
| 392 | name: "Synthetic checks (1h)", | |
| 393 | status: "green", | |
| 394 | detail: "All synthetic checks green in the last hour.", | |
| 395 | }; | |
| 396 | } | |
| 397 | const names = Array.from(new Set(reds.map((r) => r.name))).slice(0, 5); | |
| 398 | return { | |
| 399 | category: "Monitor", | |
| 400 | name: "Synthetic checks (1h)", | |
| 401 | status: "red", | |
| 402 | detail: `Red in last hour: ${names.join(", ")}.`, | |
| 403 | fix: "Open /admin/status for the full row table.", | |
| 404 | }; | |
| 405 | } catch (err) { | |
| 406 | return { | |
| 407 | category: "Monitor", | |
| 408 | name: "Synthetic checks (1h)", | |
| 409 | status: "yellow", | |
| 410 | detail: `Couldn't read: ${(err as Error).message.slice(0, 100)}`, | |
| 411 | }; | |
| 412 | } | |
| 413 | } | |
| 414 | ||
| 415 | function checkSelfHost(): CheckResult { | |
| 416 | const repo = process.env.SELF_HOST_REPO; | |
| 417 | if (!repo) { | |
| 418 | return { | |
| 419 | category: "Self-host", | |
| 420 | name: "Bootstrap", | |
| 421 | status: "yellow", | |
| 422 | detail: "SELF_HOST_REPO unset — pushes to this repo don't trigger self-deploy.", | |
| 423 | fix: "Run scripts/self-host-bootstrap.ts on the box and add SELF_HOST_REPO=ccantynz/Gluecron.com to /etc/gluecron.env.", | |
| 424 | }; | |
| 425 | } | |
| 426 | return { | |
| 427 | category: "Self-host", | |
| 428 | name: "Bootstrap", | |
| 429 | status: "green", | |
| 430 | detail: `Self-hosting ${repo} — push to main fires self-deploy.sh.`, | |
| 431 | }; | |
| 432 | } | |
| 433 | ||
| 115c66b | 434 | // ─── New checks (2026-05-16 reliability sweep) ─────────────────────────── |
| 435 | ||
| 436 | /** | |
| 437 | * Is the autopilot loop ticking on schedule? If not, half the platform's | |
| 438 | * self-healing breaks silently — mirror sync, advisory rescans, scheduled | |
| 439 | * workflows, auto-merge sweep, stale-sweep all skip. | |
| 440 | */ | |
| 441 | function checkAutopilot(): CheckResult { | |
| 442 | if (process.env.AUTOPILOT_DISABLED === "1") { | |
| 443 | return { | |
| 444 | category: "Autopilot", | |
| 445 | name: "Background loop", | |
| 446 | status: "yellow", | |
| 447 | detail: "AUTOPILOT_DISABLED=1 — background maintenance loop is OFF.", | |
| 448 | fix: "Remove or unset AUTOPILOT_DISABLED in /etc/gluecron.env to re-enable.", | |
| 449 | }; | |
| 450 | } | |
| 451 | const total = getTickCount(); | |
| 452 | const tick = getLastTick(); | |
| 453 | const intervalRaw = process.env.AUTOPILOT_INTERVAL_MS; | |
| 454 | const intervalMs = | |
| 455 | intervalRaw && Number.isFinite(Number(intervalRaw)) && Number(intervalRaw) > 0 | |
| 456 | ? Number(intervalRaw) | |
| 457 | : 5 * 60 * 1000; | |
| 458 | // Allow 2x the interval before flagging — accounts for slow ticks. | |
| 459 | const staleMs = intervalMs * 2; | |
| 460 | if (!tick) { | |
| 461 | if (total === 0) { | |
| 462 | return { | |
| 463 | category: "Autopilot", | |
| 464 | name: "Background loop", | |
| 465 | status: "yellow", | |
| 466 | detail: `Loop is enabled but has not ticked yet. First tick fires after ${Math.round(intervalMs / 1000)}s.`, | |
| 467 | }; | |
| 468 | } | |
| 469 | return { | |
| 470 | category: "Autopilot", | |
| 471 | name: "Background loop", | |
| 472 | status: "red", | |
| 473 | detail: `${total} tick(s) recorded but last tick result is missing — loop may have crashed.`, | |
| 474 | fix: "Check journalctl -u gluecron for [autopilot] errors. Run a tick manually at /admin/autopilot.", | |
| 475 | }; | |
| 476 | } | |
| 477 | const finishedAt = new Date(tick.finishedAt).getTime(); | |
| 478 | const ageMs = Date.now() - finishedAt; | |
| 479 | if (ageMs > staleMs) { | |
| 480 | return { | |
| 481 | category: "Autopilot", | |
| 482 | name: "Background loop", | |
| 483 | status: "red", | |
| 484 | detail: `Last tick was ${Math.round(ageMs / 1000)}s ago (interval is ${Math.round(intervalMs / 1000)}s). Loop is stalled.`, | |
| 485 | fix: "Run a tick manually at /admin/autopilot. Check journalctl for [autopilot] errors.", | |
| 486 | }; | |
| 487 | } | |
| 488 | const failed = tick.tasks.filter((t) => !t.ok).length; | |
| 489 | if (failed > 0) { | |
| 490 | return { | |
| 491 | category: "Autopilot", | |
| 492 | name: "Background loop", | |
| 493 | status: "yellow", | |
| 494 | detail: `Loop running but ${failed}/${tick.tasks.length} tasks failed in the last tick.`, | |
| 495 | fix: "Open /admin/autopilot for the per-task error list.", | |
| 496 | }; | |
| 497 | } | |
| 498 | return { | |
| 499 | category: "Autopilot", | |
| 500 | name: "Background loop", | |
| 501 | status: "green", | |
| 502 | detail: `Ticking on schedule (${total} tick${total === 1 ? "" : "s"} this process; last ${Math.round(ageMs / 1000)}s ago).`, | |
| 503 | }; | |
| 504 | } | |
| 505 | ||
| 506 | /** | |
| 507 | * When did we last successfully deploy? Stale deploys are an early | |
| 508 | * warning sign the deploy pipeline is broken silently (which is exactly | |
| 509 | * what happened on 2026-05-15 — 17 hours of failed deploys, no alert). | |
| 510 | */ | |
| 511 | async function checkRecentDeploy(): Promise<CheckResult> { | |
| 512 | try { | |
| 513 | const [latest] = await db | |
| 514 | .select({ | |
| 515 | sha: platformDeploys.sha, | |
| 516 | status: platformDeploys.status, | |
| 517 | startedAt: platformDeploys.startedAt, | |
| 518 | finishedAt: platformDeploys.finishedAt, | |
| 519 | error: platformDeploys.error, | |
| 520 | }) | |
| 521 | .from(platformDeploys) | |
| 522 | .orderBy(desc(platformDeploys.startedAt)) | |
| 523 | .limit(1); | |
| 524 | if (!latest) { | |
| 525 | return { | |
| 526 | category: "Deploy", | |
| 527 | name: "Latest deploy", | |
| 528 | status: "yellow", | |
| 529 | detail: "No deploys recorded yet. The hetzner-deploy.yml workflow posts events to /api/events/deploy/* — set DEPLOY_EVENT_TOKEN in the workflow env to enable.", | |
| 530 | }; | |
| 531 | } | |
| 532 | const ref = latest.finishedAt || latest.startedAt; | |
| 533 | const ageHours = (Date.now() - new Date(ref).getTime()) / (60 * 60 * 1000); | |
| 534 | const sha7 = (latest.sha || "").slice(0, 7); | |
| 535 | if (latest.status === "failed") { | |
| 536 | return { | |
| 537 | category: "Deploy", | |
| 538 | name: "Latest deploy", | |
| 539 | status: "red", | |
| 540 | detail: `Last deploy (${sha7}) FAILED ${ageHours.toFixed(1)}h ago: ${(latest.error || "no error message").slice(0, 200)}.`, | |
| 541 | fix: "Open /admin/deploys for the run timeline. Trigger a new deploy after fixing.", | |
| 542 | }; | |
| 543 | } | |
| 544 | if (latest.status === "in_progress") { | |
| 545 | return { | |
| 546 | category: "Deploy", | |
| 547 | name: "Latest deploy", | |
| 548 | status: "yellow", | |
| 549 | detail: `Deploy in progress (${sha7}, started ${ageHours.toFixed(1)}h ago).`, | |
| 550 | }; | |
| 551 | } | |
| 552 | if (latest.status === "succeeded" && ageHours > 48) { | |
| 553 | return { | |
| 554 | category: "Deploy", | |
| 555 | name: "Latest deploy", | |
| 556 | status: "yellow", | |
| 557 | detail: `Last deploy was ${sha7} ${ageHours.toFixed(1)}h ago. If you pushed to main since then, the deploy pipeline may have silently failed.`, | |
| 558 | fix: "Check the GitHub Actions Hetzner deploy run for the latest main commit.", | |
| 559 | }; | |
| 560 | } | |
| 561 | return { | |
| 562 | category: "Deploy", | |
| 563 | name: "Latest deploy", | |
| 564 | status: "green", | |
| 565 | detail: `${sha7} deployed cleanly ${ageHours.toFixed(1)}h ago.`, | |
| 566 | }; | |
| 567 | } catch (err) { | |
| 568 | return { | |
| 569 | category: "Deploy", | |
| 570 | name: "Latest deploy", | |
| 571 | status: "yellow", | |
| 572 | detail: `Couldn't read platform_deploys: ${(err as Error).message.slice(0, 100)}`, | |
| 573 | }; | |
| 574 | } | |
| 575 | } | |
| 576 | ||
| 577 | /** | |
| 578 | * Is the workflow worker draining the queue? A backed-up queue or a | |
| 579 | * stuck queued row means CI gates aren't firing. | |
| 580 | */ | |
| 581 | async function checkWorkflowQueue(): Promise<CheckResult> { | |
| 582 | try { | |
| 583 | const [queued] = await db | |
| 584 | .select({ n: sql<number>`count(*)::int` }) | |
| 585 | .from(workflowRuns) | |
| 586 | .where(eq(workflowRuns.status, "queued")); | |
| 587 | const queuedN = Number(queued?.n || 0); | |
| 588 | const [running] = await db | |
| 589 | .select({ n: sql<number>`count(*)::int` }) | |
| 590 | .from(workflowRuns) | |
| 591 | .where(eq(workflowRuns.status, "running")); | |
| 592 | const runningN = Number(running?.n || 0); | |
| 593 | if (queuedN > 25) { | |
| 594 | return { | |
| 595 | category: "Workflows", | |
| 596 | name: "Run queue", | |
| 597 | status: "red", | |
| 598 | detail: `${queuedN} runs queued (running: ${runningN}). The worker is backed up.`, | |
| 599 | fix: "Check journalctl for [workflow-runner] errors. Restart gluecron if persistent.", | |
| 600 | }; | |
| 601 | } | |
| 602 | if (queuedN > 5) { | |
| 603 | return { | |
| 604 | category: "Workflows", | |
| 605 | name: "Run queue", | |
| 606 | status: "yellow", | |
| 607 | detail: `${queuedN} runs queued, ${runningN} running. Worker may be slow.`, | |
| 608 | }; | |
| 609 | } | |
| 610 | return { | |
| 611 | category: "Workflows", | |
| 612 | name: "Run queue", | |
| 613 | status: "green", | |
| 614 | detail: `${queuedN} queued, ${runningN} running.`, | |
| 615 | }; | |
| 616 | } catch (err) { | |
| 617 | return { | |
| 618 | category: "Workflows", | |
| 619 | name: "Run queue", | |
| 620 | status: "yellow", | |
| 621 | detail: `Couldn't read workflow_runs: ${(err as Error).message.slice(0, 100)}`, | |
| 622 | }; | |
| 623 | } | |
| 624 | } | |
| 625 | ||
| 626 | /** | |
| 9ecf5a4 | 627 | * Vapron deploy webhook secret — without it, the webhook POSTs |
| 628 | * unsigned and Vapron rejects with 401, but our hook side never sees | |
| 115c66b | 629 | * the rejection because the request is fire-and-forget. |
| 630 | */ | |
| 9ecf5a4 | 631 | function checkVapronWebhook(): CheckResult { |
| 632 | const url = | |
| 633 | process.env.VAPRON_DEPLOY_URL || process.env.CRONTECH_DEPLOY_URL; | |
| 634 | const secret = | |
| 635 | process.env.VAPRON_HMAC_SECRET || | |
| 636 | process.env.CRONTECH_HMAC_SECRET || | |
| 637 | process.env.GLUECRON_WEBHOOK_SECRET; | |
| 115c66b | 638 | if (!url) { |
| 639 | return { | |
| 9ecf5a4 | 640 | category: "Vapron", |
| 115c66b | 641 | name: "Deploy webhook", |
| 642 | status: "yellow", | |
| 9ecf5a4 | 643 | detail: "VAPRON_DEPLOY_URL unset — pushes to the Vapron repo don't notify the deploy pipeline.", |
| 644 | fix: "Optional integration. Set VAPRON_DEPLOY_URL + VAPRON_HMAC_SECRET (admin → integrations) if you want push-triggered Vapron deploys.", | |
| 115c66b | 645 | }; |
| 646 | } | |
| 647 | if (!secret) { | |
| 648 | return { | |
| 9ecf5a4 | 649 | category: "Vapron", |
| 115c66b | 650 | name: "Deploy webhook", |
| 651 | status: "red", | |
| 9ecf5a4 | 652 | detail: "Webhook URL set but no HMAC secret — webhook will be rejected as unsigned.", |
| 653 | fix: "Set VAPRON_HMAC_SECRET on /admin/integrations (match the value configured on Vapron's side).", | |
| 115c66b | 654 | }; |
| 655 | } | |
| 656 | return { | |
| 9ecf5a4 | 657 | category: "Vapron", |
| 115c66b | 658 | name: "Deploy webhook", |
| 659 | status: "green", | |
| 660 | detail: `Configured (POST to ${url}).`, | |
| 661 | }; | |
| 662 | } | |
| 663 | ||
| 826eccf | 664 | // ─── Page handler ──────────────────────────────────────────────────────── |
| 665 | ||
| 666 | function pill(status: CheckStatus): any { | |
| 667 | const map: Record<CheckStatus, { bg: string; fg: string; label: string }> = { | |
| e589f77 | 668 | green: { bg: "rgba(52,211,153,0.16)", fg: "var(--green)", label: "✓ OK" }, |
| 826eccf | 669 | yellow: { bg: "rgba(245,158,11,0.16)", fg: "#f59e0b", label: "! WARN" }, |
| e589f77 | 670 | red: { bg: "rgba(248,113,113,0.16)", fg: "var(--red)", label: "× FAIL" }, |
| 826eccf | 671 | }; |
| 672 | const s = map[status]; | |
| 673 | return ( | |
| 674 | <span | |
| 675 | style={`display:inline-flex;align-items:center;padding:2px 10px;border-radius:999px;font-size:12px;font-weight:600;background:${s.bg};color:${s.fg};white-space:nowrap`} | |
| 676 | > | |
| 677 | {s.label} | |
| 678 | </span> | |
| 679 | ); | |
| 680 | } | |
| 681 | ||
| e7b4b7e | 682 | /** |
| 683 | * Map a check to its most useful "Fix" deep-link. Pure category-based | |
| 684 | * routing — keeps the per-check status logic untouched, but lets the new | |
| 685 | * polish surface a one-click jump to the page that resolves the issue. | |
| 686 | */ | |
| 687 | function fixHrefForCheck(r: CheckResult): { href: string; label: string } | null { | |
| 688 | // Anything env/secret-driven goes to /admin/integrations — the in-app | |
| 689 | // editor for /etc/gluecron.env that already gates on the same admin role. | |
| 690 | const envSurface = { href: "/admin/integrations", label: "Open integrations" }; | |
| 691 | switch (r.category) { | |
| 692 | case "Email": | |
| 693 | case "AI": | |
| 694 | case "GateTest": | |
| 695 | case "Crontech": | |
| 696 | case "Config": | |
| 697 | case "Self-host": | |
| 698 | return envSurface; | |
| 699 | case "Auto-merge": | |
| 700 | return { href: "/admin/ops", label: "Open ops" }; | |
| 701 | case "Monitor": | |
| 702 | return { href: "/admin/status", label: "Open status" }; | |
| 703 | case "Autopilot": | |
| 704 | return { href: "/admin/autopilot", label: "Open autopilot" }; | |
| 705 | case "Deploy": | |
| 706 | return { href: "/admin/deploys", label: "Open deploys" }; | |
| 707 | case "Workflows": | |
| 708 | return { href: "/admin/ops", label: "Open ops" }; | |
| 709 | case "Database": | |
| 710 | return envSurface; | |
| 711 | default: | |
| 712 | return null; | |
| 713 | } | |
| 714 | } | |
| 715 | ||
| 716 | /* ───────────────────────────────────────────────────────────────────────── | |
| 717 | * Scoped CSS — every class prefixed `.health-` so this surface can't | |
| 718 | * bleed into the wider admin. Mirrors the gradient-hairline hero + | |
| 719 | * radial-orb + per-card pattern from `admin-integrations` and | |
| 720 | * `error-page` (the just-shipped 2026 visual recipe). | |
| 721 | * ───────────────────────────────────────────────────────────────────── */ | |
| 722 | const healthStyles = ` | |
| eed4684 | 723 | .health-wrap { max-width: 1680px; margin: 0 auto; padding: var(--space-6) var(--space-4); } |
| e7b4b7e | 724 | |
| 725 | .health-hero { | |
| 726 | position: relative; | |
| 727 | margin-bottom: var(--space-5); | |
| 728 | padding: var(--space-5) var(--space-6); | |
| 729 | background: var(--bg-elevated); | |
| 730 | border: 1px solid var(--border); | |
| 731 | border-radius: 16px; | |
| 732 | overflow: hidden; | |
| 733 | } | |
| 734 | .health-hero::before { | |
| 735 | content: ''; | |
| 736 | position: absolute; | |
| 737 | top: 0; left: 0; right: 0; | |
| 738 | height: 2px; | |
| 6fd5915 | 739 | background: linear-gradient(90deg, transparent 0%, #5b6ee8 30%, #5f8fa0 70%, transparent 100%); |
| e7b4b7e | 740 | opacity: 0.75; |
| 741 | pointer-events: none; | |
| 742 | } | |
| 743 | .health-hero-orb { | |
| 744 | position: absolute; | |
| 745 | inset: -30% -15% auto auto; | |
| 746 | width: 460px; height: 460px; | |
| 6fd5915 | 747 | background: radial-gradient(circle, rgba(91,110,232,0.22), rgba(95,143,160,0.10) 45%, transparent 70%); |
| e7b4b7e | 748 | filter: blur(80px); |
| 749 | opacity: 0.75; | |
| 750 | pointer-events: none; | |
| 751 | z-index: 0; | |
| 752 | } | |
| 753 | .health-hero-inner { position: relative; z-index: 1; } | |
| 754 | .health-hero-top { | |
| 755 | display: flex; | |
| 756 | align-items: center; | |
| 757 | justify-content: space-between; | |
| 758 | gap: var(--space-3); | |
| 759 | margin-bottom: var(--space-3); | |
| 760 | flex-wrap: wrap; | |
| 761 | } | |
| 762 | .health-eyebrow { | |
| 763 | display: inline-flex; | |
| 764 | align-items: center; | |
| 765 | gap: 8px; | |
| 766 | text-transform: uppercase; | |
| 767 | font-family: var(--font-mono); | |
| 768 | font-size: 11px; | |
| 769 | letter-spacing: 0.18em; | |
| 770 | color: var(--text-muted); | |
| 771 | font-weight: 600; | |
| 772 | } | |
| 773 | .health-eyebrow-dot { | |
| 774 | width: 8px; height: 8px; | |
| 775 | border-radius: 9999px; | |
| 6fd5915 | 776 | background: linear-gradient(135deg, #5b6ee8, #5f8fa0); |
| 777 | box-shadow: 0 0 0 3px rgba(91,110,232,0.18); | |
| e7b4b7e | 778 | } |
| 779 | .health-back { | |
| 780 | font-size: 12.5px; | |
| 781 | color: var(--text-muted); | |
| 782 | text-decoration: none; | |
| 783 | padding: 6px 12px; | |
| 784 | border-radius: 8px; | |
| 785 | border: 1px solid var(--border-strong, var(--border)); | |
| 786 | background: transparent; | |
| 787 | transition: background 120ms ease, color 120ms ease, border-color 120ms ease; | |
| 788 | } | |
| 789 | .health-back:hover { | |
| 790 | color: var(--text-strong); | |
| 6fd5915 | 791 | border-color: rgba(91,110,232,0.45); |
| 792 | background: rgba(91,110,232,0.06); | |
| e7b4b7e | 793 | text-decoration: none; |
| 794 | } | |
| 795 | .health-title { | |
| 796 | font-family: var(--font-display); | |
| 797 | font-size: clamp(28px, 4vw, 40px); | |
| 798 | font-weight: 800; | |
| 799 | letter-spacing: -0.028em; | |
| 800 | line-height: 1.05; | |
| 801 | margin: 0 0 var(--space-3); | |
| 802 | color: var(--text-strong); | |
| 803 | } | |
| 804 | .health-title-grad { | |
| 6fd5915 | 805 | background-image: linear-gradient(135deg, #5b6ee8 0%, #5b6ee8 50%, #5f8fa0 100%); |
| e7b4b7e | 806 | -webkit-background-clip: text; |
| 807 | background-clip: text; | |
| 808 | -webkit-text-fill-color: transparent; | |
| 809 | color: transparent; | |
| 810 | } | |
| 811 | .health-title-grad.is-warn { | |
| 812 | background-image: linear-gradient(135deg, #fde68a 0%, #fbbf24 50%, #d97706 100%); | |
| 813 | } | |
| 814 | .health-title-grad.is-fail { | |
| 815 | background-image: linear-gradient(135deg, #fecaca 0%, #f87171 50%, #ef4444 100%); | |
| 816 | } | |
| 817 | .health-summary { | |
| 818 | display: flex; | |
| 819 | align-items: center; | |
| 820 | gap: var(--space-3); | |
| 821 | flex-wrap: wrap; | |
| 822 | } | |
| 823 | .health-summary-pill { | |
| 824 | display: inline-flex; | |
| 825 | align-items: center; | |
| 826 | gap: 8px; | |
| 827 | padding: 6px 14px; | |
| 828 | border-radius: 9999px; | |
| 829 | font-size: 13px; | |
| 830 | font-weight: 600; | |
| 831 | letter-spacing: -0.005em; | |
| 832 | } | |
| 833 | .health-summary-pill .dot { | |
| 834 | width: 9px; height: 9px; | |
| 835 | border-radius: 9999px; | |
| 836 | background: currentColor; | |
| 837 | box-shadow: 0 0 0 3px rgba(255,255,255,0.04); | |
| 838 | } | |
| 839 | .health-summary-pill.is-green { | |
| 840 | background: rgba(52,211,153,0.12); | |
| 841 | color: #6ee7b7; | |
| 842 | box-shadow: inset 0 0 0 1px rgba(52,211,153,0.32); | |
| 843 | } | |
| 844 | .health-summary-pill.is-warn { | |
| 845 | background: rgba(251,191,36,0.10); | |
| 846 | color: #fde68a; | |
| 847 | box-shadow: inset 0 0 0 1px rgba(251,191,36,0.32); | |
| 848 | } | |
| 849 | .health-summary-pill.is-fail { | |
| 850 | background: rgba(248,113,113,0.10); | |
| 851 | color: #fecaca; | |
| 852 | box-shadow: inset 0 0 0 1px rgba(248,113,113,0.34); | |
| 853 | } | |
| 854 | .health-summary-breakdown { | |
| 855 | font-size: 12.5px; | |
| 856 | color: var(--text-muted); | |
| 857 | font-family: var(--font-mono); | |
| 858 | } | |
| 859 | .health-summary-breakdown .sep { opacity: 0.45; margin: 0 6px; } | |
| 860 | .health-summary-stamp { | |
| 861 | margin-left: auto; | |
| 862 | font-size: 11.5px; | |
| 863 | color: var(--text-muted); | |
| 864 | font-family: var(--font-mono); | |
| 865 | font-variant-numeric: tabular-nums; | |
| 866 | letter-spacing: 0.01em; | |
| 867 | } | |
| 868 | ||
| 869 | .health-banner { | |
| 870 | margin-bottom: var(--space-4); | |
| 871 | padding: 10px 14px; | |
| 872 | border-radius: 10px; | |
| 873 | font-size: 13.5px; | |
| 874 | border: 1px solid var(--border); | |
| 875 | background: rgba(255,255,255,0.025); | |
| 876 | color: var(--text); | |
| 877 | } | |
| 878 | .health-banner.is-ok { | |
| 879 | border-color: rgba(52,211,153,0.40); | |
| 880 | background: rgba(52,211,153,0.08); | |
| 881 | color: #bbf7d0; | |
| 882 | } | |
| 883 | .health-banner.is-error { | |
| 884 | border-color: rgba(248,113,113,0.40); | |
| 885 | background: rgba(248,113,113,0.08); | |
| 886 | color: #fecaca; | |
| 887 | } | |
| 888 | ||
| 889 | .health-grid { | |
| 890 | display: grid; | |
| 891 | grid-template-columns: 1fr; | |
| 892 | gap: var(--space-3); | |
| 893 | margin-bottom: var(--space-5); | |
| 894 | } | |
| 895 | @media (min-width: 720px) { | |
| 896 | .health-grid { grid-template-columns: 1fr 1fr; } | |
| 897 | } | |
| 898 | ||
| 899 | .health-card { | |
| 900 | position: relative; | |
| 901 | background: var(--bg-elevated); | |
| 902 | border: 1px solid var(--border); | |
| 903 | border-radius: 14px; | |
| 904 | padding: var(--space-4); | |
| 905 | display: flex; | |
| 906 | flex-direction: column; | |
| 907 | gap: 10px; | |
| 908 | transition: border-color 120ms ease, transform 120ms ease, box-shadow 120ms ease; | |
| 909 | } | |
| 910 | .health-card:hover { | |
| 911 | border-color: var(--border-strong, var(--border)); | |
| 912 | transform: translateY(-1px); | |
| 913 | box-shadow: 0 6px 18px -10px rgba(0,0,0,0.45); | |
| 914 | } | |
| 915 | .health-card.is-red { border-color: rgba(248,113,113,0.34); } | |
| 916 | .health-card.is-yellow { border-color: rgba(251,191,36,0.30); } | |
| 917 | .health-card.is-green { border-color: rgba(52,211,153,0.22); } | |
| 918 | ||
| 919 | .health-card-head { | |
| 920 | display: flex; | |
| 921 | align-items: flex-start; | |
| 922 | gap: 12px; | |
| 923 | justify-content: space-between; | |
| 924 | } | |
| 925 | .health-card-id { display: flex; align-items: flex-start; gap: 12px; min-width: 0; } | |
| 926 | .health-card-dot { | |
| 927 | flex: 0 0 auto; | |
| 928 | width: 12px; height: 12px; | |
| 929 | border-radius: 9999px; | |
| 930 | margin-top: 5px; | |
| 931 | background: var(--text-muted); | |
| 932 | box-shadow: 0 0 0 3px rgba(255,255,255,0.04); | |
| 933 | } | |
| 934 | .health-card-dot.is-green { | |
| e589f77 | 935 | background: var(--green); |
| e7b4b7e | 936 | box-shadow: 0 0 0 3px rgba(52,211,153,0.16); |
| 937 | } | |
| 938 | .health-card-dot.is-yellow { | |
| 939 | background: #f59e0b; | |
| 940 | box-shadow: 0 0 0 3px rgba(245,158,11,0.18); | |
| 941 | } | |
| 942 | .health-card-dot.is-red { | |
| e589f77 | 943 | background: var(--red); |
| e7b4b7e | 944 | box-shadow: 0 0 0 3px rgba(248,113,113,0.22); |
| 945 | animation: health-pulse 1.8s ease-in-out infinite; | |
| 946 | } | |
| 947 | @keyframes health-pulse { | |
| 948 | 0%, 100% { box-shadow: 0 0 0 3px rgba(248,113,113,0.22); } | |
| 949 | 50% { box-shadow: 0 0 0 7px rgba(248,113,113,0.05); } | |
| 950 | } | |
| 951 | @media (prefers-reduced-motion: reduce) { | |
| 952 | .health-card-dot.is-red { animation: none; } | |
| 953 | } | |
| 954 | .health-card-title { | |
| 955 | min-width: 0; | |
| 956 | } | |
| 957 | .health-card-category { | |
| 958 | font-size: 10.5px; | |
| 959 | letter-spacing: 0.14em; | |
| 960 | text-transform: uppercase; | |
| 961 | color: var(--text-muted); | |
| 962 | font-weight: 700; | |
| 963 | margin-bottom: 2px; | |
| 964 | } | |
| 965 | .health-card-name { | |
| 966 | font-family: var(--font-mono); | |
| 967 | font-size: 14px; | |
| 968 | font-weight: 600; | |
| 969 | color: var(--text-strong); | |
| 970 | letter-spacing: -0.005em; | |
| 971 | overflow-wrap: anywhere; | |
| 972 | } | |
| 973 | ||
| 974 | .health-card-detail { | |
| 975 | font-size: 13px; | |
| 976 | line-height: 1.55; | |
| 977 | color: var(--text); | |
| 978 | margin: 0; | |
| 979 | overflow-wrap: anywhere; | |
| 980 | } | |
| 981 | ||
| 982 | .health-card-fix { | |
| 983 | font-size: 12.5px; | |
| 984 | line-height: 1.5; | |
| 985 | color: var(--text-muted); | |
| 986 | margin: 0; | |
| 987 | padding: 10px 12px; | |
| 6fd5915 | 988 | background: rgba(91,110,232,0.05); |
| 989 | border: 1px solid rgba(91,110,232,0.18); | |
| e7b4b7e | 990 | border-radius: 10px; |
| 991 | } | |
| 992 | .health-card.is-red .health-card-fix { | |
| 993 | background: rgba(248,113,113,0.05); | |
| 994 | border-color: rgba(248,113,113,0.22); | |
| 995 | } | |
| 996 | .health-card.is-yellow .health-card-fix { | |
| 997 | background: rgba(251,191,36,0.05); | |
| 998 | border-color: rgba(251,191,36,0.22); | |
| 999 | } | |
| 1000 | .health-card-fix-label { | |
| 1001 | display: block; | |
| 1002 | font-size: 10.5px; | |
| 1003 | text-transform: uppercase; | |
| 1004 | letter-spacing: 0.14em; | |
| 1005 | font-weight: 700; | |
| 1006 | color: var(--text-muted); | |
| 1007 | margin-bottom: 4px; | |
| 1008 | } | |
| 1009 | ||
| 1010 | .health-card-foot { | |
| 1011 | display: flex; | |
| 1012 | justify-content: flex-end; | |
| 1013 | margin-top: auto; | |
| 1014 | padding-top: 4px; | |
| 1015 | } | |
| 1016 | .health-card-action { | |
| 1017 | display: inline-flex; | |
| 1018 | align-items: center; | |
| 1019 | gap: 6px; | |
| 1020 | padding: 6px 12px; | |
| 1021 | border-radius: 8px; | |
| 1022 | font-size: 12.5px; | |
| 1023 | font-weight: 600; | |
| 1024 | text-decoration: none; | |
| 1025 | color: var(--text); | |
| 1026 | background: transparent; | |
| 1027 | border: 1px solid var(--border-strong, var(--border)); | |
| 1028 | transition: background 120ms ease, color 120ms ease, border-color 120ms ease, transform 120ms ease; | |
| 1029 | } | |
| 1030 | .health-card-action:hover { | |
| 1031 | color: var(--text-strong); | |
| 6fd5915 | 1032 | border-color: rgba(91,110,232,0.45); |
| 1033 | background: rgba(91,110,232,0.08); | |
| e7b4b7e | 1034 | text-decoration: none; |
| 1035 | transform: translateY(-1px); | |
| 1036 | } | |
| 1037 | .health-card-action .arrow { font-size: 14px; line-height: 1; } | |
| 1038 | ||
| 1039 | .health-test { | |
| 1040 | position: relative; | |
| 1041 | background: var(--bg-elevated); | |
| 1042 | border: 1px solid var(--border); | |
| 1043 | border-radius: 14px; | |
| 1044 | padding: var(--space-4) var(--space-5); | |
| 1045 | overflow: hidden; | |
| 1046 | } | |
| 1047 | .health-test::before { | |
| 1048 | content: ''; | |
| 1049 | position: absolute; | |
| 1050 | top: 0; left: 0; right: 0; | |
| 1051 | height: 1px; | |
| 6fd5915 | 1052 | background: linear-gradient(90deg, transparent 0%, rgba(91,110,232,0.45) 50%, transparent 100%); |
| e7b4b7e | 1053 | opacity: 0.6; |
| 1054 | } | |
| 1055 | .health-test h3 { | |
| 1056 | margin: 0 0 4px 0; | |
| 1057 | font-family: var(--font-display); | |
| 1058 | font-size: 16px; | |
| 1059 | font-weight: 700; | |
| 1060 | color: var(--text-strong); | |
| 1061 | letter-spacing: -0.012em; | |
| 1062 | } | |
| 1063 | .health-test p { | |
| 1064 | margin: 0 0 var(--space-3) 0; | |
| 1065 | font-size: 13px; | |
| 1066 | color: var(--text-muted); | |
| 1067 | line-height: 1.5; | |
| 1068 | } | |
| 1069 | .health-test form { margin: 0; } | |
| 1070 | `; | |
| 1071 | ||
| 115c66b | 1072 | async function runAllChecks(c: any): Promise<CheckResult[]> { |
| 1073 | return [ | |
| 826eccf | 1074 | checkEmail(), |
| 1075 | checkAnthropic(), | |
| 1076 | checkGateTest(), | |
| 1077 | checkBuildSha(), | |
| 1078 | checkAppBaseUrl(c), | |
| 1079 | checkDatabase(), | |
| 1080 | await checkMigrations(), | |
| 1081 | await checkAutoMerge(), | |
| 1082 | await checkSyntheticMonitor(), | |
| 1083 | checkSelfHost(), | |
| 115c66b | 1084 | // 2026-05-16 reliability sweep additions: |
| 1085 | checkAutopilot(), | |
| 1086 | await checkRecentDeploy(), | |
| 1087 | await checkWorkflowQueue(), | |
| 9ecf5a4 | 1088 | checkVapronWebhook(), |
| 826eccf | 1089 | ]; |
| 115c66b | 1090 | } |
| 1091 | ||
| 1092 | // JSON endpoint for programmatic monitoring. Same gate as the HTML page | |
| 1093 | // (site-admin only) so deploy state isn't public. | |
| 1094 | diagnose.get("/admin/diagnose.json", async (c) => { | |
| 1095 | const g = await gate(c); | |
| 1096 | if (g instanceof Response) return g; | |
| 1097 | const results = await runAllChecks(c); | |
| 1098 | const counts = { | |
| 1099 | green: results.filter((r) => r.status === "green").length, | |
| 1100 | yellow: results.filter((r) => r.status === "yellow").length, | |
| 1101 | red: results.filter((r) => r.status === "red").length, | |
| 1102 | }; | |
| 1103 | const overall = | |
| 1104 | counts.red > 0 ? "red" : counts.yellow > 0 ? "yellow" : "green"; | |
| 1105 | return c.json({ | |
| 1106 | ok: true, | |
| 1107 | overall, | |
| 1108 | counts, | |
| 1109 | checks: results, | |
| 1110 | asOf: new Date().toISOString(), | |
| 1111 | }); | |
| 1112 | }); | |
| 1113 | ||
| 1114 | // /admin/health alias — same handler, friendlier URL. The user expected | |
| 1115 | // this to exist; making the expectation reality is cheaper than arguing | |
| 1116 | // about naming. | |
| 1117 | diagnose.get("/admin/health", async (c) => { | |
| 1118 | return c.redirect("/admin/diagnose"); | |
| 1119 | }); | |
| 1120 | ||
| 1121 | diagnose.get("/admin/diagnose", async (c) => { | |
| 1122 | const g = await gate(c); | |
| 1123 | if (g instanceof Response) return g; | |
| 1124 | const { user } = g; | |
| 1125 | ||
| 1126 | const results: CheckResult[] = await runAllChecks(c); | |
| 826eccf | 1127 | |
| 1128 | const counts = { | |
| 1129 | green: results.filter((r) => r.status === "green").length, | |
| 1130 | yellow: results.filter((r) => r.status === "yellow").length, | |
| 1131 | red: results.filter((r) => r.status === "red").length, | |
| 1132 | }; | |
| e7b4b7e | 1133 | const total = results.length; |
| 1134 | const overall: CheckStatus = | |
| 1135 | counts.red > 0 ? "red" : counts.yellow > 0 ? "yellow" : "green"; | |
| 1136 | ||
| 1137 | // Headline copy reads as a verdict, not a tally. The gradient swap | |
| 1138 | // (green → yellow → red) makes the page status legible at a glance. | |
| 1139 | const verdict = | |
| 1140 | overall === "red" | |
| 1141 | ? "Issues detected." | |
| 1142 | : overall === "yellow" | |
| 1143 | ? "Degraded." | |
| 1144 | : "Healthy."; | |
| 1145 | const verdictGradClass = | |
| 1146 | overall === "red" | |
| 1147 | ? "health-title-grad is-fail" | |
| 1148 | : overall === "yellow" | |
| 1149 | ? "health-title-grad is-warn" | |
| 1150 | : "health-title-grad"; | |
| 1151 | ||
| 1152 | const summaryPillClass = | |
| 1153 | overall === "red" | |
| 1154 | ? "health-summary-pill is-fail" | |
| 1155 | : overall === "yellow" | |
| 1156 | ? "health-summary-pill is-warn" | |
| 1157 | : "health-summary-pill is-green"; | |
| 1158 | const summaryPillText = `${counts.green} of ${total} checks green`; | |
| 1159 | ||
| 1160 | // Last-checked stamp — server-rendered "asOf" in tabular-nums. The | |
| 1161 | // operator wants a real, observable timestamp on this dashboard. | |
| 1162 | const asOf = new Date(); | |
| 1163 | const asOfDisplay = asOf.toISOString().replace("T", " ").slice(0, 19) + " UTC"; | |
| 826eccf | 1164 | |
| 1165 | const flash = c.req.query("test_email"); | |
| 1166 | ||
| 1167 | return c.html( | |
| f4a1547 | 1168 | <AdminShell active="diagnose" title="Diagnose" user={user}> |
| e7b4b7e | 1169 | <div class="health-wrap"> |
| 1170 | <section class="health-hero"> | |
| 1171 | <div class="health-hero-orb" aria-hidden="true" /> | |
| 1172 | <div class="health-hero-inner"> | |
| 1173 | <div class="health-hero-top"> | |
| 1174 | <div class="health-eyebrow"> | |
| 1175 | <span class="health-eyebrow-dot" aria-hidden="true" /> | |
| 1176 | Platform health · live | |
| 1177 | </div> | |
| 1178 | <a href="/admin" class="health-back"> | |
| 1179 | ← Back to admin | |
| 1180 | </a> | |
| 1181 | </div> | |
| 1182 | <h1 class="health-title"> | |
| 1183 | <span class={verdictGradClass}>{verdict}</span> | |
| 1184 | </h1> | |
| 1185 | <div class="health-summary"> | |
| 1186 | <span class={summaryPillClass}> | |
| 1187 | <span class="dot" aria-hidden="true" /> | |
| 1188 | {summaryPillText} | |
| 1189 | </span> | |
| 1190 | <span class="health-summary-breakdown"> | |
| 1191 | {counts.green} green | |
| 1192 | <span class="sep">·</span> | |
| 1193 | {counts.yellow} warn | |
| 1194 | <span class="sep">·</span> | |
| 1195 | {counts.red} fail | |
| 1196 | </span> | |
| 1197 | <span class="health-summary-stamp" title="Server time"> | |
| 1198 | checked {asOfDisplay} | |
| 1199 | </span> | |
| 1200 | </div> | |
| 1201 | </div> | |
| 1202 | </section> | |
| 826eccf | 1203 | |
| 1204 | {flash && ( | |
| 1205 | <div | |
| e7b4b7e | 1206 | class={ |
| 1207 | "health-banner " + (flash === "ok" ? "is-ok" : "is-error") | |
| 1208 | } | |
| 826eccf | 1209 | > |
| 1210 | {flash === "ok" | |
| 1211 | ? "Test email dispatched. If the provider is 'log' you'll see it in journalctl, not your inbox." | |
| 1212 | : `Test email failed: ${decodeURIComponent(flash)}`} | |
| 1213 | </div> | |
| 1214 | )} | |
| 1215 | ||
| e7b4b7e | 1216 | <div class="health-grid"> |
| 1217 | {results.map((r) => { | |
| 1218 | const cardClass = `health-card is-${r.status}`; | |
| 1219 | const dotClass = `health-card-dot is-${r.status}`; | |
| 1220 | const fix = fixHrefForCheck(r); | |
| 1221 | return ( | |
| 1222 | <article class={cardClass} data-category={r.category}> | |
| 1223 | <div class="health-card-head"> | |
| 1224 | <div class="health-card-id"> | |
| 1225 | <span class={dotClass} aria-hidden="true" /> | |
| 1226 | <div class="health-card-title"> | |
| 1227 | <div class="health-card-category">{r.category}</div> | |
| 1228 | <div class="health-card-name">{r.name}</div> | |
| 1229 | </div> | |
| 1230 | </div> | |
| 1231 | {pill(r.status)} | |
| 826eccf | 1232 | </div> |
| e7b4b7e | 1233 | <p class="health-card-detail">{r.detail}</p> |
| 826eccf | 1234 | {r.fix && ( |
| e7b4b7e | 1235 | <div class="health-card-fix"> |
| 1236 | <span class="health-card-fix-label">How to fix</span> | |
| 1237 | {r.fix} | |
| 826eccf | 1238 | </div> |
| 1239 | )} | |
| e7b4b7e | 1240 | {fix && ( |
| 1241 | <div class="health-card-foot"> | |
| 1242 | <a class="health-card-action" href={fix.href}> | |
| 1243 | {fix.label} | |
| 1244 | <span class="arrow" aria-hidden="true">→</span> | |
| 1245 | </a> | |
| 1246 | </div> | |
| 1247 | )} | |
| 1248 | </article> | |
| 1249 | ); | |
| 1250 | })} | |
| 826eccf | 1251 | </div> |
| 1252 | ||
| c6018a5 | 1253 | <div class="health-test"> |
| 1254 | <h3>AI background tasks</h3> | |
| 1255 | <p> | |
| 1256 | These tasks run continuously inside the autopilot tick — no | |
| 1257 | external scheduler. Each one fires on every signal it cares | |
| 1258 | about (CI failure / gate finding / monitor heartbeat) and | |
| 1259 | degrades gracefully when <code>ANTHROPIC_API_KEY</code> is | |
| 1260 | unset. | |
| 1261 | </p> | |
| 1262 | <ul style="margin: 8px 0 0; padding-left: 20px; line-height: 1.7; font-size: 13.5px;"> | |
| 1263 | <li> | |
| 1264 | <strong>AI CI healer</strong> — on every failed workflow | |
| 1265 | run, Claude reads the failure log + recent diff and proposes | |
| 1266 | targeted file edits. Source: <code>src/lib/ai-ci-healer.ts</code>. | |
| 1267 | </li> | |
| 1268 | <li> | |
| 1269 | <strong>AI patch generator</strong> — when GateTest or | |
| 1270 | advisory scan reports a finding, this generates a concrete | |
| 1271 | diff PR proposing the fix. Source: <code>src/lib/ai-patch-generator.ts</code>. | |
| 1272 | </li> | |
| 1273 | <li> | |
| 1274 | <strong>AI proactive monitor</strong> — sweeps every repo | |
| 1275 | looking for stale TODOs, suspicious patterns, and stuck PRs; | |
| 1276 | files issues automatically. Findings surface in{" "} | |
| 1277 | <a href="/settings/audit">/settings/audit</a>. Source:{" "} | |
| 1278 | <code>src/lib/ai-proactive-monitor.ts</code>. | |
| 1279 | </li> | |
| 1280 | <li> | |
| 1281 | <strong>AI build tasks</strong> — picks up issues labelled | |
| 1282 | <code>ai:build</code> and ships a PR for them. Source:{" "} | |
| 1283 | <code>src/lib/ai-build-tasks.ts</code>. | |
| 1284 | </li> | |
| 1285 | </ul> | |
| 1286 | <p style="margin-top: 12px;"> | |
| 1287 | See <a href="/admin/autopilot">/admin/autopilot</a> for the | |
| 1288 | per-task tick log and force-run controls. | |
| 1289 | </p> | |
| 1290 | </div> | |
| 1291 | ||
| e7b4b7e | 1292 | <div class="health-test"> |
| 1293 | <h3>Test email delivery</h3> | |
| 1294 | <p> | |
| 826eccf | 1295 | Fires a one-line test email to <strong>{user.email}</strong> using |
| 1296 | the configured provider. If EMAIL_PROVIDER=log it appears in | |
| 1297 | journalctl; if resend, in your inbox in <30s. | |
| 1298 | </p> | |
| e7b4b7e | 1299 | <form method="post" action="/admin/diagnose/test-email"> |
| 826eccf | 1300 | <input |
| 1301 | type="hidden" | |
| 1302 | name="_csrf" | |
| 1303 | value={(c.get("csrfToken") as string | undefined) || ""} | |
| 1304 | /> | |
| 1305 | <button type="submit" class="btn btn-sm btn-primary"> | |
| 1306 | Send test email | |
| 1307 | </button> | |
| 1308 | </form> | |
| 1309 | </div> | |
| 1310 | </div> | |
| e7b4b7e | 1311 | <style dangerouslySetInnerHTML={{ __html: healthStyles }} /> |
| f4a1547 | 1312 | </AdminShell> |
| 826eccf | 1313 | ); |
| 1314 | }); | |
| 1315 | ||
| 1316 | diagnose.post("/admin/diagnose/test-email", async (c) => { | |
| 1317 | const g = await gate(c); | |
| 1318 | if (g instanceof Response) return g; | |
| 1319 | const { user } = g; | |
| 1320 | if (!user.email) { | |
| 1321 | return c.redirect( | |
| 1322 | `/admin/diagnose?test_email=${encodeURIComponent("admin has no email on record")}` | |
| 1323 | ); | |
| 1324 | } | |
| 1325 | const stamp = new Date().toISOString(); | |
| 1326 | const result = await sendEmail({ | |
| 1327 | to: user.email, | |
| 1328 | subject: "Gluecron — diagnose test email", | |
| 1329 | text: | |
| 1330 | `This is a test email from /admin/diagnose at ${stamp}.\n\n` + | |
| 1331 | `If you received this in your inbox, EMAIL_PROVIDER=resend is wired correctly.\n` + | |
| 1332 | `If you only see it in journalctl, EMAIL_PROVIDER is still 'log'.\n`, | |
| 1333 | }); | |
| 1334 | if (!result.ok) { | |
| 1335 | return c.redirect( | |
| 1336 | `/admin/diagnose?test_email=${encodeURIComponent(result.error || result.skipped || "unknown failure")}` | |
| 1337 | ); | |
| 1338 | } | |
| 1339 | return c.redirect("/admin/diagnose?test_email=ok"); | |
| 1340 | }); | |
| 1341 | ||
| 1342 | export default diagnose; |