Blame · Line-by-line history
synthetic-monitor.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| b1be050 | 1 | /** |
| 2 | * BLOCK S4 — Synthetic monitor. | |
| 3 | * | |
| 4 | * Runs a small URL-only smoke suite against the live site on every | |
| 5 | * autopilot tick (5 min for v1; 60 s goal tracked as a follow-up). Each | |
| 6 | * check has a 5 s timeout. Results are persisted to `synthetic_checks` | |
| 7 | * and published on the SSE topic `monitor:synthetic` so the /admin/status | |
| 8 | * dashboard can light up red without a page reload. | |
| 9 | * | |
| 10 | * The check list is URL-only on purpose — this module must never depend | |
| 11 | * on the DB, the AI client, or anything else that the autopilot tick | |
| 12 | * itself owns. If the site is on fire, this is the layer that tells us. | |
| 13 | */ | |
| 14 | import { desc, sql } from "drizzle-orm"; | |
| 15 | import { db } from "../db"; | |
| 16 | import { syntheticChecks } from "../db/schema"; | |
| 17 | import { config } from "./config"; | |
| 18 | import { publish } from "./sse"; | |
| 19 | ||
| 20 | export type SyntheticCheckStatus = "green" | "red" | "yellow"; | |
| 21 | ||
| 22 | export interface SyntheticCheckResult { | |
| 23 | name: string; | |
| 24 | status: SyntheticCheckStatus; | |
| 25 | statusCode?: number; | |
| 26 | durationMs: number; | |
| 27 | error?: string; | |
| 28 | } | |
| 29 | ||
| 30 | export interface SyntheticCheckSpec { | |
| 31 | name: string; | |
| 32 | /** Relative path; prepended with APP_BASE_URL or `opts.baseUrl`. */ | |
| 33 | url: string; | |
| 34 | /** Acceptable HTTP status(es). Defaults to 200. */ | |
| 35 | expectStatus?: number | number[]; | |
| 36 | /** | |
| 37 | * If set, response body must parse as JSON and contain this top-level | |
| 38 | * key (key presence — value can be any non-undefined). Implies a JSON | |
| 39 | * Accept header. | |
| 40 | */ | |
| 41 | expectKeyInJson?: string; | |
| 42 | /** If set, body string must contain this substring (case-sensitive). */ | |
| 43 | expectContains?: string; | |
| 44 | /** Per-check timeout in ms. Defaults to DEFAULT_TIMEOUT_MS. */ | |
| 45 | timeoutMs?: number; | |
| 46 | } | |
| 47 | ||
| 48 | /** Topic published to whenever a check completes. */ | |
| 49 | export const SSE_TOPIC = "monitor:synthetic"; | |
| 50 | ||
| 51 | /** Default per-check timeout. */ | |
| 52 | export const DEFAULT_TIMEOUT_MS = 5000; | |
| 53 | ||
| 54 | /** | |
| 55 | * The S4 check list — URL-only, mirrors the S1+S3 smoke suite minus the | |
| 56 | * migration check (which would have to talk to the DB). | |
| 57 | */ | |
| 58 | export const SYNTHETIC_CHECKS: ReadonlyArray<SyntheticCheckSpec> = [ | |
| 59 | { name: "healthz", url: "/healthz", expectKeyInJson: "ok" }, | |
| 60 | { name: "readyz", url: "/readyz" }, | |
| 61 | { name: "version", url: "/api/version", expectKeyInJson: "sha" }, | |
| 62 | { name: "login", url: "/login", expectContains: "Sign in" }, | |
| 63 | { name: "register", url: "/register", expectContains: "Create account" }, | |
| 64 | { name: "landing", url: "/" }, | |
| 65 | { name: "explore", url: "/explore" }, | |
| 66 | { name: "pricing", url: "/pricing" }, | |
| 67 | { name: "status", url: "/status" }, | |
| 68 | { name: "manifest", url: "/manifest.webmanifest" }, | |
| 69 | { name: "sw.js", url: "/sw.js" }, | |
| 70 | { name: "gluecron.dxt", url: "/gluecron.dxt" }, | |
| 71 | { name: "mcp discovery", url: "/mcp", expectKeyInJson: "serverInfo" }, | |
| 72 | { name: "robots.txt", url: "/robots.txt" }, | |
| 73 | ]; | |
| 74 | ||
| 75 | function statusMatches( | |
| 76 | expect: number | number[] | undefined, | |
| 77 | actual: number | |
| 78 | ): boolean { | |
| 79 | if (expect === undefined) return actual === 200; | |
| 80 | if (Array.isArray(expect)) return expect.includes(actual); | |
| 81 | return expect === actual; | |
| 82 | } | |
| 83 | ||
| 84 | async function runOneCheck( | |
| 85 | spec: SyntheticCheckSpec, | |
| 86 | baseUrl: string, | |
| 87 | fetchImpl: typeof fetch | |
| 88 | ): Promise<SyntheticCheckResult> { | |
| 89 | const timeoutMs = spec.timeoutMs ?? DEFAULT_TIMEOUT_MS; | |
| 90 | const fullUrl = baseUrl.replace(/\/+$/, "") + spec.url; | |
| 91 | const t0 = Date.now(); | |
| 92 | ||
| 93 | const controller = new AbortController(); | |
| 94 | const timer = setTimeout(() => controller.abort(), timeoutMs); | |
| 95 | ||
| 96 | try { | |
| 97 | const headers: Record<string, string> = { | |
| 98 | "User-Agent": "gluecron-synthetic-monitor/1", | |
| 99 | }; | |
| 100 | if (spec.expectKeyInJson) headers["Accept"] = "application/json"; | |
| 101 | ||
| 102 | const res = await fetchImpl(fullUrl, { | |
| 103 | method: "GET", | |
| 104 | headers, | |
| 105 | signal: controller.signal, | |
| 106 | redirect: "manual", | |
| 107 | }); | |
| 108 | const durationMs = Date.now() - t0; | |
| 109 | ||
| 110 | if (!statusMatches(spec.expectStatus, res.status)) { | |
| 111 | return { | |
| 112 | name: spec.name, | |
| 113 | status: "red", | |
| 114 | statusCode: res.status, | |
| 115 | durationMs, | |
| 116 | error: `expected status ${JSON.stringify(spec.expectStatus ?? 200)}, got ${res.status}`, | |
| 117 | }; | |
| 118 | } | |
| 119 | ||
| 120 | if (spec.expectKeyInJson || spec.expectContains) { | |
| 121 | const body = await res.text(); | |
| 122 | if (spec.expectKeyInJson) { | |
| 123 | let parsed: any; | |
| 124 | try { | |
| 125 | parsed = JSON.parse(body); | |
| 126 | } catch { | |
| 127 | return { | |
| 128 | name: spec.name, | |
| 129 | status: "red", | |
| 130 | statusCode: res.status, | |
| 131 | durationMs, | |
| 132 | error: `expected JSON body with key "${spec.expectKeyInJson}", got non-JSON`, | |
| 133 | }; | |
| 134 | } | |
| 135 | if ( | |
| 136 | !parsed || | |
| 137 | typeof parsed !== "object" || | |
| 138 | !(spec.expectKeyInJson in parsed) | |
| 139 | ) { | |
| 140 | return { | |
| 141 | name: spec.name, | |
| 142 | status: "red", | |
| 143 | statusCode: res.status, | |
| 144 | durationMs, | |
| 145 | error: `expected key "${spec.expectKeyInJson}" in JSON response`, | |
| 146 | }; | |
| 147 | } | |
| 148 | } | |
| 149 | if (spec.expectContains && !body.includes(spec.expectContains)) { | |
| 150 | return { | |
| 151 | name: spec.name, | |
| 152 | status: "red", | |
| 153 | statusCode: res.status, | |
| 154 | durationMs, | |
| 155 | error: `expected body to contain "${spec.expectContains}"`, | |
| 156 | }; | |
| 157 | } | |
| 158 | } | |
| 159 | ||
| 160 | return { | |
| 161 | name: spec.name, | |
| 162 | status: "green", | |
| 163 | statusCode: res.status, | |
| 164 | durationMs, | |
| 165 | }; | |
| 166 | } catch (err) { | |
| 167 | const durationMs = Date.now() - t0; | |
| 168 | const message = | |
| 169 | err instanceof Error | |
| 170 | ? err.name === "AbortError" | |
| 171 | ? `timeout after ${timeoutMs}ms` | |
| 172 | : err.message | |
| 173 | : String(err ?? "unknown error"); | |
| 174 | return { | |
| 175 | name: spec.name, | |
| 176 | status: "red", | |
| 177 | durationMs, | |
| 178 | error: message, | |
| 179 | }; | |
| 180 | } finally { | |
| 181 | clearTimeout(timer); | |
| 182 | } | |
| 183 | } | |
| 184 | ||
| 185 | /** | |
| 186 | * Run every check in SYNTHETIC_CHECKS in parallel and return one result | |
| 187 | * per check. Each individual check is wrapped in try/catch and obeys its | |
| 188 | * own timeout — a slow check cannot wedge the others. | |
| 189 | * | |
| 190 | * Pure-ish: takes a fetch implementation + base URL so unit tests can | |
| 191 | * inject fakes without touching the network. | |
| 192 | */ | |
| 193 | export async function runSyntheticChecks(opts?: { | |
| 194 | baseUrl?: string; | |
| 195 | fetchImpl?: typeof fetch; | |
| 196 | checks?: ReadonlyArray<SyntheticCheckSpec>; | |
| 197 | }): Promise<SyntheticCheckResult[]> { | |
| 198 | const baseUrl = opts?.baseUrl ?? config.appBaseUrl; | |
| 199 | const fetchImpl = opts?.fetchImpl ?? fetch; | |
| 200 | const checks = opts?.checks ?? SYNTHETIC_CHECKS; | |
| 201 | return Promise.all(checks.map((c) => runOneCheck(c, baseUrl, fetchImpl))); | |
| 202 | } | |
| 203 | ||
| 204 | /** | |
| 205 | * Persist a batch of results into `synthetic_checks` and publish each | |
| 206 | * one onto the SSE topic. Never throws — a DB hiccup must not kill the | |
| 207 | * autopilot tick. | |
| 208 | */ | |
| 209 | export async function persistChecks( | |
| 210 | results: SyntheticCheckResult[] | |
| 211 | ): Promise<void> { | |
| 212 | if (results.length === 0) return; | |
| 213 | try { | |
| 214 | await db.insert(syntheticChecks).values( | |
| 215 | results.map((r) => ({ | |
| 216 | checkName: r.name, | |
| 217 | status: r.status, | |
| 218 | statusCode: r.statusCode ?? null, | |
| 219 | durationMs: r.durationMs, | |
| 220 | error: r.error ?? null, | |
| 221 | })) | |
| 222 | ); | |
| 223 | } catch (err) { | |
| 224 | console.error("[synthetic-monitor] persist failed:", err); | |
| 225 | } | |
| 226 | for (const r of results) { | |
| 227 | try { | |
| 228 | publish(SSE_TOPIC, { event: "check", data: r }); | |
| 229 | } catch { | |
| 230 | // sse.publish already swallows; belt-and-braces. | |
| 231 | } | |
| 232 | } | |
| 233 | } | |
| 234 | ||
| 235 | /** | |
| 236 | * Return the most-recent recorded result per check_name. Used by the | |
| 237 | * /admin/status renderer + the auto-merge transition detector. Returns | |
| 238 | * an empty object on any DB error. | |
| 239 | */ | |
| 240 | export async function latestStatusByCheck(): Promise< | |
| 241 | Record<string, SyntheticCheckResult & { checkedAt: Date }> | |
| 242 | > { | |
| 243 | try { | |
| 244 | // DISTINCT ON (check_name) — Postgres pattern: ORDER BY check_name, | |
| 245 | // checked_at DESC and keep the first row per name. | |
| 246 | const rows = await db.execute(sql` | |
| 247 | SELECT DISTINCT ON (check_name) | |
| 248 | check_name, status, status_code, duration_ms, error, checked_at | |
| 249 | FROM synthetic_checks | |
| 250 | ORDER BY check_name ASC, checked_at DESC | |
| 251 | `); | |
| 252 | const out: Record<string, SyntheticCheckResult & { checkedAt: Date }> = {}; | |
| 253 | // Drizzle's `db.execute(sql\`...\`)` returns an iterable of records. | |
| 254 | const list: any[] = Array.isArray(rows) | |
| 255 | ? (rows as any[]) | |
| 256 | : (rows as any).rows ?? []; | |
| 257 | for (const r of list) { | |
| 258 | const name = String(r.check_name ?? r.checkName ?? ""); | |
| 259 | if (!name) continue; | |
| 260 | out[name] = { | |
| 261 | name, | |
| 262 | status: (r.status as SyntheticCheckStatus) ?? "red", | |
| 263 | statusCode: | |
| 264 | r.status_code === null || r.status_code === undefined | |
| 265 | ? undefined | |
| 266 | : Number(r.status_code), | |
| 267 | durationMs: Number(r.duration_ms ?? r.durationMs ?? 0), | |
| 268 | error: r.error ?? undefined, | |
| 269 | checkedAt: new Date(r.checked_at ?? r.checkedAt ?? Date.now()), | |
| 270 | }; | |
| 271 | } | |
| 272 | return out; | |
| 273 | } catch (err) { | |
| 274 | console.error("[synthetic-monitor] latestStatusByCheck failed:", err); | |
| 275 | return {}; | |
| 276 | } | |
| 277 | } | |
| 278 | ||
| 279 | /** | |
| 280 | * Return red rows from the last `hours` hours, newest-first. Used by | |
| 281 | * `/status` "Recent incidents" + the /admin/status detail page. | |
| 282 | */ | |
| 283 | export async function recentRedChecks( | |
| 284 | hours: number = 24, | |
| 285 | limit: number = 50 | |
| 286 | ): Promise<Array<SyntheticCheckResult & { checkedAt: Date }>> { | |
| 287 | try { | |
| 288 | const rows = await db | |
| 289 | .select() | |
| 290 | .from(syntheticChecks) | |
| 291 | .where(sql`${syntheticChecks.status} = 'red' AND ${syntheticChecks.checkedAt} > now() - (${hours} || ' hours')::interval`) | |
| 292 | .orderBy(desc(syntheticChecks.checkedAt)) | |
| 293 | .limit(limit); | |
| 294 | return rows.map((r) => ({ | |
| 295 | name: r.checkName, | |
| 296 | status: r.status as SyntheticCheckStatus, | |
| 297 | statusCode: r.statusCode ?? undefined, | |
| 298 | durationMs: r.durationMs, | |
| 299 | error: r.error ?? undefined, | |
| 300 | checkedAt: r.checkedAt, | |
| 301 | })); | |
| 302 | } catch (err) { | |
| 303 | console.error("[synthetic-monitor] recentRedChecks failed:", err); | |
| 304 | return []; | |
| 305 | } | |
| 306 | } | |
| 307 | ||
| 308 | /** Exported for tests + the autopilot transition detector. */ | |
| 309 | export const __test = { | |
| 310 | runOneCheck, | |
| 311 | statusMatches, | |
| 312 | }; |