Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
claude/adoring-hopper-5x74bqclaude/affectionate-feynman-ykrf1hclaude/architecture-audit-design-wxprenclaude/build-status-update-3MXsfclaude/charming-meitner-mllb5rclaude/compare-gate-gluecron-s4mFQclaude/confident-faraday-tikcwbclaude/continue-work-XMTlIclaude/crontech-gluecron-deploy-7MIECclaude/crontech-platform-setup-SeKfwclaude/design-2026claude/ecstatic-ptolemy-jMdigclaude/enhance-github-integration-QNHdGclaude/fix-aa-loop-issue-PonMQclaude/fix-actions-and-processclaude/fix-desktop-errors-XqoW8claude/fix-red-workflowsclaude/fix-website-access-6FKJNclaude/gatetest-integration-hardeningclaude/github-audit-improvements-bDFr9claude/gluecron-launch-status-FoMRlclaude/hopeful-lamport-olfCTclaude/issue-to-pr-and-protectionsclaude/jolly-heisenberg-2sg1Qclaude/launch-preparation-QmTb6claude/new-session-xk1l7claude/plan-platform-architecture-kkN4yclaude/platform-analysis-roadmap-1nUGLclaude/platform-launch-assessment-8dWV8claude/polish-platform-release-AeDrUclaude/resume-previous-work-KzyLwclaude/review-crontech-handoff-qYEVqclaude/review-project-completeness-lHhS2claude/review-readme-docs-ulqPKclaude/serene-edison-rj87weclaude/setup-multi-repo-dev-BCwNQclaude/ship-fixes-and-tests-Jvz1cclaude/site-audit-competitive-pctlwgclaude/site-migration-vercel-XstpKclaude/standalone-product-repos-XHFTDcopilot/feat-smart-empty-states-keyboard-first-enhancementcopilot/feat-smart-morning-digest-review-context-restorecopilot/fix-and-process-workflowscopilot/update-ai-powered-code-reviewfeat/debt-mapfeat/push-policy-codeowners-hardeningfeat/smart-digest-contextfeat/stage-impactfeat/t1-secret-migrationfeat/u-polishfeat/w-self-hostfeat/w2-claude-configfix/agent-journey-orphan-sweepgatetest/auto-fix-1776586424172gatetest/auto-fix-1776586534814gatetest/auto-fix-1776590685143gatetest/auto-fix-1776590808199mainops/redeploy-retriggerstyle/dxt-cta-themeworktree-agent-a3377aad30d55da26worktree-agent-a7ef607b7ee1d6c74
synthetic-monitor.ts9.7 KB · 312 lines
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
/**
 * BLOCK S4 — Synthetic monitor.
 *
 * Runs a small URL-only smoke suite against the live site on every
 * autopilot tick (5 min for v1; 60 s goal tracked as a follow-up). Each
 * check has a 5 s timeout. Results are persisted to `synthetic_checks`
 * and published on the SSE topic `monitor:synthetic` so the /admin/status
 * dashboard can light up red without a page reload.
 *
 * The check list is URL-only on purpose — this module must never depend
 * on the DB, the AI client, or anything else that the autopilot tick
 * itself owns. If the site is on fire, this is the layer that tells us.
 */
import { desc, sql } from "drizzle-orm";
import { db } from "../db";
import { syntheticChecks } from "../db/schema";
import { config } from "./config";
import { publish } from "./sse";

export type SyntheticCheckStatus = "green" | "red" | "yellow";

export interface SyntheticCheckResult {
  name: string;
  status: SyntheticCheckStatus;
  statusCode?: number;
  durationMs: number;
  error?: string;
}

export interface SyntheticCheckSpec {
  name: string;
  /** Relative path; prepended with APP_BASE_URL or `opts.baseUrl`. */
  url: string;
  /** Acceptable HTTP status(es). Defaults to 200. */
  expectStatus?: number | number[];
  /**
   * If set, response body must parse as JSON and contain this top-level
   * key (key presence — value can be any non-undefined). Implies a JSON
   * Accept header.
   */
  expectKeyInJson?: string;
  /** If set, body string must contain this substring (case-sensitive). */
  expectContains?: string;
  /** Per-check timeout in ms. Defaults to DEFAULT_TIMEOUT_MS. */
  timeoutMs?: number;
}

/** Topic published to whenever a check completes. */
export const SSE_TOPIC = "monitor:synthetic";

/** Default per-check timeout. */
export const DEFAULT_TIMEOUT_MS = 5000;

/**
 * The S4 check list — URL-only, mirrors the S1+S3 smoke suite minus the
 * migration check (which would have to talk to the DB).
 */
export const SYNTHETIC_CHECKS: ReadonlyArray<SyntheticCheckSpec> = [
  { name: "healthz", url: "/healthz", expectKeyInJson: "ok" },
  { name: "readyz", url: "/readyz" },
  { name: "version", url: "/api/version", expectKeyInJson: "sha" },
  { name: "login", url: "/login", expectContains: "Sign in" },
  { name: "register", url: "/register", expectContains: "Create account" },
  { name: "landing", url: "/" },
  { name: "explore", url: "/explore" },
  { name: "pricing", url: "/pricing" },
  { name: "status", url: "/status" },
  { name: "manifest", url: "/manifest.webmanifest" },
  { name: "sw.js", url: "/sw.js" },
  { name: "gluecron.dxt", url: "/gluecron.dxt" },
  { name: "mcp discovery", url: "/mcp", expectKeyInJson: "serverInfo" },
  { name: "robots.txt", url: "/robots.txt" },
];

function statusMatches(
  expect: number | number[] | undefined,
  actual: number
): boolean {
  if (expect === undefined) return actual === 200;
  if (Array.isArray(expect)) return expect.includes(actual);
  return expect === actual;
}

async function runOneCheck(
  spec: SyntheticCheckSpec,
  baseUrl: string,
  fetchImpl: typeof fetch
): Promise<SyntheticCheckResult> {
  const timeoutMs = spec.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  const fullUrl = baseUrl.replace(/\/+$/, "") + spec.url;
  const t0 = Date.now();

  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);

  try {
    const headers: Record<string, string> = {
      "User-Agent": "gluecron-synthetic-monitor/1",
    };
    if (spec.expectKeyInJson) headers["Accept"] = "application/json";

    const res = await fetchImpl(fullUrl, {
      method: "GET",
      headers,
      signal: controller.signal,
      redirect: "manual",
    });
    const durationMs = Date.now() - t0;

    if (!statusMatches(spec.expectStatus, res.status)) {
      return {
        name: spec.name,
        status: "red",
        statusCode: res.status,
        durationMs,
        error: `expected status ${JSON.stringify(spec.expectStatus ?? 200)}, got ${res.status}`,
      };
    }

    if (spec.expectKeyInJson || spec.expectContains) {
      const body = await res.text();
      if (spec.expectKeyInJson) {
        let parsed: any;
        try {
          parsed = JSON.parse(body);
        } catch {
          return {
            name: spec.name,
            status: "red",
            statusCode: res.status,
            durationMs,
            error: `expected JSON body with key "${spec.expectKeyInJson}", got non-JSON`,
          };
        }
        if (
          !parsed ||
          typeof parsed !== "object" ||
          !(spec.expectKeyInJson in parsed)
        ) {
          return {
            name: spec.name,
            status: "red",
            statusCode: res.status,
            durationMs,
            error: `expected key "${spec.expectKeyInJson}" in JSON response`,
          };
        }
      }
      if (spec.expectContains && !body.includes(spec.expectContains)) {
        return {
          name: spec.name,
          status: "red",
          statusCode: res.status,
          durationMs,
          error: `expected body to contain "${spec.expectContains}"`,
        };
      }
    }

    return {
      name: spec.name,
      status: "green",
      statusCode: res.status,
      durationMs,
    };
  } catch (err) {
    const durationMs = Date.now() - t0;
    const message =
      err instanceof Error
        ? err.name === "AbortError"
          ? `timeout after ${timeoutMs}ms`
          : err.message
        : String(err ?? "unknown error");
    return {
      name: spec.name,
      status: "red",
      durationMs,
      error: message,
    };
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Run every check in SYNTHETIC_CHECKS in parallel and return one result
 * per check. Each individual check is wrapped in try/catch and obeys its
 * own timeout — a slow check cannot wedge the others.
 *
 * Pure-ish: takes a fetch implementation + base URL so unit tests can
 * inject fakes without touching the network.
 */
export async function runSyntheticChecks(opts?: {
  baseUrl?: string;
  fetchImpl?: typeof fetch;
  checks?: ReadonlyArray<SyntheticCheckSpec>;
}): Promise<SyntheticCheckResult[]> {
  const baseUrl = opts?.baseUrl ?? config.appBaseUrl;
  const fetchImpl = opts?.fetchImpl ?? fetch;
  const checks = opts?.checks ?? SYNTHETIC_CHECKS;
  return Promise.all(checks.map((c) => runOneCheck(c, baseUrl, fetchImpl)));
}

/**
 * Persist a batch of results into `synthetic_checks` and publish each
 * one onto the SSE topic. Never throws — a DB hiccup must not kill the
 * autopilot tick.
 */
export async function persistChecks(
  results: SyntheticCheckResult[]
): Promise<void> {
  if (results.length === 0) return;
  try {
    await db.insert(syntheticChecks).values(
      results.map((r) => ({
        checkName: r.name,
        status: r.status,
        statusCode: r.statusCode ?? null,
        durationMs: r.durationMs,
        error: r.error ?? null,
      }))
    );
  } catch (err) {
    console.error("[synthetic-monitor] persist failed:", err);
  }
  for (const r of results) {
    try {
      publish(SSE_TOPIC, { event: "check", data: r });
    } catch {
      // sse.publish already swallows; belt-and-braces.
    }
  }
}

/**
 * Return the most-recent recorded result per check_name. Used by the
 * /admin/status renderer + the auto-merge transition detector. Returns
 * an empty object on any DB error.
 */
export async function latestStatusByCheck(): Promise<
  Record<string, SyntheticCheckResult & { checkedAt: Date }>
> {
  try {
    // DISTINCT ON (check_name) — Postgres pattern: ORDER BY check_name,
    // checked_at DESC and keep the first row per name.
    const rows = await db.execute(sql`
      SELECT DISTINCT ON (check_name)
        check_name, status, status_code, duration_ms, error, checked_at
      FROM synthetic_checks
      ORDER BY check_name ASC, checked_at DESC
    `);
    const out: Record<string, SyntheticCheckResult & { checkedAt: Date }> = {};
    // Drizzle's `db.execute(sql\`...\`)` returns an iterable of records.
    const list: any[] = Array.isArray(rows)
      ? (rows as any[])
      : (rows as any).rows ?? [];
    for (const r of list) {
      const name = String(r.check_name ?? r.checkName ?? "");
      if (!name) continue;
      out[name] = {
        name,
        status: (r.status as SyntheticCheckStatus) ?? "red",
        statusCode:
          r.status_code === null || r.status_code === undefined
            ? undefined
            : Number(r.status_code),
        durationMs: Number(r.duration_ms ?? r.durationMs ?? 0),
        error: r.error ?? undefined,
        checkedAt: new Date(r.checked_at ?? r.checkedAt ?? Date.now()),
      };
    }
    return out;
  } catch (err) {
    console.error("[synthetic-monitor] latestStatusByCheck failed:", err);
    return {};
  }
}

/**
 * Return red rows from the last `hours` hours, newest-first. Used by
 * `/status` "Recent incidents" + the /admin/status detail page.
 */
export async function recentRedChecks(
  hours: number = 24,
  limit: number = 50
): Promise<Array<SyntheticCheckResult & { checkedAt: Date }>> {
  try {
    const rows = await db
      .select()
      .from(syntheticChecks)
      .where(sql`${syntheticChecks.status} = 'red' AND ${syntheticChecks.checkedAt} > now() - (${hours} || ' hours')::interval`)
      .orderBy(desc(syntheticChecks.checkedAt))
      .limit(limit);
    return rows.map((r) => ({
      name: r.checkName,
      status: r.status as SyntheticCheckStatus,
      statusCode: r.statusCode ?? undefined,
      durationMs: r.durationMs,
      error: r.error ?? undefined,
      checkedAt: r.checkedAt,
    }));
  } catch (err) {
    console.error("[synthetic-monitor] recentRedChecks failed:", err);
    return [];
  }
}

/** Exported for tests + the autopilot transition detector. */
export const __test = {
  runOneCheck,
  statusMatches,
};