Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
CodeIssuesPull RequestsActionsSecurityInsightsSettings
✨ AI
More
Blame · Line-by-line history

ai-proactive-monitor.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-proactive-monitor.tsBlame650 lines · 1 contributor
e9aa4d8Claude1/**
2 * AI Proactive Monitor — hourly platform-health surveillance.
3 *
4 * Pulls the last 24 hours of platform telemetry (audit log, platform
5 * deploys, workflow runs) and asks Claude to spot anomalies — degraded
6 * deploy times, recurring failures, suspicious audit patterns, etc.
7 *
8 * For each finding above `severity=info` it opens an issue on the
9 * platform self-host repo (`SELF_HOST_REPO` env, defaults to
10 * `ccantynz-alt/Gluecron.com`) tagged with the `ai:proactive-finding`
11 * label. A deterministic dedupe key (sha256 of the title) is embedded
12 * in the issue body as an HTML marker so the same alert never fires
13 * twice in a 24h window — even across process restarts.
14 *
15 * Every finding is also written to `audit_log` under action
16 * `ai.proactive.finding` so operators have a queryable trail of what
17 * Claude noticed and when.
18 *
19 * Hooks into autopilot via `aiProactiveMonitorTick()` — wrapped in
20 * try/catch in the registration so a single failure cannot wedge the
21 * surrounding tick. Skips cleanly when:
22 * - `ANTHROPIC_API_KEY` is unset (graceful no-op).
23 * - `AUTOPILOT_DISABLED=1` (the surrounding loop never invokes us).
24 *
25 * Same dependency-injection seam pattern as `runAutoMergeSweep` /
26 * `runAiBuildTaskOnce` — every DB / Claude / clock interaction is
27 * overridable so tests don't need the DB or the AI client.
28 */
29
30import { createHash } from "crypto";
31import { and, desc, eq, gte, like } from "drizzle-orm";
32import { db } from "../db";
33import {
34 auditLog,
35 issueLabels,
36 issues,
37 labels,
38 repositories,
39 users,
40} from "../db/schema";
41import { platformDeploys } from "../db/schema-deploys";
42import { workflowRuns } from "../db/schema";
43import {
44 MODEL_SONNET,
45 extractText,
46 getAnthropic,
47 isAiAvailable,
48 parseJsonResponse,
49} from "./ai-client";
50import { audit } from "./notify";
51
52/** Default self-host repo when SELF_HOST_REPO is not configured. */
53export const DEFAULT_SELF_HOST_REPO = "ccantynz-alt/Gluecron.com";
54
55/** Label attached to every issue opened by this monitor. */
56export const PROACTIVE_LABEL_NAME = "ai:proactive-finding";
57
58/** Stable marker embedded in issue bodies for dedupe lookups. */
59export const PROACTIVE_DEDUPE_MARKER_PREFIX =
60 "<!-- gluecron:ai-proactive:dedupe=";
61export const PROACTIVE_DEDUPE_MARKER_SUFFIX = " -->";
62
63/** Lookback window we feed Claude — and also the dedupe horizon. */
64export const PROACTIVE_LOOKBACK_HOURS = 24;
65
66/** Hard cap on rows pulled per table so the prompt stays bounded. */
67const MAX_AUDIT_ROWS = 200;
68const MAX_DEPLOY_ROWS = 50;
69const MAX_WORKFLOW_RUN_ROWS = 200;
70
71/** Hard cap on issues we'll open in a single tick (runaway protection). */
72const MAX_FINDINGS_PER_TICK = 5;
73
74export type ProactiveSeverity = "info" | "warning" | "critical";
75
76export interface ProactiveFinding {
77 title: string;
78 severity: ProactiveSeverity;
79 body_markdown: string;
80 target_url?: string | null;
81}
82
83export interface ProactiveTelemetry {
84 auditLog: Array<{
85 action: string;
86 targetType: string | null;
87 targetId: string | null;
88 userId: string | null;
89 repositoryId: string | null;
90 createdAt: Date;
91 }>;
92 platformDeploys: Array<{
93 runId: string;
94 sha: string;
95 status: string;
96 durationMs: number | null;
97 error: string | null;
98 startedAt: Date;
99 finishedAt: Date | null;
100 }>;
101 workflowRuns: Array<{
102 id: string;
103 status: string;
104 conclusion: string | null;
105 event: string;
106 queuedAt: Date;
107 startedAt: Date | null;
108 finishedAt: Date | null;
109 }>;
110}
111
112export interface ProactiveMonitorDeps {
113 /** Override telemetry loader (DI for tests). */
114 loadTelemetry?: (lookbackHours: number) => Promise<ProactiveTelemetry>;
115 /** Override Claude call — returns the parsed findings array. */
116 askClaude?: (
117 telemetry: ProactiveTelemetry
118 ) => Promise<ProactiveFinding[]>;
119 /** Override the self-host repo resolver (returns null when missing). */
120 resolveSelfHostRepo?: () => Promise<{
121 repositoryId: string;
122 ownerId: string;
123 } | null>;
124 /** Override dedupe lookup. Returns true if a finding with this key already exists in the lookback window. */
125 isDuplicate?: (
126 repositoryId: string,
127 dedupeKey: string,
128 lookbackHours: number
129 ) => Promise<boolean>;
130 /** Override the issue + label writer. Returns the new issue number. */
131 createFindingIssue?: (args: {
132 repositoryId: string;
133 authorId: string;
134 title: string;
135 body: string;
136 }) => Promise<number | null>;
137 /** Override the audit writer (DI for tests). */
138 recordAudit?: (
139 finding: ProactiveFinding,
140 repositoryId: string | null,
141 issueNumber: number | null,
142 dedupeKey: string
143 ) => Promise<void>;
144 /** Override clock for deterministic windows in tests. */
145 now?: () => Date;
146 /** Override AI-key check (lets tests run the full pipeline). */
147 aiAvailable?: () => boolean;
148 /** Override the per-tick cap. */
149 maxFindings?: number;
150}
151
152export interface ProactiveMonitorSummary {
153 considered: number;
154 opened: number;
155 skippedDedupe: number;
156 skippedSeverity: number;
157 errors: number;
158}
159
160/**
161 * sha256 of the title, used as the deterministic dedupe key. Same
162 * title => same key, so we can detect "already filed this in the last
163 * 24h" with a single LIKE query on issue body.
164 */
165export function dedupeKeyForTitle(title: string): string {
166 return createHash("sha256")
167 .update(title.trim().toLowerCase())
168 .digest("hex")
169 .slice(0, 32);
170}
171
172function dedupeMarker(key: string): string {
173 return `${PROACTIVE_DEDUPE_MARKER_PREFIX}${key}${PROACTIVE_DEDUPE_MARKER_SUFFIX}`;
174}
175
176/** Compact telemetry summary embedded in the prompt — keeps tokens bounded. */
177function summariseTelemetryForPrompt(t: ProactiveTelemetry): string {
178 const auditByAction = new Map<string, number>();
179 for (const row of t.auditLog) {
180 auditByAction.set(row.action, (auditByAction.get(row.action) || 0) + 1);
181 }
182 const auditLines = Array.from(auditByAction.entries())
183 .sort((a, b) => b[1] - a[1])
184 .slice(0, 30)
185 .map(([action, count]) => `- ${action}: ${count}`)
186 .join("\n");
187
188 const deployLines = t.platformDeploys
189 .slice(0, 30)
190 .map((d) => {
191 const dur = d.durationMs !== null ? `${d.durationMs}ms` : "n/a";
192 const err = d.error ? ` error="${d.error.slice(0, 120)}"` : "";
193 return `- run=${d.runId} sha=${d.sha.slice(0, 7)} status=${d.status} dur=${dur}${err}`;
194 })
195 .join("\n");
196
197 const wfStatusCounts = new Map<string, number>();
198 let durSum = 0;
199 let durCount = 0;
200 for (const r of t.workflowRuns) {
201 const key = `${r.status}/${r.conclusion ?? "n/a"}`;
202 wfStatusCounts.set(key, (wfStatusCounts.get(key) || 0) + 1);
203 if (r.startedAt && r.finishedAt) {
204 durSum += r.finishedAt.getTime() - r.startedAt.getTime();
205 durCount += 1;
206 }
207 }
208 const wfLines = Array.from(wfStatusCounts.entries())
209 .sort((a, b) => b[1] - a[1])
210 .map(([k, v]) => `- ${k}: ${v}`)
211 .join("\n");
212 const avgWfDuration =
213 durCount > 0 ? `${Math.round(durSum / durCount)}ms (n=${durCount})` : "n/a";
214
215 return [
216 `## Audit log (${t.auditLog.length} rows, top actions)`,
217 auditLines || "(none)",
218 "",
219 `## Platform deploys (${t.platformDeploys.length} rows, most recent first)`,
220 deployLines || "(none)",
221 "",
222 `## Workflow runs (${t.workflowRuns.length} rows, avg duration ${avgWfDuration})`,
223 wfLines || "(none)",
224 ].join("\n");
225}
226
227async function defaultLoadTelemetry(
228 lookbackHours: number
229): Promise<ProactiveTelemetry> {
230 const cutoff = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
231 const empty: ProactiveTelemetry = {
232 auditLog: [],
233 platformDeploys: [],
234 workflowRuns: [],
235 };
236 try {
237 const [audits, deploys, runs] = await Promise.all([
238 db
239 .select({
240 action: auditLog.action,
241 targetType: auditLog.targetType,
242 targetId: auditLog.targetId,
243 userId: auditLog.userId,
244 repositoryId: auditLog.repositoryId,
245 createdAt: auditLog.createdAt,
246 })
247 .from(auditLog)
248 .where(gte(auditLog.createdAt, cutoff))
249 .orderBy(desc(auditLog.createdAt))
250 .limit(MAX_AUDIT_ROWS)
251 .catch(() => [] as ProactiveTelemetry["auditLog"]),
252 db
253 .select({
254 runId: platformDeploys.runId,
255 sha: platformDeploys.sha,
256 status: platformDeploys.status,
257 durationMs: platformDeploys.durationMs,
258 error: platformDeploys.error,
259 startedAt: platformDeploys.startedAt,
260 finishedAt: platformDeploys.finishedAt,
261 })
262 .from(platformDeploys)
263 .where(gte(platformDeploys.startedAt, cutoff))
264 .orderBy(desc(platformDeploys.startedAt))
265 .limit(MAX_DEPLOY_ROWS)
266 .catch(() => [] as ProactiveTelemetry["platformDeploys"]),
267 db
268 .select({
269 id: workflowRuns.id,
270 status: workflowRuns.status,
271 conclusion: workflowRuns.conclusion,
272 event: workflowRuns.event,
273 queuedAt: workflowRuns.queuedAt,
274 startedAt: workflowRuns.startedAt,
275 finishedAt: workflowRuns.finishedAt,
276 })
277 .from(workflowRuns)
278 .where(gte(workflowRuns.queuedAt, cutoff))
279 .orderBy(desc(workflowRuns.queuedAt))
280 .limit(MAX_WORKFLOW_RUN_ROWS)
281 .catch(() => [] as ProactiveTelemetry["workflowRuns"]),
282 ]);
283 return { auditLog: audits, platformDeploys: deploys, workflowRuns: runs };
284 } catch (err) {
285 console.error("[ai-proactive] telemetry load failed:", err);
286 return empty;
287 }
288}
289
290async function defaultAskClaude(
291 telemetry: ProactiveTelemetry
292): Promise<ProactiveFinding[]> {
293 try {
294 const client = getAnthropic();
295 const message = await client.messages.create({
296 model: MODEL_SONNET,
297 max_tokens: 2048,
298 messages: [
299 {
300 role: "user",
301 content: `You are monitoring Gluecron's own platform health. Here is the last ${PROACTIVE_LOOKBACK_HOURS}h of telemetry. Spot anomalies — degraded deploy times, recurring failures, suspicious audit patterns, memory-leak suspects in long-running workers, abnormal workflow run durations, repeated permission denials, etc.
302
303For each finding, return an entry in a JSON array of the form:
304
305{"findings": [
306 {
307 "title": "<one-line problem summary, prefixed with the affected subsystem>",
308 "severity": "info" | "warning" | "critical",
309 "body_markdown": "<2-6 paragraphs of markdown: what you saw, why it might matter, suggested next step>",
310 "target_url": "<optional admin URL the operator should visit, or null>"
311 }
312]}
313
314Return ONLY the JSON. If nothing looks anomalous, return {"findings": []}. Do not invent findings — silence is the correct answer for a healthy platform.
315
316Telemetry follows:
317
318${summariseTelemetryForPrompt(telemetry)}`,
319 },
320 ],
321 });
322 const parsed = parseJsonResponse<{ findings: ProactiveFinding[] }>(
323 extractText(message)
324 );
325 if (!parsed || !Array.isArray(parsed.findings)) return [];
326 return parsed.findings.filter(
327 (f) =>
328 typeof f.title === "string" &&
329 f.title.trim().length > 0 &&
330 typeof f.body_markdown === "string" &&
331 (f.severity === "info" ||
332 f.severity === "warning" ||
333 f.severity === "critical")
334 );
335 } catch (err) {
336 console.error("[ai-proactive] Claude call failed:", err);
337 return [];
338 }
339}
340
341async function defaultResolveSelfHostRepo(): Promise<{
342 repositoryId: string;
343 ownerId: string;
344} | null> {
345 const fullName = process.env.SELF_HOST_REPO || DEFAULT_SELF_HOST_REPO;
346 const [ownerName, repoName] = fullName.includes("/")
347 ? fullName.split("/")
348 : [fullName, "Gluecron.com"];
349 try {
350 const [row] = await db
351 .select({
352 repositoryId: repositories.id,
353 ownerId: repositories.ownerId,
354 })
355 .from(repositories)
356 .innerJoin(users, eq(users.id, repositories.ownerId))
357 .where(and(eq(users.username, ownerName), eq(repositories.name, repoName)))
358 .limit(1);
359 return row || null;
360 } catch (err) {
361 console.error("[ai-proactive] self-host repo resolve failed:", err);
362 return null;
363 }
364}
365
366async function defaultIsDuplicate(
367 repositoryId: string,
368 dedupeKey: string,
369 lookbackHours: number
370): Promise<boolean> {
371 const cutoff = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
372 try {
373 const [row] = await db
374 .select({ id: issues.id })
375 .from(issues)
376 .where(
377 and(
378 eq(issues.repositoryId, repositoryId),
379 gte(issues.createdAt, cutoff),
380 like(issues.body, `%${dedupeMarker(dedupeKey)}%`)
381 )
382 )
383 .limit(1);
384 return !!row;
385 } catch (err) {
386 console.error("[ai-proactive] dedupe lookup failed:", err);
387 // Fail-closed on dedupe: pretend it's a duplicate so we don't double-fire.
388 return true;
389 }
390}
391
392/** Best-effort label upsert — returns the label id or null. */
393async function ensureProactiveLabel(
394 repositoryId: string
395): Promise<string | null> {
396 try {
397 const [existing] = await db
398 .select({ id: labels.id })
399 .from(labels)
400 .where(
401 and(
402 eq(labels.repositoryId, repositoryId),
403 eq(labels.name, PROACTIVE_LABEL_NAME)
404 )
405 )
406 .limit(1);
407 if (existing) return existing.id;
408 const [inserted] = await db
409 .insert(labels)
410 .values({
411 repositoryId,
412 name: PROACTIVE_LABEL_NAME,
413 color: "#a371f7",
414 description: "Auto-filed by the AI proactive monitor.",
415 })
416 .returning({ id: labels.id });
417 return inserted?.id ?? null;
418 } catch (err) {
419 console.error("[ai-proactive] label ensure failed:", err);
420 return null;
421 }
422}
423
424async function defaultCreateFindingIssue(args: {
425 repositoryId: string;
426 authorId: string;
427 title: string;
428 body: string;
429}): Promise<number | null> {
430 try {
431 const [inserted] = await db
432 .insert(issues)
433 .values({
434 repositoryId: args.repositoryId,
435 authorId: args.authorId,
436 title: args.title,
437 body: args.body,
438 state: "open",
439 })
440 .returning({ id: issues.id, number: issues.number });
441 if (!inserted) return null;
442 const labelId = await ensureProactiveLabel(args.repositoryId);
443 if (labelId) {
444 await db
445 .insert(issueLabels)
446 .values({ issueId: inserted.id, labelId })
447 .catch(() => {
448 /* duplicate label link — ignore */
449 });
450 }
451 return inserted.number ?? null;
452 } catch (err) {
453 console.error("[ai-proactive] issue insert failed:", err);
454 return null;
455 }
456}
457
458async function defaultRecordAudit(
459 finding: ProactiveFinding,
460 repositoryId: string | null,
461 issueNumber: number | null,
462 dedupeKey: string
463): Promise<void> {
464 await audit({
465 repositoryId: repositoryId ?? undefined,
466 action: "ai.proactive.finding",
467 targetType: issueNumber !== null ? "issue" : undefined,
468 targetId: issueNumber !== null ? String(issueNumber) : undefined,
469 metadata: {
470 title: finding.title,
471 severity: finding.severity,
472 dedupeKey,
473 targetUrl: finding.target_url ?? null,
474 },
475 });
476}
477
478/**
479 * One iteration of the proactive monitor. Never throws. Returns a
480 * summary suitable for the autopilot tick log.
481 *
482 * Pipeline:
483 * 1. Skip if AI is unavailable (no ANTHROPIC_API_KEY).
484 * 2. Resolve the self-host repo (env override + sensible default).
485 * 3. Load 24h of telemetry (audit + platform deploys + workflow runs).
486 * 4. Ask Claude for findings as structured JSON.
487 * 5. For each finding above severity=info:
488 * - Skip if a duplicate (sha256 of title) was filed in the last 24h.
489 * - Open an issue tagged `ai:proactive-finding` with the dedupe marker.
490 * - Record an `ai.proactive.finding` audit row.
491 */
492export async function aiProactiveMonitorTick(
493 deps: ProactiveMonitorDeps = {}
494): Promise<ProactiveMonitorSummary> {
495 const aiAvailable = deps.aiAvailable ?? isAiAvailable;
496 const summary: ProactiveMonitorSummary = {
497 considered: 0,
498 opened: 0,
499 skippedDedupe: 0,
500 skippedSeverity: 0,
501 errors: 0,
502 };
503
504 if (!aiAvailable()) {
505 return summary;
506 }
507
508 const loadTelemetry = deps.loadTelemetry ?? defaultLoadTelemetry;
509 const askClaude = deps.askClaude ?? defaultAskClaude;
510 const resolveSelfHostRepo =
511 deps.resolveSelfHostRepo ?? defaultResolveSelfHostRepo;
512 const isDuplicate = deps.isDuplicate ?? defaultIsDuplicate;
513 const createFindingIssue =
514 deps.createFindingIssue ?? defaultCreateFindingIssue;
515 const recordAudit = deps.recordAudit ?? defaultRecordAudit;
516 const maxFindings = deps.maxFindings ?? MAX_FINDINGS_PER_TICK;
517
518 let repo: { repositoryId: string; ownerId: string } | null = null;
519 try {
520 repo = await resolveSelfHostRepo();
521 } catch (err) {
522 console.error("[ai-proactive] resolveSelfHostRepo threw:", err);
523 summary.errors += 1;
524 return summary;
525 }
526 if (!repo) {
527 console.warn(
528 "[ai-proactive] self-host repo not found; skipping tick (set SELF_HOST_REPO to the owner/name of the platform repo)"
529 );
530 return summary;
531 }
532
533 let telemetry: ProactiveTelemetry;
534 try {
535 telemetry = await loadTelemetry(PROACTIVE_LOOKBACK_HOURS);
536 } catch (err) {
537 console.error("[ai-proactive] loadTelemetry threw:", err);
538 summary.errors += 1;
539 return summary;
540 }
541
542 let findings: ProactiveFinding[] = [];
543 try {
544 findings = await askClaude(telemetry);
545 } catch (err) {
546 console.error("[ai-proactive] askClaude threw:", err);
547 summary.errors += 1;
548 return summary;
549 }
550
551 for (const finding of findings.slice(0, maxFindings)) {
552 summary.considered += 1;
553 try {
554 if (finding.severity === "info") {
555 summary.skippedSeverity += 1;
556 continue;
557 }
558 const dedupeKey = dedupeKeyForTitle(finding.title);
559 const dup = await isDuplicate(
560 repo.repositoryId,
561 dedupeKey,
562 PROACTIVE_LOOKBACK_HOURS
563 );
564 if (dup) {
565 summary.skippedDedupe += 1;
566 continue;
567 }
568 const body = renderFindingBody(finding, dedupeKey);
569 const issueNumber = await createFindingIssue({
570 repositoryId: repo.repositoryId,
571 authorId: repo.ownerId,
572 title: finding.title.slice(0, 200),
573 body,
574 });
575 if (issueNumber !== null) {
576 summary.opened += 1;
577 } else {
578 summary.errors += 1;
579 }
580 // Audit fires regardless of issue-insert success so we still have a
581 // trail of what Claude flagged.
582 await recordAudit(finding, repo.repositoryId, issueNumber, dedupeKey);
583 } catch (err) {
584 summary.errors += 1;
585 console.error(
586 `[ai-proactive] per-finding failure for "${finding.title}":`,
587 err
588 );
589 }
590 }
591
592 if (findings.length > maxFindings) {
593 // Count the overflow under `skippedSeverity` since we never even
594 // looked at them — keeps the summary surface flat (no need for a
595 // separate "overflow" counter for an autopilot log line).
596 summary.skippedSeverity += findings.length - maxFindings;
597 }
598
599 console.log(
600 `[ai-proactive] tick considered=${summary.considered} opened=${summary.opened} dedup=${summary.skippedDedupe} skipped=${summary.skippedSeverity} errors=${summary.errors}`
601 );
602 return summary;
603}
604
605/**
606 * Pure helper — renders the issue body markdown for a single finding,
607 * including the dedupe marker. Exported so tests can pin the format
608 * without an Anthropic call.
609 */
610export function renderFindingBody(
611 finding: ProactiveFinding,
612 dedupeKey: string
613): string {
614 const sevBadge =
615 finding.severity === "critical"
616 ? "**Severity:** :rotating_light: critical"
617 : finding.severity === "warning"
618 ? "**Severity:** :warning: warning"
619 : "**Severity:** :information_source: info";
620 const target = finding.target_url
621 ? `**Suggested admin URL:** ${finding.target_url}`
622 : "";
623 return [
624 dedupeMarker(dedupeKey),
625 "_Filed automatically by the GlueCron AI proactive monitor._",
626 "",
627 sevBadge,
628 target,
629 "",
630 finding.body_markdown.trim(),
631 "",
632 "---",
633 `_Dedupe key: \`${dedupeKey}\`. The same finding will not be re-filed for ${PROACTIVE_LOOKBACK_HOURS}h._`,
634 ]
635 .filter((line) => line !== "")
636 .join("\n");
637}
638
639/** Test-only export of internals. */
640export const __test = {
641 summariseTelemetryForPrompt,
642 defaultLoadTelemetry,
643 defaultResolveSelfHostRepo,
644 defaultIsDuplicate,
645 defaultCreateFindingIssue,
646 defaultRecordAudit,
647 dedupeMarker,
648 ensureProactiveLabel,
649 MAX_FINDINGS_PER_TICK,
650};