Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesPull RequestsActionsSecurityInsightsSettings
✨ AI
ExplainAsk AIWorkspaceSpec to PRTestsNL SearchDebt MapArchaeology
More
CommitsDiscussionsWikiProjectsReleasesContributorsPulseGatesDeploymentsPipelineAgents
Blame · Line-by-line history

ai-proactive-monitor.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/ai-proactive-monitor.ts
ai-proactive-monitor.tsBlame663 lines · 1 contributor
Normal viewRaw
e9aa4d8Claude1/**
2 * AI Proactive Monitor — hourly platform-health surveillance.
3 *
4 * Pulls the last 24 hours of platform telemetry (audit log, platform
5 * deploys, workflow runs) and asks Claude to spot anomalies — degraded
6 * deploy times, recurring failures, suspicious audit patterns, etc.
7 *
8 * For each finding above `severity=info` it opens an issue on the
9 * platform self-host repo (`SELF_HOST_REPO` env, defaults to
10 * `ccantynz-alt/Gluecron.com`) tagged with the `ai:proactive-finding`
11 * label. A deterministic dedupe key (sha256 of the title) is embedded
12 * in the issue body as an HTML marker so the same alert never fires
13 * twice in a 24h window — even across process restarts.
14 *
15 * Every finding is also written to `audit_log` under action
16 * `ai.proactive.finding` so operators have a queryable trail of what
17 * Claude noticed and when.
18 *
19 * Hooks into autopilot via `aiProactiveMonitorTick()` — wrapped in
20 * try/catch in the registration so a single failure cannot wedge the
21 * surrounding tick. Skips cleanly when:
22 * - `ANTHROPIC_API_KEY` is unset (graceful no-op).
23 * - `AUTOPILOT_DISABLED=1` (the surrounding loop never invokes us).
24 *
25 * Same dependency-injection seam pattern as `runAutoMergeSweep` /
26 * `runAiBuildTaskOnce` — every DB / Claude / clock interaction is
27 * overridable so tests don't need the DB or the AI client.
28 */
29
30import { createHash } from "crypto";
31import { and, desc, eq, gte, like } from "drizzle-orm";
32import { db } from "../db";
33import {
34 auditLog,
35 issueLabels,
36 issues,
37 labels,
38 repositories,
39 users,
40} from "../db/schema";
41import { platformDeploys } from "../db/schema-deploys";
42import { workflowRuns } from "../db/schema";
43import {
44 MODEL_SONNET,
45 extractText,
46 getAnthropic,
47 isAiAvailable,
48 parseJsonResponse,
49} from "./ai-client";
50import { audit } from "./notify";
51
52/** Default self-host repo when SELF_HOST_REPO is not configured. */
53export const DEFAULT_SELF_HOST_REPO = "ccantynz-alt/Gluecron.com";
54
55/** Label attached to every issue opened by this monitor. */
56export const PROACTIVE_LABEL_NAME = "ai:proactive-finding";
57
58/** Stable marker embedded in issue bodies for dedupe lookups. */
59export const PROACTIVE_DEDUPE_MARKER_PREFIX =
60 "<!-- gluecron:ai-proactive:dedupe=";
61export const PROACTIVE_DEDUPE_MARKER_SUFFIX = " -->";
62
63/** Lookback window we feed Claude — and also the dedupe horizon. */
64export const PROACTIVE_LOOKBACK_HOURS = 24;
65
66/** Hard cap on rows pulled per table so the prompt stays bounded. */
67const MAX_AUDIT_ROWS = 200;
68const MAX_DEPLOY_ROWS = 50;
69const MAX_WORKFLOW_RUN_ROWS = 200;
70
71/** Hard cap on issues we'll open in a single tick (runaway protection). */
72const MAX_FINDINGS_PER_TICK = 5;
73
74export type ProactiveSeverity = "info" | "warning" | "critical";
75
76export interface ProactiveFinding {
77 title: string;
78 severity: ProactiveSeverity;
79 body_markdown: string;
80 target_url?: string | null;
81}
82
83export interface ProactiveTelemetry {
84 auditLog: Array<{
85 action: string;
86 targetType: string | null;
87 targetId: string | null;
88 userId: string | null;
89 repositoryId: string | null;
90 createdAt: Date;
91 }>;
92 platformDeploys: Array<{
93 runId: string;
94 sha: string;
95 status: string;
96 durationMs: number | null;
97 error: string | null;
98 startedAt: Date;
99 finishedAt: Date | null;
100 }>;
101 workflowRuns: Array<{
102 id: string;
103 status: string;
104 conclusion: string | null;
105 event: string;
106 queuedAt: Date;
107 startedAt: Date | null;
108 finishedAt: Date | null;
109 }>;
110}
111
112export interface ProactiveMonitorDeps {
113 /** Override telemetry loader (DI for tests). */
114 loadTelemetry?: (lookbackHours: number) => Promise<ProactiveTelemetry>;
115 /** Override Claude call — returns the parsed findings array. */
116 askClaude?: (
117 telemetry: ProactiveTelemetry
118 ) => Promise<ProactiveFinding[]>;
119 /** Override the self-host repo resolver (returns null when missing). */
120 resolveSelfHostRepo?: () => Promise<{
121 repositoryId: string;
122 ownerId: string;
123 } | null>;
124 /** Override dedupe lookup. Returns true if a finding with this key already exists in the lookback window. */
125 isDuplicate?: (
126 repositoryId: string,
127 dedupeKey: string,
128 lookbackHours: number
129 ) => Promise<boolean>;
130 /** Override the issue + label writer. Returns the new issue number. */
131 createFindingIssue?: (args: {
132 repositoryId: string;
133 authorId: string;
134 title: string;
135 body: string;
136 }) => Promise<number | null>;
137 /** Override the audit writer (DI for tests). */
138 recordAudit?: (
139 finding: ProactiveFinding,
140 repositoryId: string | null,
141 issueNumber: number | null,
142 dedupeKey: string
143 ) => Promise<void>;
144 /** Override clock for deterministic windows in tests. */
145 now?: () => Date;
146 /** Override AI-key check (lets tests run the full pipeline). */
147 aiAvailable?: () => boolean;
148 /** Override the per-tick cap. */
149 maxFindings?: number;
150}
151
152export interface ProactiveMonitorSummary {
153 considered: number;
154 opened: number;
155 skippedDedupe: number;
156 skippedSeverity: number;
157 errors: number;
158}
159
160/**
161 * sha256 of the title, used as the deterministic dedupe key. Same
162 * title => same key, so we can detect "already filed this in the last
163 * 24h" with a single LIKE query on issue body.
164 */
165export function dedupeKeyForTitle(title: string): string {
166 return createHash("sha256")
167 .update(title.trim().toLowerCase())
168 .digest("hex")
169 .slice(0, 32);
170}
171
172function dedupeMarker(key: string): string {
173 return `${PROACTIVE_DEDUPE_MARKER_PREFIX}${key}${PROACTIVE_DEDUPE_MARKER_SUFFIX}`;
174}
175
176/** Compact telemetry summary embedded in the prompt — keeps tokens bounded. */
177function summariseTelemetryForPrompt(t: ProactiveTelemetry): string {
178 const auditByAction = new Map<string, number>();
179 for (const row of t.auditLog) {
180 auditByAction.set(row.action, (auditByAction.get(row.action) || 0) + 1);
181 }
182 const auditLines = Array.from(auditByAction.entries())
183 .sort((a, b) => b[1] - a[1])
184 .slice(0, 30)
185 .map(([action, count]) => `- ${action}: ${count}`)
186 .join("\n");
187
188 const deployLines = t.platformDeploys
189 .slice(0, 30)
190 .map((d) => {
191 const dur = d.durationMs !== null ? `${d.durationMs}ms` : "n/a";
192 const err = d.error ? ` error="${d.error.slice(0, 120)}"` : "";
193 return `- run=${d.runId} sha=${d.sha.slice(0, 7)} status=${d.status} dur=${dur}${err}`;
194 })
195 .join("\n");
196
197 const wfStatusCounts = new Map<string, number>();
198 let durSum = 0;
199 let durCount = 0;
200 for (const r of t.workflowRuns) {
201 const key = `${r.status}/${r.conclusion ?? "n/a"}`;
202 wfStatusCounts.set(key, (wfStatusCounts.get(key) || 0) + 1);
203 if (r.startedAt && r.finishedAt) {
204 durSum += r.finishedAt.getTime() - r.startedAt.getTime();
205 durCount += 1;
206 }
207 }
208 const wfLines = Array.from(wfStatusCounts.entries())
209 .sort((a, b) => b[1] - a[1])
210 .map(([k, v]) => `- ${k}: ${v}`)
211 .join("\n");
212 const avgWfDuration =
213 durCount > 0 ? `${Math.round(durSum / durCount)}ms (n=${durCount})` : "n/a";
214
215 return [
216 `## Audit log (${t.auditLog.length} rows, top actions)`,
217 auditLines || "(none)",
218 "",
219 `## Platform deploys (${t.platformDeploys.length} rows, most recent first)`,
220 deployLines || "(none)",
221 "",
222 `## Workflow runs (${t.workflowRuns.length} rows, avg duration ${avgWfDuration})`,
223 wfLines || "(none)",
224 ].join("\n");
225}
226
227async function defaultLoadTelemetry(
228 lookbackHours: number
229): Promise<ProactiveTelemetry> {
230 const cutoff = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
231 const empty: ProactiveTelemetry = {
232 auditLog: [],
233 platformDeploys: [],
234 workflowRuns: [],
235 };
236 try {
237 const [audits, deploys, runs] = await Promise.all([
238 db
239 .select({
240 action: auditLog.action,
241 targetType: auditLog.targetType,
242 targetId: auditLog.targetId,
243 userId: auditLog.userId,
244 repositoryId: auditLog.repositoryId,
245 createdAt: auditLog.createdAt,
246 })
247 .from(auditLog)
248 .where(gte(auditLog.createdAt, cutoff))
249 .orderBy(desc(auditLog.createdAt))
250 .limit(MAX_AUDIT_ROWS)
251 .catch(() => [] as ProactiveTelemetry["auditLog"]),
252 db
253 .select({
254 runId: platformDeploys.runId,
255 sha: platformDeploys.sha,
256 status: platformDeploys.status,
257 durationMs: platformDeploys.durationMs,
258 error: platformDeploys.error,
259 startedAt: platformDeploys.startedAt,
260 finishedAt: platformDeploys.finishedAt,
261 })
262 .from(platformDeploys)
263 .where(gte(platformDeploys.startedAt, cutoff))
264 .orderBy(desc(platformDeploys.startedAt))
265 .limit(MAX_DEPLOY_ROWS)
266 .catch(() => [] as ProactiveTelemetry["platformDeploys"]),
267 db
268 .select({
269 id: workflowRuns.id,
270 status: workflowRuns.status,
271 conclusion: workflowRuns.conclusion,
272 event: workflowRuns.event,
273 queuedAt: workflowRuns.queuedAt,
274 startedAt: workflowRuns.startedAt,
275 finishedAt: workflowRuns.finishedAt,
276 })
277 .from(workflowRuns)
278 .where(gte(workflowRuns.queuedAt, cutoff))
279 .orderBy(desc(workflowRuns.queuedAt))
280 .limit(MAX_WORKFLOW_RUN_ROWS)
281 .catch(() => [] as ProactiveTelemetry["workflowRuns"]),
282 ]);
283 return { auditLog: audits, platformDeploys: deploys, workflowRuns: runs };
284 } catch (err) {
285 console.error("[ai-proactive] telemetry load failed:", err);
286 return empty;
287 }
288}
289
290async function defaultAskClaude(
291 telemetry: ProactiveTelemetry
292): Promise<ProactiveFinding[]> {
293 try {
294 const client = getAnthropic();
295 const message = await client.messages.create({
296 model: MODEL_SONNET,
297 max_tokens: 2048,
298 messages: [
299 {
300 role: "user",
301 content: `You are monitoring Gluecron's own platform health. Here is the last ${PROACTIVE_LOOKBACK_HOURS}h of telemetry. Spot anomalies — degraded deploy times, recurring failures, suspicious audit patterns, memory-leak suspects in long-running workers, abnormal workflow run durations, repeated permission denials, etc.
302
303For each finding, return an entry in a JSON array of the form:
304
305{"findings": [
306 {
307 "title": "<one-line problem summary, prefixed with the affected subsystem>",
308 "severity": "info" | "warning" | "critical",
309 "body_markdown": "<2-6 paragraphs of markdown: what you saw, why it might matter, suggested next step>",
310 "target_url": "<optional admin URL the operator should visit, or null>"
311 }
312]}
313
314Return ONLY the JSON. If nothing looks anomalous, return {"findings": []}. Do not invent findings — silence is the correct answer for a healthy platform.
315
316Telemetry follows:
317
318${summariseTelemetryForPrompt(telemetry)}`,
319 },
320 ],
321 });
1d4ff60Claude322 try {
323 const { recordAiCost, extractUsage } = await import("./ai-cost-tracker");
324 const usage = extractUsage(message);
325 await recordAiCost({
326 model: MODEL_SONNET,
327 inputTokens: usage.input,
328 outputTokens: usage.output,
329 category: "other",
330 sourceKind: "proactive_monitor",
331 });
332 } catch {
333 /* swallow — best-effort */
334 }
e9aa4d8Claude335 const parsed = parseJsonResponse<{ findings: ProactiveFinding[] }>(
336 extractText(message)
337 );
338 if (!parsed || !Array.isArray(parsed.findings)) return [];
339 return parsed.findings.filter(
340 (f) =>
341 typeof f.title === "string" &&
342 f.title.trim().length > 0 &&
343 typeof f.body_markdown === "string" &&
344 (f.severity === "info" ||
345 f.severity === "warning" ||
346 f.severity === "critical")
347 );
348 } catch (err) {
349 console.error("[ai-proactive] Claude call failed:", err);
350 return [];
351 }
352}
353
354async function defaultResolveSelfHostRepo(): Promise<{
355 repositoryId: string;
356 ownerId: string;
357} | null> {
358 const fullName = process.env.SELF_HOST_REPO || DEFAULT_SELF_HOST_REPO;
359 const [ownerName, repoName] = fullName.includes("/")
360 ? fullName.split("/")
361 : [fullName, "Gluecron.com"];
362 try {
363 const [row] = await db
364 .select({
365 repositoryId: repositories.id,
366 ownerId: repositories.ownerId,
367 })
368 .from(repositories)
369 .innerJoin(users, eq(users.id, repositories.ownerId))
370 .where(and(eq(users.username, ownerName), eq(repositories.name, repoName)))
371 .limit(1);
372 return row || null;
373 } catch (err) {
374 console.error("[ai-proactive] self-host repo resolve failed:", err);
375 return null;
376 }
377}
378
379async function defaultIsDuplicate(
380 repositoryId: string,
381 dedupeKey: string,
382 lookbackHours: number
383): Promise<boolean> {
384 const cutoff = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
385 try {
386 const [row] = await db
387 .select({ id: issues.id })
388 .from(issues)
389 .where(
390 and(
391 eq(issues.repositoryId, repositoryId),
392 gte(issues.createdAt, cutoff),
393 like(issues.body, `%${dedupeMarker(dedupeKey)}%`)
394 )
395 )
396 .limit(1);
397 return !!row;
398 } catch (err) {
399 console.error("[ai-proactive] dedupe lookup failed:", err);
400 // Fail-closed on dedupe: pretend it's a duplicate so we don't double-fire.
401 return true;
402 }
403}
404
405/** Best-effort label upsert — returns the label id or null. */
406async function ensureProactiveLabel(
407 repositoryId: string
408): Promise<string | null> {
409 try {
410 const [existing] = await db
411 .select({ id: labels.id })
412 .from(labels)
413 .where(
414 and(
415 eq(labels.repositoryId, repositoryId),
416 eq(labels.name, PROACTIVE_LABEL_NAME)
417 )
418 )
419 .limit(1);
420 if (existing) return existing.id;
421 const [inserted] = await db
422 .insert(labels)
423 .values({
424 repositoryId,
425 name: PROACTIVE_LABEL_NAME,
426 color: "#a371f7",
427 description: "Auto-filed by the AI proactive monitor.",
428 })
429 .returning({ id: labels.id });
430 return inserted?.id ?? null;
431 } catch (err) {
432 console.error("[ai-proactive] label ensure failed:", err);
433 return null;
434 }
435}
436
437async function defaultCreateFindingIssue(args: {
438 repositoryId: string;
439 authorId: string;
440 title: string;
441 body: string;
442}): Promise<number | null> {
443 try {
444 const [inserted] = await db
445 .insert(issues)
446 .values({
447 repositoryId: args.repositoryId,
448 authorId: args.authorId,
449 title: args.title,
450 body: args.body,
451 state: "open",
452 })
453 .returning({ id: issues.id, number: issues.number });
454 if (!inserted) return null;
455 const labelId = await ensureProactiveLabel(args.repositoryId);
456 if (labelId) {
457 await db
458 .insert(issueLabels)
459 .values({ issueId: inserted.id, labelId })
460 .catch(() => {
461 /* duplicate label link — ignore */
462 });
463 }
464 return inserted.number ?? null;
465 } catch (err) {
466 console.error("[ai-proactive] issue insert failed:", err);
467 return null;
468 }
469}
470
471async function defaultRecordAudit(
472 finding: ProactiveFinding,
473 repositoryId: string | null,
474 issueNumber: number | null,
475 dedupeKey: string
476): Promise<void> {
477 await audit({
478 repositoryId: repositoryId ?? undefined,
479 action: "ai.proactive.finding",
480 targetType: issueNumber !== null ? "issue" : undefined,
481 targetId: issueNumber !== null ? String(issueNumber) : undefined,
482 metadata: {
483 title: finding.title,
484 severity: finding.severity,
485 dedupeKey,
486 targetUrl: finding.target_url ?? null,
487 },
488 });
489}
490
491/**
492 * One iteration of the proactive monitor. Never throws. Returns a
493 * summary suitable for the autopilot tick log.
494 *
495 * Pipeline:
496 * 1. Skip if AI is unavailable (no ANTHROPIC_API_KEY).
497 * 2. Resolve the self-host repo (env override + sensible default).
498 * 3. Load 24h of telemetry (audit + platform deploys + workflow runs).
499 * 4. Ask Claude for findings as structured JSON.
500 * 5. For each finding above severity=info:
501 * - Skip if a duplicate (sha256 of title) was filed in the last 24h.
502 * - Open an issue tagged `ai:proactive-finding` with the dedupe marker.
503 * - Record an `ai.proactive.finding` audit row.
504 */
505export async function aiProactiveMonitorTick(
506 deps: ProactiveMonitorDeps = {}
507): Promise<ProactiveMonitorSummary> {
508 const aiAvailable = deps.aiAvailable ?? isAiAvailable;
509 const summary: ProactiveMonitorSummary = {
510 considered: 0,
511 opened: 0,
512 skippedDedupe: 0,
513 skippedSeverity: 0,
514 errors: 0,
515 };
516
517 if (!aiAvailable()) {
518 return summary;
519 }
520
521 const loadTelemetry = deps.loadTelemetry ?? defaultLoadTelemetry;
522 const askClaude = deps.askClaude ?? defaultAskClaude;
523 const resolveSelfHostRepo =
524 deps.resolveSelfHostRepo ?? defaultResolveSelfHostRepo;
525 const isDuplicate = deps.isDuplicate ?? defaultIsDuplicate;
526 const createFindingIssue =
527 deps.createFindingIssue ?? defaultCreateFindingIssue;
528 const recordAudit = deps.recordAudit ?? defaultRecordAudit;
529 const maxFindings = deps.maxFindings ?? MAX_FINDINGS_PER_TICK;
530
531 let repo: { repositoryId: string; ownerId: string } | null = null;
532 try {
533 repo = await resolveSelfHostRepo();
534 } catch (err) {
535 console.error("[ai-proactive] resolveSelfHostRepo threw:", err);
536 summary.errors += 1;
537 return summary;
538 }
539 if (!repo) {
540 console.warn(
541 "[ai-proactive] self-host repo not found; skipping tick (set SELF_HOST_REPO to the owner/name of the platform repo)"
542 );
543 return summary;
544 }
545
546 let telemetry: ProactiveTelemetry;
547 try {
548 telemetry = await loadTelemetry(PROACTIVE_LOOKBACK_HOURS);
549 } catch (err) {
550 console.error("[ai-proactive] loadTelemetry threw:", err);
551 summary.errors += 1;
552 return summary;
553 }
554
555 let findings: ProactiveFinding[] = [];
556 try {
557 findings = await askClaude(telemetry);
558 } catch (err) {
559 console.error("[ai-proactive] askClaude threw:", err);
560 summary.errors += 1;
561 return summary;
562 }
563
564 for (const finding of findings.slice(0, maxFindings)) {
565 summary.considered += 1;
566 try {
567 if (finding.severity === "info") {
568 summary.skippedSeverity += 1;
569 continue;
570 }
571 const dedupeKey = dedupeKeyForTitle(finding.title);
572 const dup = await isDuplicate(
573 repo.repositoryId,
574 dedupeKey,
575 PROACTIVE_LOOKBACK_HOURS
576 );
577 if (dup) {
578 summary.skippedDedupe += 1;
579 continue;
580 }
581 const body = renderFindingBody(finding, dedupeKey);
582 const issueNumber = await createFindingIssue({
583 repositoryId: repo.repositoryId,
584 authorId: repo.ownerId,
585 title: finding.title.slice(0, 200),
586 body,
587 });
588 if (issueNumber !== null) {
589 summary.opened += 1;
590 } else {
591 summary.errors += 1;
592 }
593 // Audit fires regardless of issue-insert success so we still have a
594 // trail of what Claude flagged.
595 await recordAudit(finding, repo.repositoryId, issueNumber, dedupeKey);
596 } catch (err) {
597 summary.errors += 1;
598 console.error(
599 `[ai-proactive] per-finding failure for "${finding.title}":`,
600 err
601 );
602 }
603 }
604
605 if (findings.length > maxFindings) {
606 // Count the overflow under `skippedSeverity` since we never even
607 // looked at them — keeps the summary surface flat (no need for a
608 // separate "overflow" counter for an autopilot log line).
609 summary.skippedSeverity += findings.length - maxFindings;
610 }
611
612 console.log(
613 `[ai-proactive] tick considered=${summary.considered} opened=${summary.opened} dedup=${summary.skippedDedupe} skipped=${summary.skippedSeverity} errors=${summary.errors}`
614 );
615 return summary;
616}
617
618/**
619 * Pure helper — renders the issue body markdown for a single finding,
620 * including the dedupe marker. Exported so tests can pin the format
621 * without an Anthropic call.
622 */
623export function renderFindingBody(
624 finding: ProactiveFinding,
625 dedupeKey: string
626): string {
627 const sevBadge =
628 finding.severity === "critical"
629 ? "**Severity:** :rotating_light: critical"
630 : finding.severity === "warning"
631 ? "**Severity:** :warning: warning"
632 : "**Severity:** :information_source: info";
633 const target = finding.target_url
634 ? `**Suggested admin URL:** ${finding.target_url}`
635 : "";
636 return [
637 dedupeMarker(dedupeKey),
638 "_Filed automatically by the GlueCron AI proactive monitor._",
639 "",
640 sevBadge,
641 target,
642 "",
643 finding.body_markdown.trim(),
644 "",
645 "---",
646 `_Dedupe key: \`${dedupeKey}\`. The same finding will not be re-filed for ${PROACTIVE_LOOKBACK_HOURS}h._`,
647 ]
648 .filter((line) => line !== "")
649 .join("\n");
650}
651
652/** Test-only export of internals. */
653export const __test = {
654 summariseTelemetryForPrompt,
655 defaultLoadTelemetry,
656 defaultResolveSelfHostRepo,
657 defaultIsDuplicate,
658 defaultCreateFindingIssue,
659 defaultRecordAudit,
660 dedupeMarker,
661 ensureProactiveLabel,
662 MAX_FINDINGS_PER_TICK,
663};