Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesPull RequestsActionsSecurityInsightsSettings
✨ AI
ExplainAsk AIWorkspaceSpec to PRTestsNL SearchDebt MapArchaeology
More
CommitsDiscussionsWikiProjectsReleasesContributorsPulseGatesDeploymentsPipelineAgents
Blame · Line-by-line history

ai-ci-healer.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/ai-ci-healer.ts
ai-ci-healer.tsBlame667 lines · 1 contributor
Normal viewRaw
9336f45Claude1/**
2 * AI CI Healer — autonomous failure → root-cause → patch PR loop.
3 *
4 * When a `workflow_runs` row lands in status="failure", this module:
5 * 1. Pulls the run + all its `workflow_jobs` (especially the failed ones'
6 * `logs` column).
7 * 2. Asks Claude to identify the root cause + whether it's fixable from
8 * inside this repo.
9 * 3. If patchable, hands the suggested fixes off to the existing
10 * `generatePatchForGateTestFinding` (the finding-shape maps cleanly
11 * onto its `GateTestFinding[]` API — both surfaces want
12 * `{ path, description, severity }`).
13 * 4. Records an `ai.ci.healed` or `ai.ci.gave_up` audit row keyed on the
14 * run id so the autopilot poller doesn't re-process the same failure
15 * every 5 minutes.
16 *
17 * Degrades silently when ANTHROPIC_API_KEY is unset (autopilot also gates
18 * on the env var, but the lib double-checks so it's safe to call directly).
19 * Everything is wrapped in try/catch — `analyzeFailedWorkflowRun` returns
20 * `null` rather than throwing on any failure.
21 *
22 * Marker convention:
23 * - Successful patch open → audit action `ai.ci.healed`, targetId = runId.
24 * - Claude says unfixable → audit action `ai.ci.gave_up`, targetId = runId.
25 * The autopilot poller skips any run that already has either marker, so we
26 * never retry forever.
27 */
28
29import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
30import type Anthropic from "@anthropic-ai/sdk";
31import { db } from "../db";
32import {
33 auditLog,
34 repositories,
35 workflowJobs,
36 workflowRuns,
37 workflows,
38} from "../db/schema";
39import {
40 MODEL_SONNET,
41 extractText,
42 getAnthropic,
43 isAiAvailable,
44 parseJsonResponse,
45} from "./ai-client";
46import { audit } from "./notify";
47import {
48 generatePatchForGateTestFinding,
49 type GateTestFinding,
50} from "./ai-patch-generator";
51
52// ---------------------------------------------------------------------------
53// Tunables
54// ---------------------------------------------------------------------------
55
56/** Cap per-job log slice we ship to Claude — protect the prompt budget. */
57const LOG_SNIPPET_BYTES = 8 * 1024;
58
59/** Hard cap on runs processed per autopilot tick — runaway protection. */
60const HEAL_CAP_PER_TICK = 5;
61
62/** Only consider runs that finished at least this long ago. Gives the
63 * workflow runner time to flush logs + final job rows before we sample. */
64const HEAL_MIN_AGE_MS = 60 * 1_000;
65
66/** Don't bother healing ancient failures — older than this and we assume
67 * the human has already triaged or the SHA has been rewritten. */
68const HEAL_MAX_AGE_MS = 24 * 60 * 60 * 1_000;
69
70// ---------------------------------------------------------------------------
71// Public surface
72// ---------------------------------------------------------------------------
73
74export interface SuggestedFix {
75 /** Relative path inside the repo that the AI thinks needs touching. */
76 path: string;
77 /** What's wrong / what the fix should look like (one-liner). */
78 description: string;
79 /** Severity inherited from the AI's confidence call. */
80 severity?: "low" | "medium" | "high" | "critical";
81}
82
83export interface CiHealAnalysis {
84 /** Plain-English root cause, 1-3 sentences. */
85 rootCause: string;
86 /** Concrete fix proposals (may be empty if Claude can't pinpoint files). */
87 suggestedFixes: SuggestedFix[];
88 /** Convenience: the paths from `suggestedFixes` deduped. */
89 patchablePaths: string[];
90}
91
92export interface CiHealerTickSummary {
93 considered: number;
94 healed: number;
95 gaveUp: number;
96 skipped: number;
97}
98
99/**
100 * Claude's response shape. Kept loose so we tolerate minor schema drift.
101 */
102interface ClaudeCiResponse {
103 rootCause?: string;
104 fixable?: boolean;
105 suggestedFixes?: Array<{
106 path?: string;
107 description?: string;
108 severity?: string;
109 }>;
110 /** Optional: Claude's reason it gave up. We just persist it for ops. */
111 unfixableReason?: string;
112}
113
114// ---------------------------------------------------------------------------
115// Helpers
116// ---------------------------------------------------------------------------
117
118function truncate(s: string | null | undefined, max: number): string {
119 if (!s) return "";
120 if (s.length <= max) return s;
121 return s.slice(0, max) + "\n…(truncated)";
122}
123
124function normaliseSeverity(s: unknown): SuggestedFix["severity"] | undefined {
125 if (typeof s !== "string") return undefined;
126 const v = s.toLowerCase();
127 if (v === "low" || v === "medium" || v === "high" || v === "critical") {
128 return v;
129 }
130 return undefined;
131}
132
133/**
134 * Map our AI-derived `SuggestedFix` set onto the
135 * `GateTestFinding[]` shape that `generatePatchForGateTestFinding`
136 * already accepts. The patch generator only needs `{path, description,
137 * severity}` — exactly what we already have. Each fix becomes one
138 * candidate finding; the generator stops at the first one that produces a
139 * non-empty patch set so a bad suggestion doesn't drown a good one.
140 */
141export function fixesToFindings(
142 fixes: SuggestedFix[],
143 runShortId: string
144): GateTestFinding[] {
145 return fixes
146 .filter((f) => f.path && f.path.trim().length > 0)
147 .map((f, i) => ({
148 id: `ci-heal-${runShortId}-${i}`,
149 ruleId: "ci-failure",
150 path: f.path.trim(),
151 severity: f.severity || "high",
152 title: "CI failure auto-heal",
153 description: f.description || "CI failure",
154 }));
155}
156
157/**
158 * Build the Claude prompt. Pure function so tests can pin the shape.
159 */
160export function buildCiHealPrompt(args: {
161 repoFullName: string;
162 commitSha: string;
163 workflowYaml: string;
164 failedJobs: Array<{ name: string; conclusion: string | null; logs: string }>;
165}): string {
166 const jobsBlock = args.failedJobs
167 .map(
168 (j) =>
169 `### Job: ${j.name} (${j.conclusion || "failure"})\n\`\`\`\n${truncate(
170 j.logs,
171 LOG_SNIPPET_BYTES
172 )}\n\`\`\``
173 )
174 .join("\n\n");
175 return [
176 "You are GlueCron's CI healer. A workflow run just failed. Decide whether the failure is a code bug fixable in THIS repository, and if so propose concrete file edits.",
177 "",
178 `**Repository:** ${args.repoFullName}`,
179 `**Commit:** ${args.commitSha.slice(0, 12)}`,
180 "",
181 "## Workflow YAML",
182 "```yaml",
183 truncate(args.workflowYaml, 4_000),
184 "```",
185 "",
186 "## Failed job logs",
187 jobsBlock || "(no failed-job logs available)",
188 "",
189 "Respond ONLY with JSON of this exact shape:",
190 "{",
191 ' "rootCause": "1-3 sentence diagnosis (what failed, why)",',
192 ' "fixable": true | false,',
193 ' "suggestedFixes": [',
194 ' { "path": "src/foo.ts", "description": "what to change", "severity": "high" }',
195 " ],",
196 ' "unfixableReason": "(only if fixable=false) why the human must intervene"',
197 "}",
198 "",
199 "Rules:",
200 "- `fixable` MUST be false if the failure is an env/infra/external-service problem (missing secret, registry down, network, GitHub Actions runner image, etc.) — anything you can't fix by editing files in this repo.",
201 "- `suggestedFixes` must be empty when fixable=false.",
202 "- Only suggest paths you can identify with high confidence from the logs or YAML. Do not guess at random files.",
203 "- Keep `description` short — the patch generator will be invoked with this finding to produce the actual diff.",
204 ].join("\n");
205}
206
207interface FailedJobRow {
208 name: string;
209 conclusion: string | null;
210 logs: string;
211}
212
213async function loadFailedJobs(runId: string): Promise<FailedJobRow[]> {
214 try {
215 const rows = await db
216 .select({
217 name: workflowJobs.name,
218 conclusion: workflowJobs.conclusion,
219 logs: workflowJobs.logs,
220 status: workflowJobs.status,
221 })
222 .from(workflowJobs)
223 .where(eq(workflowJobs.runId, runId));
224 return rows
225 .filter((r) => r.status === "failure" || r.conclusion === "failure")
226 .map((r) => ({
227 name: r.name,
228 conclusion: r.conclusion,
229 logs: r.logs || "",
230 }));
231 } catch (err) {
232 console.error("[ai-ci-healer] loadFailedJobs failed:", err);
233 return [];
234 }
235}
236
237/**
238 * Has this run already been processed (success or give-up)? We use the
239 * audit log as the marker store so we don't need a schema change.
240 */
241async function hasMarker(runId: string): Promise<boolean> {
242 try {
243 const [row] = await db
244 .select({ id: auditLog.id })
245 .from(auditLog)
246 .where(
247 and(
248 eq(auditLog.targetType, "workflow_run"),
249 eq(auditLog.targetId, runId),
250 sql`${auditLog.action} IN ('ai.ci.healed', 'ai.ci.gave_up')`
251 )
252 )
253 .limit(1);
254 return !!row;
255 } catch (err) {
256 console.warn("[ai-ci-healer] hasMarker query failed:", err);
257 // Fail closed — if we can't check, skip to avoid duplicate work.
258 return true;
259 }
260}
261
262// ---------------------------------------------------------------------------
263// analyzeFailedWorkflowRun — public entry point #1
264// ---------------------------------------------------------------------------
265
266export interface AnalyzeOptions {
267 /** Test-only Anthropic client injection. */
268 client?: Pick<Anthropic, "messages">;
269}
270
271/**
272 * Diagnose a failed run. Returns `null` when:
273 * - The run doesn't exist or isn't actually a failure.
274 * - ANTHROPIC_API_KEY is unset AND no client was injected.
275 * - Claude says the failure isn't fixable from this repo (env/infra).
276 * - Any DB / network step blows up (logged, swallowed).
277 */
278export async function analyzeFailedWorkflowRun(
279 runId: string,
280 opts: AnalyzeOptions = {}
281): Promise<CiHealAnalysis | null> {
282 // Resolve client lazily — tests inject, production reads env.
283 let client: Pick<Anthropic, "messages">;
284 if (opts.client) {
285 client = opts.client;
286 } else {
287 if (!isAiAvailable()) return null;
288 try {
289 client = getAnthropic();
290 } catch {
291 return null;
292 }
293 }
294
295 // Load run row.
296 let run: typeof workflowRuns.$inferSelect | null = null;
297 try {
298 const [row] = await db
299 .select()
300 .from(workflowRuns)
301 .where(eq(workflowRuns.id, runId))
302 .limit(1);
303 run = row || null;
304 } catch (err) {
305 console.error("[ai-ci-healer] loadRun failed:", err);
306 return null;
307 }
308 if (!run || run.status !== "failure") return null;
309
310 // Load workflow + repo (for YAML + naming context).
311 let workflowYaml = "";
312 let repoFullName = "unknown/unknown";
313 try {
314 const [w] = await db
315 .select({ yaml: workflows.yaml })
316 .from(workflows)
317 .where(eq(workflows.id, run.workflowId))
318 .limit(1);
319 if (w?.yaml) workflowYaml = w.yaml;
320 } catch (err) {
321 console.warn("[ai-ci-healer] load workflow failed:", err);
322 }
323 try {
324 const [r] = await db
325 .select({
326 name: repositories.name,
327 ownerId: repositories.ownerId,
328 })
329 .from(repositories)
330 .where(eq(repositories.id, run.repositoryId))
331 .limit(1);
332 if (r) {
333 repoFullName = `${r.ownerId.slice(0, 8)}/${r.name}`;
334 }
335 } catch (err) {
336 console.warn("[ai-ci-healer] load repo failed:", err);
337 }
338
339 const failedJobs = await loadFailedJobs(runId);
340
341 // Ask Claude.
342 let parsed: ClaudeCiResponse | null = null;
343 try {
344 const message = await client.messages.create({
345 model: MODEL_SONNET,
346 max_tokens: 2048,
347 messages: [
348 {
349 role: "user",
350 content: buildCiHealPrompt({
351 repoFullName,
352 commitSha: run.commitSha || "(unknown)",
353 workflowYaml,
354 failedJobs,
355 }),
356 },
357 ],
358 });
0c3eee5Claude359 try {
360 const { recordAiCost, extractUsage } = await import("./ai-cost-tracker");
361 const usage = extractUsage(message);
362 await recordAiCost({
363 repositoryId: run.repositoryId ?? null,
364 model: MODEL_SONNET,
365 inputTokens: usage.input,
366 outputTokens: usage.output,
367 category: "ci_healer",
368 sourceId: runId,
369 sourceKind: "workflow_run",
370 });
371 } catch {
372 /* swallow — best-effort */
373 }
9336f45Claude374 parsed = parseJsonResponse<ClaudeCiResponse>(extractText(message));
375 } catch (err) {
376 console.warn(
377 "[ai-ci-healer] Claude call failed:",
378 err instanceof Error ? err.message : err
379 );
380 return null;
381 }
382 if (!parsed) return null;
383
384 const rootCause =
385 typeof parsed.rootCause === "string" && parsed.rootCause.trim()
386 ? parsed.rootCause.trim()
387 : "(no root cause provided)";
388
389 // "Not fixable" branch — return null so the caller can mark `ai.ci.gave_up`.
390 if (parsed.fixable === false) {
391 return null;
392 }
393
394 const rawFixes = Array.isArray(parsed.suggestedFixes)
395 ? parsed.suggestedFixes
396 : [];
397 const suggestedFixes: SuggestedFix[] = rawFixes
398 .filter(
399 (f): f is { path: string; description?: string; severity?: string } =>
400 !!f && typeof f.path === "string" && f.path.trim().length > 0
401 )
402 .map((f) => ({
403 path: f.path.trim(),
404 description:
405 typeof f.description === "string" && f.description.trim()
406 ? f.description.trim()
407 : rootCause,
408 severity: normaliseSeverity(f.severity),
409 }));
410
411 // Claude said fixable but produced zero usable paths → treat as
412 // unfixable so we don't loop.
413 if (suggestedFixes.length === 0) return null;
414
415 const patchablePaths = Array.from(new Set(suggestedFixes.map((f) => f.path)));
416
417 return { rootCause, suggestedFixes, patchablePaths };
418}
419
420// ---------------------------------------------------------------------------
421// healOneRun — drives one run end-to-end. Public so callers (autopilot,
422// tests, manual retrigger) share the same pipeline.
423// ---------------------------------------------------------------------------
424
425export interface HealOneOptions extends AnalyzeOptions {
426 /** Test override — pin the patch generator output. */
427 generatePatch?: typeof generatePatchForGateTestFinding;
428}
429
430export interface HealOneResult {
431 outcome: "healed" | "gave_up" | "skipped";
432 prNumber?: number;
433 branch?: string;
434 reason?: string;
435}
436
437export async function healOneRun(
438 runId: string,
439 opts: HealOneOptions = {}
440): Promise<HealOneResult> {
441 if (process.env.AUTOPILOT_DISABLED === "1") {
442 return { outcome: "skipped", reason: "autopilot disabled" };
443 }
444 if (!opts.client && !isAiAvailable()) {
445 return { outcome: "skipped", reason: "ANTHROPIC_API_KEY missing" };
446 }
447
448 // Skip if already processed (audit marker present).
449 if (await hasMarker(runId)) {
450 return { outcome: "skipped", reason: "already processed" };
451 }
452
453 const analysis = await analyzeFailedWorkflowRun(runId, {
454 client: opts.client,
455 });
456
457 // Look up the run for audit metadata (repo + sha).
458 let repositoryId: string | null = null;
459 let commitSha: string | null = null;
460 try {
461 const [row] = await db
462 .select({
463 repositoryId: workflowRuns.repositoryId,
464 commitSha: workflowRuns.commitSha,
465 })
466 .from(workflowRuns)
467 .where(eq(workflowRuns.id, runId))
468 .limit(1);
469 if (row) {
470 repositoryId = row.repositoryId;
471 commitSha = row.commitSha;
472 }
473 } catch (err) {
474 console.warn("[ai-ci-healer] post-analyze run lookup failed:", err);
475 }
476
477 if (!analysis) {
478 await audit({
479 userId: null,
480 repositoryId,
481 action: "ai.ci.gave_up",
482 targetType: "workflow_run",
483 targetId: runId,
484 metadata: { commitSha, reason: "unfixable or analysis returned null" },
485 });
486 return { outcome: "gave_up", reason: "unfixable" };
487 }
488
489 if (!repositoryId || !commitSha) {
490 // No base sha → can't seed a patch branch. Mark gave_up so we don't loop.
491 await audit({
492 userId: null,
493 repositoryId,
494 action: "ai.ci.gave_up",
495 targetType: "workflow_run",
496 targetId: runId,
497 metadata: {
498 commitSha,
499 reason: "missing repositoryId or commitSha for patch branch",
500 },
501 });
502 return { outcome: "gave_up", reason: "missing base sha" };
503 }
504
505 const generator = opts.generatePatch ?? generatePatchForGateTestFinding;
506 const findings = fixesToFindings(analysis.suggestedFixes, runId.slice(0, 8));
507
508 const patch = await generator({
509 repositoryId,
510 baseSha: commitSha,
511 findings,
512 client: opts.client,
513 });
514
515 if (!patch) {
516 await audit({
517 userId: null,
518 repositoryId,
519 action: "ai.ci.gave_up",
520 targetType: "workflow_run",
521 targetId: runId,
522 metadata: {
523 commitSha,
524 reason: "patch generator returned null",
525 rootCause: analysis.rootCause,
526 patchablePaths: analysis.patchablePaths,
527 },
528 });
529 return { outcome: "gave_up", reason: "patch generator returned null" };
530 }
531
532 await audit({
533 userId: null,
534 repositoryId,
535 action: "ai.ci.healed",
536 targetType: "workflow_run",
537 targetId: runId,
538 metadata: {
539 commitSha,
540 rootCause: analysis.rootCause,
541 patchablePaths: analysis.patchablePaths,
542 prNumber: patch.prNumber,
543 branch: patch.branch,
544 },
545 });
546
547 return {
548 outcome: "healed",
549 prNumber: patch.prNumber,
550 branch: patch.branch,
551 };
552}
553
554// ---------------------------------------------------------------------------
555// runCiHealerTick — public autopilot entry point
556// ---------------------------------------------------------------------------
557
558export interface CiHealerTickDeps {
559 /** Inject a candidate finder for tests. */
560 findCandidates?: (limit: number) => Promise<{ id: string }[]>;
561 /** Inject the per-run handler for tests. */
562 healOne?: (runId: string) => Promise<HealOneResult>;
563 /** Inject the per-tick cap. */
564 cap?: number;
565}
566
567/**
568 * Find failed workflow runs that:
569 * - finished at least HEAL_MIN_AGE_MS ago (let the runner flush logs)
570 * - finished less than HEAL_MAX_AGE_MS ago (don't chase ancient failures)
571 * - do NOT yet have an ai.ci.healed / ai.ci.gave_up audit marker
572 *
573 * The `hasMarker` filter is applied per-row in `healOneRun` rather than
574 * the SQL select — keeps the query simple and the index on (status,
575 * createdAt) doing most of the work.
576 */
577async function defaultFindCandidates(
578 limit: number
579): Promise<{ id: string }[]> {
580 const now = Date.now();
581 const cutoffNew = new Date(now - HEAL_MIN_AGE_MS);
582 const cutoffOld = new Date(now - HEAL_MAX_AGE_MS);
583 try {
584 const rows = await db
585 .select({ id: workflowRuns.id })
586 .from(workflowRuns)
587 .where(
588 and(
589 eq(workflowRuns.status, "failure"),
590 lt(workflowRuns.createdAt, cutoffNew),
591 gte(workflowRuns.createdAt, cutoffOld)
592 )
593 )
594 .orderBy(desc(workflowRuns.createdAt))
595 .limit(limit);
596 return rows;
597 } catch (err) {
598 console.error("[ai-ci-healer] candidate query failed:", err);
599 return [];
600 }
601}
602
603/**
604 * One autopilot tick: scan recent failures, heal what we can. Returns a
605 * counts summary. Never throws.
606 *
607 * No-op when:
608 * - AUTOPILOT_DISABLED=1
609 * - ANTHROPIC_API_KEY is unset
610 */
611export async function runCiHealerTick(
612 deps: CiHealerTickDeps = {}
613): Promise<CiHealerTickSummary> {
614 if (process.env.AUTOPILOT_DISABLED === "1") {
615 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
616 }
617 if (!isAiAvailable()) {
618 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
619 }
620
621 const findCandidates = deps.findCandidates ?? defaultFindCandidates;
622 const healOne = deps.healOne ?? ((id: string) => healOneRun(id));
623 const cap = deps.cap ?? HEAL_CAP_PER_TICK;
624
625 let candidates: { id: string }[] = [];
626 try {
627 candidates = await findCandidates(cap);
628 } catch (err) {
629 console.error("[ai-ci-healer] findCandidates threw:", err);
630 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
631 }
632
633 let healed = 0;
634 let gaveUp = 0;
635 let skipped = 0;
636 for (const cand of candidates) {
637 try {
638 const res = await healOne(cand.id);
639 if (res.outcome === "healed") healed += 1;
640 else if (res.outcome === "gave_up") gaveUp += 1;
641 else skipped += 1;
642 } catch (err) {
643 skipped += 1;
644 console.error(
645 `[ai-ci-healer] per-run failure for run=${cand.id}:`,
646 err instanceof Error ? err.message : err
647 );
648 }
649 }
650
651 return { considered: candidates.length, healed, gaveUp, skipped };
652}
653
654// ---------------------------------------------------------------------------
655// Test-only re-exports
656// ---------------------------------------------------------------------------
657
658export const __test = {
659 loadFailedJobs,
660 hasMarker,
661 defaultFindCandidates,
662 normaliseSeverity,
663 truncate,
664 HEAL_MIN_AGE_MS,
665 HEAL_MAX_AGE_MS,
666 HEAL_CAP_PER_TICK,
667};