Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

ai-ci-healer.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/ai-ci-healer.ts
ai-ci-healer.tsBlame701 lines · 1 contributor
Normal viewRaw
9336f45Claude1/**
2 * AI CI Healer — autonomous failure → root-cause → patch PR loop.
3 *
4 * When a `workflow_runs` row lands in status="failure", this module:
5 * 1. Pulls the run + all its `workflow_jobs` (especially the failed ones'
6 * `logs` column).
7 * 2. Asks Claude to identify the root cause + whether it's fixable from
8 * inside this repo.
9 * 3. If patchable, hands the suggested fixes off to the existing
10 * `generatePatchForGateTestFinding` (the finding-shape maps cleanly
11 * onto its `GateTestFinding[]` API — both surfaces want
12 * `{ path, description, severity }`).
13 * 4. Records an `ai.ci.healed` or `ai.ci.gave_up` audit row keyed on the
14 * run id so the autopilot poller doesn't re-process the same failure
15 * every 5 minutes.
16 *
17 * Degrades silently when ANTHROPIC_API_KEY is unset (autopilot also gates
18 * on the env var, but the lib double-checks so it's safe to call directly).
19 * Everything is wrapped in try/catch — `analyzeFailedWorkflowRun` returns
20 * `null` rather than throwing on any failure.
21 *
22 * Marker convention:
23 * - Successful patch open → audit action `ai.ci.healed`, targetId = runId.
24 * - Claude says unfixable → audit action `ai.ci.gave_up`, targetId = runId.
25 * The autopilot poller skips any run that already has either marker, so we
26 * never retry forever.
27 */
28
29import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
30import type Anthropic from "@anthropic-ai/sdk";
31import { db } from "../db";
32import {
33 auditLog,
34 repositories,
35 workflowJobs,
36 workflowRuns,
37 workflows,
38} from "../db/schema";
39import {
40 MODEL_SONNET,
41 extractText,
42 getAnthropic,
43 isAiAvailable,
44 parseJsonResponse,
45} from "./ai-client";
46import { audit } from "./notify";
47import {
48 generatePatchForGateTestFinding,
49 type GateTestFinding,
50} from "./ai-patch-generator";
0b49751Claude51import { assertAiQuota, AiQuotaExceededError } from "./billing";
9336f45Claude52
53// ---------------------------------------------------------------------------
54// Tunables
55// ---------------------------------------------------------------------------
56
57/** Cap per-job log slice we ship to Claude — protect the prompt budget. */
58const LOG_SNIPPET_BYTES = 8 * 1024;
59
60/** Hard cap on runs processed per autopilot tick — runaway protection. */
61const HEAL_CAP_PER_TICK = 5;
62
63/** Only consider runs that finished at least this long ago. Gives the
64 * workflow runner time to flush logs + final job rows before we sample. */
65const HEAL_MIN_AGE_MS = 60 * 1_000;
66
67/** Don't bother healing ancient failures — older than this and we assume
68 * the human has already triaged or the SHA has been rewritten. */
69const HEAL_MAX_AGE_MS = 24 * 60 * 60 * 1_000;
70
71// ---------------------------------------------------------------------------
72// Public surface
73// ---------------------------------------------------------------------------
74
75export interface SuggestedFix {
76 /** Relative path inside the repo that the AI thinks needs touching. */
77 path: string;
78 /** What's wrong / what the fix should look like (one-liner). */
79 description: string;
80 /** Severity inherited from the AI's confidence call. */
81 severity?: "low" | "medium" | "high" | "critical";
82}
83
84export interface CiHealAnalysis {
85 /** Plain-English root cause, 1-3 sentences. */
86 rootCause: string;
87 /** Concrete fix proposals (may be empty if Claude can't pinpoint files). */
88 suggestedFixes: SuggestedFix[];
89 /** Convenience: the paths from `suggestedFixes` deduped. */
90 patchablePaths: string[];
91}
92
93export interface CiHealerTickSummary {
94 considered: number;
95 healed: number;
96 gaveUp: number;
97 skipped: number;
98}
99
100/**
101 * Claude's response shape. Kept loose so we tolerate minor schema drift.
102 */
103interface ClaudeCiResponse {
104 rootCause?: string;
105 fixable?: boolean;
106 suggestedFixes?: Array<{
107 path?: string;
108 description?: string;
109 severity?: string;
110 }>;
111 /** Optional: Claude's reason it gave up. We just persist it for ops. */
112 unfixableReason?: string;
113}
114
115// ---------------------------------------------------------------------------
116// Helpers
117// ---------------------------------------------------------------------------
118
119function truncate(s: string | null | undefined, max: number): string {
120 if (!s) return "";
121 if (s.length <= max) return s;
122 return s.slice(0, max) + "\n…(truncated)";
123}
124
125function normaliseSeverity(s: unknown): SuggestedFix["severity"] | undefined {
126 if (typeof s !== "string") return undefined;
127 const v = s.toLowerCase();
128 if (v === "low" || v === "medium" || v === "high" || v === "critical") {
129 return v;
130 }
131 return undefined;
132}
133
134/**
135 * Map our AI-derived `SuggestedFix` set onto the
136 * `GateTestFinding[]` shape that `generatePatchForGateTestFinding`
137 * already accepts. The patch generator only needs `{path, description,
138 * severity}` — exactly what we already have. Each fix becomes one
139 * candidate finding; the generator stops at the first one that produces a
140 * non-empty patch set so a bad suggestion doesn't drown a good one.
141 */
142export function fixesToFindings(
143 fixes: SuggestedFix[],
144 runShortId: string
145): GateTestFinding[] {
146 return fixes
147 .filter((f) => f.path && f.path.trim().length > 0)
148 .map((f, i) => ({
149 id: `ci-heal-${runShortId}-${i}`,
150 ruleId: "ci-failure",
151 path: f.path.trim(),
152 severity: f.severity || "high",
153 title: "CI failure auto-heal",
154 description: f.description || "CI failure",
155 }));
156}
157
158/**
159 * Build the Claude prompt. Pure function so tests can pin the shape.
160 */
161export function buildCiHealPrompt(args: {
162 repoFullName: string;
163 commitSha: string;
164 workflowYaml: string;
165 failedJobs: Array<{ name: string; conclusion: string | null; logs: string }>;
166}): string {
167 const jobsBlock = args.failedJobs
168 .map(
169 (j) =>
170 `### Job: ${j.name} (${j.conclusion || "failure"})\n\`\`\`\n${truncate(
171 j.logs,
172 LOG_SNIPPET_BYTES
173 )}\n\`\`\``
174 )
175 .join("\n\n");
176 return [
177 "You are GlueCron's CI healer. A workflow run just failed. Decide whether the failure is a code bug fixable in THIS repository, and if so propose concrete file edits.",
178 "",
179 `**Repository:** ${args.repoFullName}`,
180 `**Commit:** ${args.commitSha.slice(0, 12)}`,
181 "",
182 "## Workflow YAML",
183 "```yaml",
184 truncate(args.workflowYaml, 4_000),
185 "```",
186 "",
187 "## Failed job logs",
188 jobsBlock || "(no failed-job logs available)",
189 "",
190 "Respond ONLY with JSON of this exact shape:",
191 "{",
192 ' "rootCause": "1-3 sentence diagnosis (what failed, why)",',
193 ' "fixable": true | false,',
194 ' "suggestedFixes": [',
195 ' { "path": "src/foo.ts", "description": "what to change", "severity": "high" }',
196 " ],",
197 ' "unfixableReason": "(only if fixable=false) why the human must intervene"',
198 "}",
199 "",
200 "Rules:",
201 "- `fixable` MUST be false if the failure is an env/infra/external-service problem (missing secret, registry down, network, GitHub Actions runner image, etc.) — anything you can't fix by editing files in this repo.",
202 "- `suggestedFixes` must be empty when fixable=false.",
203 "- Only suggest paths you can identify with high confidence from the logs or YAML. Do not guess at random files.",
204 "- Keep `description` short — the patch generator will be invoked with this finding to produce the actual diff.",
205 ].join("\n");
206}
207
208interface FailedJobRow {
209 name: string;
210 conclusion: string | null;
211 logs: string;
212}
213
214async function loadFailedJobs(runId: string): Promise<FailedJobRow[]> {
215 try {
216 const rows = await db
217 .select({
218 name: workflowJobs.name,
219 conclusion: workflowJobs.conclusion,
220 logs: workflowJobs.logs,
221 status: workflowJobs.status,
222 })
223 .from(workflowJobs)
224 .where(eq(workflowJobs.runId, runId));
225 return rows
226 .filter((r) => r.status === "failure" || r.conclusion === "failure")
227 .map((r) => ({
228 name: r.name,
229 conclusion: r.conclusion,
230 logs: r.logs || "",
231 }));
232 } catch (err) {
233 console.error("[ai-ci-healer] loadFailedJobs failed:", err);
234 return [];
235 }
236}
237
238/**
239 * Has this run already been processed (success or give-up)? We use the
240 * audit log as the marker store so we don't need a schema change.
241 */
242async function hasMarker(runId: string): Promise<boolean> {
243 try {
244 const [row] = await db
245 .select({ id: auditLog.id })
246 .from(auditLog)
247 .where(
248 and(
249 eq(auditLog.targetType, "workflow_run"),
250 eq(auditLog.targetId, runId),
251 sql`${auditLog.action} IN ('ai.ci.healed', 'ai.ci.gave_up')`
252 )
253 )
254 .limit(1);
255 return !!row;
256 } catch (err) {
257 console.warn("[ai-ci-healer] hasMarker query failed:", err);
258 // Fail closed — if we can't check, skip to avoid duplicate work.
259 return true;
260 }
261}
262
263// ---------------------------------------------------------------------------
264// analyzeFailedWorkflowRun — public entry point #1
265// ---------------------------------------------------------------------------
266
267export interface AnalyzeOptions {
268 /** Test-only Anthropic client injection. */
269 client?: Pick<Anthropic, "messages">;
270}
271
272/**
273 * Diagnose a failed run. Returns `null` when:
274 * - The run doesn't exist or isn't actually a failure.
275 * - ANTHROPIC_API_KEY is unset AND no client was injected.
276 * - Claude says the failure isn't fixable from this repo (env/infra).
277 * - Any DB / network step blows up (logged, swallowed).
278 */
279export async function analyzeFailedWorkflowRun(
280 runId: string,
281 opts: AnalyzeOptions = {}
282): Promise<CiHealAnalysis | null> {
283 // Resolve client lazily — tests inject, production reads env.
284 let client: Pick<Anthropic, "messages">;
285 if (opts.client) {
286 client = opts.client;
287 } else {
288 if (!isAiAvailable()) return null;
289 try {
290 client = getAnthropic();
291 } catch {
292 return null;
293 }
294 }
295
296 // Load run row.
297 let run: typeof workflowRuns.$inferSelect | null = null;
298 try {
299 const [row] = await db
300 .select()
301 .from(workflowRuns)
302 .where(eq(workflowRuns.id, runId))
303 .limit(1);
304 run = row || null;
305 } catch (err) {
306 console.error("[ai-ci-healer] loadRun failed:", err);
307 return null;
308 }
309 if (!run || run.status !== "failure") return null;
310
311 // Load workflow + repo (for YAML + naming context).
312 let workflowYaml = "";
313 let repoFullName = "unknown/unknown";
314 try {
315 const [w] = await db
316 .select({ yaml: workflows.yaml })
317 .from(workflows)
318 .where(eq(workflows.id, run.workflowId))
319 .limit(1);
320 if (w?.yaml) workflowYaml = w.yaml;
321 } catch (err) {
322 console.warn("[ai-ci-healer] load workflow failed:", err);
323 }
324 try {
325 const [r] = await db
326 .select({
327 name: repositories.name,
328 ownerId: repositories.ownerId,
329 })
330 .from(repositories)
331 .where(eq(repositories.id, run.repositoryId))
332 .limit(1);
333 if (r) {
334 repoFullName = `${r.ownerId.slice(0, 8)}/${r.name}`;
335 }
336 } catch (err) {
337 console.warn("[ai-ci-healer] load repo failed:", err);
338 }
339
340 const failedJobs = await loadFailedJobs(runId);
341
342 // Ask Claude.
343 let parsed: ClaudeCiResponse | null = null;
344 try {
345 const message = await client.messages.create({
346 model: MODEL_SONNET,
347 max_tokens: 2048,
348 messages: [
349 {
350 role: "user",
351 content: buildCiHealPrompt({
352 repoFullName,
353 commitSha: run.commitSha || "(unknown)",
354 workflowYaml,
355 failedJobs,
356 }),
357 },
358 ],
359 });
0c3eee5Claude360 try {
361 const { recordAiCost, extractUsage } = await import("./ai-cost-tracker");
362 const usage = extractUsage(message);
363 await recordAiCost({
364 repositoryId: run.repositoryId ?? null,
365 model: MODEL_SONNET,
366 inputTokens: usage.input,
367 outputTokens: usage.output,
368 category: "ci_healer",
369 sourceId: runId,
370 sourceKind: "workflow_run",
371 });
372 } catch {
373 /* swallow — best-effort */
374 }
9336f45Claude375 parsed = parseJsonResponse<ClaudeCiResponse>(extractText(message));
376 } catch (err) {
377 console.warn(
378 "[ai-ci-healer] Claude call failed:",
379 err instanceof Error ? err.message : err
380 );
381 return null;
382 }
383 if (!parsed) return null;
384
385 const rootCause =
386 typeof parsed.rootCause === "string" && parsed.rootCause.trim()
387 ? parsed.rootCause.trim()
388 : "(no root cause provided)";
389
390 // "Not fixable" branch — return null so the caller can mark `ai.ci.gave_up`.
391 if (parsed.fixable === false) {
392 return null;
393 }
394
395 const rawFixes = Array.isArray(parsed.suggestedFixes)
396 ? parsed.suggestedFixes
397 : [];
398 const suggestedFixes: SuggestedFix[] = rawFixes
399 .filter(
400 (f): f is { path: string; description?: string; severity?: string } =>
401 !!f && typeof f.path === "string" && f.path.trim().length > 0
402 )
403 .map((f) => ({
404 path: f.path.trim(),
405 description:
406 typeof f.description === "string" && f.description.trim()
407 ? f.description.trim()
408 : rootCause,
409 severity: normaliseSeverity(f.severity),
410 }));
411
412 // Claude said fixable but produced zero usable paths → treat as
413 // unfixable so we don't loop.
414 if (suggestedFixes.length === 0) return null;
415
416 const patchablePaths = Array.from(new Set(suggestedFixes.map((f) => f.path)));
417
418 return { rootCause, suggestedFixes, patchablePaths };
419}
420
421// ---------------------------------------------------------------------------
422// healOneRun — drives one run end-to-end. Public so callers (autopilot,
423// tests, manual retrigger) share the same pipeline.
424// ---------------------------------------------------------------------------
425
426export interface HealOneOptions extends AnalyzeOptions {
427 /** Test override — pin the patch generator output. */
428 generatePatch?: typeof generatePatchForGateTestFinding;
429}
430
431export interface HealOneResult {
432 outcome: "healed" | "gave_up" | "skipped";
433 prNumber?: number;
434 branch?: string;
435 reason?: string;
436}
437
438export async function healOneRun(
439 runId: string,
440 opts: HealOneOptions = {}
441): Promise<HealOneResult> {
442 if (process.env.AUTOPILOT_DISABLED === "1") {
443 return { outcome: "skipped", reason: "autopilot disabled" };
444 }
445 if (!opts.client && !isAiAvailable()) {
446 return { outcome: "skipped", reason: "ANTHROPIC_API_KEY missing" };
447 }
448
449 // Skip if already processed (audit marker present).
450 if (await hasMarker(runId)) {
451 return { outcome: "skipped", reason: "already processed" };
452 }
453
0b49751Claude454 // Look up the run for audit metadata (repo + sha) and owner for quota check.
9336f45Claude455 let repositoryId: string | null = null;
456 let commitSha: string | null = null;
0b49751Claude457 let repoOwnerId: string | null = null;
9336f45Claude458 try {
459 const [row] = await db
460 .select({
461 repositoryId: workflowRuns.repositoryId,
462 commitSha: workflowRuns.commitSha,
463 })
464 .from(workflowRuns)
465 .where(eq(workflowRuns.id, runId))
466 .limit(1);
467 if (row) {
468 repositoryId = row.repositoryId;
469 commitSha = row.commitSha;
470 }
471 } catch (err) {
472 console.warn("[ai-ci-healer] post-analyze run lookup failed:", err);
473 }
474
0b49751Claude475 // Resolve repo owner for the quota check.
476 if (repositoryId) {
477 try {
478 const [repoRow] = await db
479 .select({ ownerId: repositories.ownerId })
480 .from(repositories)
481 .where(eq(repositories.id, repositoryId))
482 .limit(1);
483 if (repoRow) repoOwnerId = repoRow.ownerId;
484 } catch {
485 /* tolerate */
486 }
487 }
488
489 // Hard quota gate — skip silently when the repo owner's AI budget is
490 // exhausted. We don't fail the CI run; the autopilot marker is not written
491 // so the healer will retry on the next tick once budget resets.
492 if (repoOwnerId) {
493 try {
494 await assertAiQuota(repoOwnerId);
495 } catch (err) {
496 if (err instanceof AiQuotaExceededError) {
497 console.log(
498 `[ai-ci-healer] skipping run=${runId}: AI quota exceeded for owner=${repoOwnerId}`
499 );
500 return { outcome: "skipped", reason: "AI quota exceeded" };
501 }
502 // Unexpected error — log and proceed (fail open).
503 console.warn("[ai-ci-healer] assertAiQuota failed unexpectedly:", err);
504 }
505 }
506
507 const analysis = await analyzeFailedWorkflowRun(runId, {
508 client: opts.client,
509 });
510
9336f45Claude511 if (!analysis) {
512 await audit({
513 userId: null,
514 repositoryId,
515 action: "ai.ci.gave_up",
516 targetType: "workflow_run",
517 targetId: runId,
518 metadata: { commitSha, reason: "unfixable or analysis returned null" },
519 });
520 return { outcome: "gave_up", reason: "unfixable" };
521 }
522
523 if (!repositoryId || !commitSha) {
524 // No base sha → can't seed a patch branch. Mark gave_up so we don't loop.
525 await audit({
526 userId: null,
527 repositoryId,
528 action: "ai.ci.gave_up",
529 targetType: "workflow_run",
530 targetId: runId,
531 metadata: {
532 commitSha,
533 reason: "missing repositoryId or commitSha for patch branch",
534 },
535 });
536 return { outcome: "gave_up", reason: "missing base sha" };
537 }
538
539 const generator = opts.generatePatch ?? generatePatchForGateTestFinding;
540 const findings = fixesToFindings(analysis.suggestedFixes, runId.slice(0, 8));
541
542 const patch = await generator({
543 repositoryId,
544 baseSha: commitSha,
545 findings,
546 client: opts.client,
547 });
548
549 if (!patch) {
550 await audit({
551 userId: null,
552 repositoryId,
553 action: "ai.ci.gave_up",
554 targetType: "workflow_run",
555 targetId: runId,
556 metadata: {
557 commitSha,
558 reason: "patch generator returned null",
559 rootCause: analysis.rootCause,
560 patchablePaths: analysis.patchablePaths,
561 },
562 });
563 return { outcome: "gave_up", reason: "patch generator returned null" };
564 }
565
566 await audit({
567 userId: null,
568 repositoryId,
569 action: "ai.ci.healed",
570 targetType: "workflow_run",
571 targetId: runId,
572 metadata: {
573 commitSha,
574 rootCause: analysis.rootCause,
575 patchablePaths: analysis.patchablePaths,
576 prNumber: patch.prNumber,
577 branch: patch.branch,
578 },
579 });
580
581 return {
582 outcome: "healed",
583 prNumber: patch.prNumber,
584 branch: patch.branch,
585 };
586}
587
588// ---------------------------------------------------------------------------
589// runCiHealerTick — public autopilot entry point
590// ---------------------------------------------------------------------------
591
592export interface CiHealerTickDeps {
593 /** Inject a candidate finder for tests. */
594 findCandidates?: (limit: number) => Promise<{ id: string }[]>;
595 /** Inject the per-run handler for tests. */
596 healOne?: (runId: string) => Promise<HealOneResult>;
597 /** Inject the per-tick cap. */
598 cap?: number;
599}
600
601/**
602 * Find failed workflow runs that:
603 * - finished at least HEAL_MIN_AGE_MS ago (let the runner flush logs)
604 * - finished less than HEAL_MAX_AGE_MS ago (don't chase ancient failures)
605 * - do NOT yet have an ai.ci.healed / ai.ci.gave_up audit marker
606 *
607 * The `hasMarker` filter is applied per-row in `healOneRun` rather than
608 * the SQL select — keeps the query simple and the index on (status,
609 * createdAt) doing most of the work.
610 */
611async function defaultFindCandidates(
612 limit: number
613): Promise<{ id: string }[]> {
614 const now = Date.now();
615 const cutoffNew = new Date(now - HEAL_MIN_AGE_MS);
616 const cutoffOld = new Date(now - HEAL_MAX_AGE_MS);
617 try {
618 const rows = await db
619 .select({ id: workflowRuns.id })
620 .from(workflowRuns)
621 .where(
622 and(
623 eq(workflowRuns.status, "failure"),
624 lt(workflowRuns.createdAt, cutoffNew),
625 gte(workflowRuns.createdAt, cutoffOld)
626 )
627 )
628 .orderBy(desc(workflowRuns.createdAt))
629 .limit(limit);
630 return rows;
631 } catch (err) {
632 console.error("[ai-ci-healer] candidate query failed:", err);
633 return [];
634 }
635}
636
637/**
638 * One autopilot tick: scan recent failures, heal what we can. Returns a
639 * counts summary. Never throws.
640 *
641 * No-op when:
642 * - AUTOPILOT_DISABLED=1
643 * - ANTHROPIC_API_KEY is unset
644 */
645export async function runCiHealerTick(
646 deps: CiHealerTickDeps = {}
647): Promise<CiHealerTickSummary> {
648 if (process.env.AUTOPILOT_DISABLED === "1") {
649 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
650 }
651 if (!isAiAvailable()) {
652 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
653 }
654
655 const findCandidates = deps.findCandidates ?? defaultFindCandidates;
656 const healOne = deps.healOne ?? ((id: string) => healOneRun(id));
657 const cap = deps.cap ?? HEAL_CAP_PER_TICK;
658
659 let candidates: { id: string }[] = [];
660 try {
661 candidates = await findCandidates(cap);
662 } catch (err) {
663 console.error("[ai-ci-healer] findCandidates threw:", err);
664 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
665 }
666
667 let healed = 0;
668 let gaveUp = 0;
669 let skipped = 0;
670 for (const cand of candidates) {
671 try {
672 const res = await healOne(cand.id);
673 if (res.outcome === "healed") healed += 1;
674 else if (res.outcome === "gave_up") gaveUp += 1;
675 else skipped += 1;
676 } catch (err) {
677 skipped += 1;
678 console.error(
679 `[ai-ci-healer] per-run failure for run=${cand.id}:`,
680 err instanceof Error ? err.message : err
681 );
682 }
683 }
684
685 return { considered: candidates.length, healed, gaveUp, skipped };
686}
687
688// ---------------------------------------------------------------------------
689// Test-only re-exports
690// ---------------------------------------------------------------------------
691
692export const __test = {
693 loadFailedJobs,
694 hasMarker,
695 defaultFindCandidates,
696 normaliseSeverity,
697 truncate,
698 HEAL_MIN_AGE_MS,
699 HEAL_MAX_AGE_MS,
700 HEAL_CAP_PER_TICK,
701};