Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesPull RequestsActionsSecurityInsightsSettings
✨ AI
ExplainAsk AIWorkspaceSpec to PRTestsNL SearchDebt MapArchaeology
More
CommitsDiscussionsWikiProjectsReleasesContributorsPulseGatesDeploymentsPipelineAgents
Blame · Line-by-line history

ai-ci-healer.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/ai-ci-healer.ts
ai-ci-healer.tsBlame652 lines · 1 contributor
Normal viewRaw
9336f45Claude1/**
2 * AI CI Healer — autonomous failure → root-cause → patch PR loop.
3 *
4 * When a `workflow_runs` row lands in status="failure", this module:
5 * 1. Pulls the run + all its `workflow_jobs` (especially the failed ones'
6 * `logs` column).
7 * 2. Asks Claude to identify the root cause + whether it's fixable from
8 * inside this repo.
9 * 3. If patchable, hands the suggested fixes off to the existing
10 * `generatePatchForGateTestFinding` (the finding-shape maps cleanly
11 * onto its `GateTestFinding[]` API — both surfaces want
12 * `{ path, description, severity }`).
13 * 4. Records an `ai.ci.healed` or `ai.ci.gave_up` audit row keyed on the
14 * run id so the autopilot poller doesn't re-process the same failure
15 * every 5 minutes.
16 *
17 * Degrades silently when ANTHROPIC_API_KEY is unset (autopilot also gates
18 * on the env var, but the lib double-checks so it's safe to call directly).
19 * Everything is wrapped in try/catch — `analyzeFailedWorkflowRun` returns
20 * `null` rather than throwing on any failure.
21 *
22 * Marker convention:
23 * - Successful patch open → audit action `ai.ci.healed`, targetId = runId.
24 * - Claude says unfixable → audit action `ai.ci.gave_up`, targetId = runId.
25 * The autopilot poller skips any run that already has either marker, so we
26 * never retry forever.
27 */
28
29import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
30import type Anthropic from "@anthropic-ai/sdk";
31import { db } from "../db";
32import {
33 auditLog,
34 repositories,
35 workflowJobs,
36 workflowRuns,
37 workflows,
38} from "../db/schema";
39import {
40 MODEL_SONNET,
41 extractText,
42 getAnthropic,
43 isAiAvailable,
44 parseJsonResponse,
45} from "./ai-client";
46import { audit } from "./notify";
47import {
48 generatePatchForGateTestFinding,
49 type GateTestFinding,
50} from "./ai-patch-generator";
51
52// ---------------------------------------------------------------------------
53// Tunables
54// ---------------------------------------------------------------------------
55
56/** Cap per-job log slice we ship to Claude — protect the prompt budget. */
57const LOG_SNIPPET_BYTES = 8 * 1024;
58
59/** Hard cap on runs processed per autopilot tick — runaway protection. */
60const HEAL_CAP_PER_TICK = 5;
61
62/** Only consider runs that finished at least this long ago. Gives the
63 * workflow runner time to flush logs + final job rows before we sample. */
64const HEAL_MIN_AGE_MS = 60 * 1_000;
65
66/** Don't bother healing ancient failures — older than this and we assume
67 * the human has already triaged or the SHA has been rewritten. */
68const HEAL_MAX_AGE_MS = 24 * 60 * 60 * 1_000;
69
70// ---------------------------------------------------------------------------
71// Public surface
72// ---------------------------------------------------------------------------
73
74export interface SuggestedFix {
75 /** Relative path inside the repo that the AI thinks needs touching. */
76 path: string;
77 /** What's wrong / what the fix should look like (one-liner). */
78 description: string;
79 /** Severity inherited from the AI's confidence call. */
80 severity?: "low" | "medium" | "high" | "critical";
81}
82
83export interface CiHealAnalysis {
84 /** Plain-English root cause, 1-3 sentences. */
85 rootCause: string;
86 /** Concrete fix proposals (may be empty if Claude can't pinpoint files). */
87 suggestedFixes: SuggestedFix[];
88 /** Convenience: the paths from `suggestedFixes` deduped. */
89 patchablePaths: string[];
90}
91
92export interface CiHealerTickSummary {
93 considered: number;
94 healed: number;
95 gaveUp: number;
96 skipped: number;
97}
98
99/**
100 * Claude's response shape. Kept loose so we tolerate minor schema drift.
101 */
102interface ClaudeCiResponse {
103 rootCause?: string;
104 fixable?: boolean;
105 suggestedFixes?: Array<{
106 path?: string;
107 description?: string;
108 severity?: string;
109 }>;
110 /** Optional: Claude's reason it gave up. We just persist it for ops. */
111 unfixableReason?: string;
112}
113
114// ---------------------------------------------------------------------------
115// Helpers
116// ---------------------------------------------------------------------------
117
118function truncate(s: string | null | undefined, max: number): string {
119 if (!s) return "";
120 if (s.length <= max) return s;
121 return s.slice(0, max) + "\n…(truncated)";
122}
123
124function normaliseSeverity(s: unknown): SuggestedFix["severity"] | undefined {
125 if (typeof s !== "string") return undefined;
126 const v = s.toLowerCase();
127 if (v === "low" || v === "medium" || v === "high" || v === "critical") {
128 return v;
129 }
130 return undefined;
131}
132
133/**
134 * Map our AI-derived `SuggestedFix` set onto the
135 * `GateTestFinding[]` shape that `generatePatchForGateTestFinding`
136 * already accepts. The patch generator only needs `{path, description,
137 * severity}` — exactly what we already have. Each fix becomes one
138 * candidate finding; the generator stops at the first one that produces a
139 * non-empty patch set so a bad suggestion doesn't drown a good one.
140 */
141export function fixesToFindings(
142 fixes: SuggestedFix[],
143 runShortId: string
144): GateTestFinding[] {
145 return fixes
146 .filter((f) => f.path && f.path.trim().length > 0)
147 .map((f, i) => ({
148 id: `ci-heal-${runShortId}-${i}`,
149 ruleId: "ci-failure",
150 path: f.path.trim(),
151 severity: f.severity || "high",
152 title: "CI failure auto-heal",
153 description: f.description || "CI failure",
154 }));
155}
156
157/**
158 * Build the Claude prompt. Pure function so tests can pin the shape.
159 */
160export function buildCiHealPrompt(args: {
161 repoFullName: string;
162 commitSha: string;
163 workflowYaml: string;
164 failedJobs: Array<{ name: string; conclusion: string | null; logs: string }>;
165}): string {
166 const jobsBlock = args.failedJobs
167 .map(
168 (j) =>
169 `### Job: ${j.name} (${j.conclusion || "failure"})\n\`\`\`\n${truncate(
170 j.logs,
171 LOG_SNIPPET_BYTES
172 )}\n\`\`\``
173 )
174 .join("\n\n");
175 return [
176 "You are GlueCron's CI healer. A workflow run just failed. Decide whether the failure is a code bug fixable in THIS repository, and if so propose concrete file edits.",
177 "",
178 `**Repository:** ${args.repoFullName}`,
179 `**Commit:** ${args.commitSha.slice(0, 12)}`,
180 "",
181 "## Workflow YAML",
182 "```yaml",
183 truncate(args.workflowYaml, 4_000),
184 "```",
185 "",
186 "## Failed job logs",
187 jobsBlock || "(no failed-job logs available)",
188 "",
189 "Respond ONLY with JSON of this exact shape:",
190 "{",
191 ' "rootCause": "1-3 sentence diagnosis (what failed, why)",',
192 ' "fixable": true | false,',
193 ' "suggestedFixes": [',
194 ' { "path": "src/foo.ts", "description": "what to change", "severity": "high" }',
195 " ],",
196 ' "unfixableReason": "(only if fixable=false) why the human must intervene"',
197 "}",
198 "",
199 "Rules:",
200 "- `fixable` MUST be false if the failure is an env/infra/external-service problem (missing secret, registry down, network, GitHub Actions runner image, etc.) — anything you can't fix by editing files in this repo.",
201 "- `suggestedFixes` must be empty when fixable=false.",
202 "- Only suggest paths you can identify with high confidence from the logs or YAML. Do not guess at random files.",
203 "- Keep `description` short — the patch generator will be invoked with this finding to produce the actual diff.",
204 ].join("\n");
205}
206
207interface FailedJobRow {
208 name: string;
209 conclusion: string | null;
210 logs: string;
211}
212
213async function loadFailedJobs(runId: string): Promise<FailedJobRow[]> {
214 try {
215 const rows = await db
216 .select({
217 name: workflowJobs.name,
218 conclusion: workflowJobs.conclusion,
219 logs: workflowJobs.logs,
220 status: workflowJobs.status,
221 })
222 .from(workflowJobs)
223 .where(eq(workflowJobs.runId, runId));
224 return rows
225 .filter((r) => r.status === "failure" || r.conclusion === "failure")
226 .map((r) => ({
227 name: r.name,
228 conclusion: r.conclusion,
229 logs: r.logs || "",
230 }));
231 } catch (err) {
232 console.error("[ai-ci-healer] loadFailedJobs failed:", err);
233 return [];
234 }
235}
236
237/**
238 * Has this run already been processed (success or give-up)? We use the
239 * audit log as the marker store so we don't need a schema change.
240 */
241async function hasMarker(runId: string): Promise<boolean> {
242 try {
243 const [row] = await db
244 .select({ id: auditLog.id })
245 .from(auditLog)
246 .where(
247 and(
248 eq(auditLog.targetType, "workflow_run"),
249 eq(auditLog.targetId, runId),
250 sql`${auditLog.action} IN ('ai.ci.healed', 'ai.ci.gave_up')`
251 )
252 )
253 .limit(1);
254 return !!row;
255 } catch (err) {
256 console.warn("[ai-ci-healer] hasMarker query failed:", err);
257 // Fail closed — if we can't check, skip to avoid duplicate work.
258 return true;
259 }
260}
261
262// ---------------------------------------------------------------------------
263// analyzeFailedWorkflowRun — public entry point #1
264// ---------------------------------------------------------------------------
265
266export interface AnalyzeOptions {
267 /** Test-only Anthropic client injection. */
268 client?: Pick<Anthropic, "messages">;
269}
270
271/**
272 * Diagnose a failed run. Returns `null` when:
273 * - The run doesn't exist or isn't actually a failure.
274 * - ANTHROPIC_API_KEY is unset AND no client was injected.
275 * - Claude says the failure isn't fixable from this repo (env/infra).
276 * - Any DB / network step blows up (logged, swallowed).
277 */
278export async function analyzeFailedWorkflowRun(
279 runId: string,
280 opts: AnalyzeOptions = {}
281): Promise<CiHealAnalysis | null> {
282 // Resolve client lazily — tests inject, production reads env.
283 let client: Pick<Anthropic, "messages">;
284 if (opts.client) {
285 client = opts.client;
286 } else {
287 if (!isAiAvailable()) return null;
288 try {
289 client = getAnthropic();
290 } catch {
291 return null;
292 }
293 }
294
295 // Load run row.
296 let run: typeof workflowRuns.$inferSelect | null = null;
297 try {
298 const [row] = await db
299 .select()
300 .from(workflowRuns)
301 .where(eq(workflowRuns.id, runId))
302 .limit(1);
303 run = row || null;
304 } catch (err) {
305 console.error("[ai-ci-healer] loadRun failed:", err);
306 return null;
307 }
308 if (!run || run.status !== "failure") return null;
309
310 // Load workflow + repo (for YAML + naming context).
311 let workflowYaml = "";
312 let repoFullName = "unknown/unknown";
313 try {
314 const [w] = await db
315 .select({ yaml: workflows.yaml })
316 .from(workflows)
317 .where(eq(workflows.id, run.workflowId))
318 .limit(1);
319 if (w?.yaml) workflowYaml = w.yaml;
320 } catch (err) {
321 console.warn("[ai-ci-healer] load workflow failed:", err);
322 }
323 try {
324 const [r] = await db
325 .select({
326 name: repositories.name,
327 ownerId: repositories.ownerId,
328 })
329 .from(repositories)
330 .where(eq(repositories.id, run.repositoryId))
331 .limit(1);
332 if (r) {
333 repoFullName = `${r.ownerId.slice(0, 8)}/${r.name}`;
334 }
335 } catch (err) {
336 console.warn("[ai-ci-healer] load repo failed:", err);
337 }
338
339 const failedJobs = await loadFailedJobs(runId);
340
341 // Ask Claude.
342 let parsed: ClaudeCiResponse | null = null;
343 try {
344 const message = await client.messages.create({
345 model: MODEL_SONNET,
346 max_tokens: 2048,
347 messages: [
348 {
349 role: "user",
350 content: buildCiHealPrompt({
351 repoFullName,
352 commitSha: run.commitSha || "(unknown)",
353 workflowYaml,
354 failedJobs,
355 }),
356 },
357 ],
358 });
359 parsed = parseJsonResponse<ClaudeCiResponse>(extractText(message));
360 } catch (err) {
361 console.warn(
362 "[ai-ci-healer] Claude call failed:",
363 err instanceof Error ? err.message : err
364 );
365 return null;
366 }
367 if (!parsed) return null;
368
369 const rootCause =
370 typeof parsed.rootCause === "string" && parsed.rootCause.trim()
371 ? parsed.rootCause.trim()
372 : "(no root cause provided)";
373
374 // "Not fixable" branch — return null so the caller can mark `ai.ci.gave_up`.
375 if (parsed.fixable === false) {
376 return null;
377 }
378
379 const rawFixes = Array.isArray(parsed.suggestedFixes)
380 ? parsed.suggestedFixes
381 : [];
382 const suggestedFixes: SuggestedFix[] = rawFixes
383 .filter(
384 (f): f is { path: string; description?: string; severity?: string } =>
385 !!f && typeof f.path === "string" && f.path.trim().length > 0
386 )
387 .map((f) => ({
388 path: f.path.trim(),
389 description:
390 typeof f.description === "string" && f.description.trim()
391 ? f.description.trim()
392 : rootCause,
393 severity: normaliseSeverity(f.severity),
394 }));
395
396 // Claude said fixable but produced zero usable paths → treat as
397 // unfixable so we don't loop.
398 if (suggestedFixes.length === 0) return null;
399
400 const patchablePaths = Array.from(new Set(suggestedFixes.map((f) => f.path)));
401
402 return { rootCause, suggestedFixes, patchablePaths };
403}
404
405// ---------------------------------------------------------------------------
406// healOneRun — drives one run end-to-end. Public so callers (autopilot,
407// tests, manual retrigger) share the same pipeline.
408// ---------------------------------------------------------------------------
409
410export interface HealOneOptions extends AnalyzeOptions {
411 /** Test override — pin the patch generator output. */
412 generatePatch?: typeof generatePatchForGateTestFinding;
413}
414
415export interface HealOneResult {
416 outcome: "healed" | "gave_up" | "skipped";
417 prNumber?: number;
418 branch?: string;
419 reason?: string;
420}
421
422export async function healOneRun(
423 runId: string,
424 opts: HealOneOptions = {}
425): Promise<HealOneResult> {
426 if (process.env.AUTOPILOT_DISABLED === "1") {
427 return { outcome: "skipped", reason: "autopilot disabled" };
428 }
429 if (!opts.client && !isAiAvailable()) {
430 return { outcome: "skipped", reason: "ANTHROPIC_API_KEY missing" };
431 }
432
433 // Skip if already processed (audit marker present).
434 if (await hasMarker(runId)) {
435 return { outcome: "skipped", reason: "already processed" };
436 }
437
438 const analysis = await analyzeFailedWorkflowRun(runId, {
439 client: opts.client,
440 });
441
442 // Look up the run for audit metadata (repo + sha).
443 let repositoryId: string | null = null;
444 let commitSha: string | null = null;
445 try {
446 const [row] = await db
447 .select({
448 repositoryId: workflowRuns.repositoryId,
449 commitSha: workflowRuns.commitSha,
450 })
451 .from(workflowRuns)
452 .where(eq(workflowRuns.id, runId))
453 .limit(1);
454 if (row) {
455 repositoryId = row.repositoryId;
456 commitSha = row.commitSha;
457 }
458 } catch (err) {
459 console.warn("[ai-ci-healer] post-analyze run lookup failed:", err);
460 }
461
462 if (!analysis) {
463 await audit({
464 userId: null,
465 repositoryId,
466 action: "ai.ci.gave_up",
467 targetType: "workflow_run",
468 targetId: runId,
469 metadata: { commitSha, reason: "unfixable or analysis returned null" },
470 });
471 return { outcome: "gave_up", reason: "unfixable" };
472 }
473
474 if (!repositoryId || !commitSha) {
475 // No base sha → can't seed a patch branch. Mark gave_up so we don't loop.
476 await audit({
477 userId: null,
478 repositoryId,
479 action: "ai.ci.gave_up",
480 targetType: "workflow_run",
481 targetId: runId,
482 metadata: {
483 commitSha,
484 reason: "missing repositoryId or commitSha for patch branch",
485 },
486 });
487 return { outcome: "gave_up", reason: "missing base sha" };
488 }
489
490 const generator = opts.generatePatch ?? generatePatchForGateTestFinding;
491 const findings = fixesToFindings(analysis.suggestedFixes, runId.slice(0, 8));
492
493 const patch = await generator({
494 repositoryId,
495 baseSha: commitSha,
496 findings,
497 client: opts.client,
498 });
499
500 if (!patch) {
501 await audit({
502 userId: null,
503 repositoryId,
504 action: "ai.ci.gave_up",
505 targetType: "workflow_run",
506 targetId: runId,
507 metadata: {
508 commitSha,
509 reason: "patch generator returned null",
510 rootCause: analysis.rootCause,
511 patchablePaths: analysis.patchablePaths,
512 },
513 });
514 return { outcome: "gave_up", reason: "patch generator returned null" };
515 }
516
517 await audit({
518 userId: null,
519 repositoryId,
520 action: "ai.ci.healed",
521 targetType: "workflow_run",
522 targetId: runId,
523 metadata: {
524 commitSha,
525 rootCause: analysis.rootCause,
526 patchablePaths: analysis.patchablePaths,
527 prNumber: patch.prNumber,
528 branch: patch.branch,
529 },
530 });
531
532 return {
533 outcome: "healed",
534 prNumber: patch.prNumber,
535 branch: patch.branch,
536 };
537}
538
539// ---------------------------------------------------------------------------
540// runCiHealerTick — public autopilot entry point
541// ---------------------------------------------------------------------------
542
543export interface CiHealerTickDeps {
544 /** Inject a candidate finder for tests. */
545 findCandidates?: (limit: number) => Promise<{ id: string }[]>;
546 /** Inject the per-run handler for tests. */
547 healOne?: (runId: string) => Promise<HealOneResult>;
548 /** Inject the per-tick cap. */
549 cap?: number;
550}
551
552/**
553 * Find failed workflow runs that:
554 * - finished at least HEAL_MIN_AGE_MS ago (let the runner flush logs)
555 * - finished less than HEAL_MAX_AGE_MS ago (don't chase ancient failures)
556 * - do NOT yet have an ai.ci.healed / ai.ci.gave_up audit marker
557 *
558 * The `hasMarker` filter is applied per-row in `healOneRun` rather than
559 * the SQL select — keeps the query simple and the index on (status,
560 * createdAt) doing most of the work.
561 */
562async function defaultFindCandidates(
563 limit: number
564): Promise<{ id: string }[]> {
565 const now = Date.now();
566 const cutoffNew = new Date(now - HEAL_MIN_AGE_MS);
567 const cutoffOld = new Date(now - HEAL_MAX_AGE_MS);
568 try {
569 const rows = await db
570 .select({ id: workflowRuns.id })
571 .from(workflowRuns)
572 .where(
573 and(
574 eq(workflowRuns.status, "failure"),
575 lt(workflowRuns.createdAt, cutoffNew),
576 gte(workflowRuns.createdAt, cutoffOld)
577 )
578 )
579 .orderBy(desc(workflowRuns.createdAt))
580 .limit(limit);
581 return rows;
582 } catch (err) {
583 console.error("[ai-ci-healer] candidate query failed:", err);
584 return [];
585 }
586}
587
588/**
589 * One autopilot tick: scan recent failures, heal what we can. Returns a
590 * counts summary. Never throws.
591 *
592 * No-op when:
593 * - AUTOPILOT_DISABLED=1
594 * - ANTHROPIC_API_KEY is unset
595 */
596export async function runCiHealerTick(
597 deps: CiHealerTickDeps = {}
598): Promise<CiHealerTickSummary> {
599 if (process.env.AUTOPILOT_DISABLED === "1") {
600 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
601 }
602 if (!isAiAvailable()) {
603 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
604 }
605
606 const findCandidates = deps.findCandidates ?? defaultFindCandidates;
607 const healOne = deps.healOne ?? ((id: string) => healOneRun(id));
608 const cap = deps.cap ?? HEAL_CAP_PER_TICK;
609
610 let candidates: { id: string }[] = [];
611 try {
612 candidates = await findCandidates(cap);
613 } catch (err) {
614 console.error("[ai-ci-healer] findCandidates threw:", err);
615 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
616 }
617
618 let healed = 0;
619 let gaveUp = 0;
620 let skipped = 0;
621 for (const cand of candidates) {
622 try {
623 const res = await healOne(cand.id);
624 if (res.outcome === "healed") healed += 1;
625 else if (res.outcome === "gave_up") gaveUp += 1;
626 else skipped += 1;
627 } catch (err) {
628 skipped += 1;
629 console.error(
630 `[ai-ci-healer] per-run failure for run=${cand.id}:`,
631 err instanceof Error ? err.message : err
632 );
633 }
634 }
635
636 return { considered: candidates.length, healed, gaveUp, skipped };
637}
638
639// ---------------------------------------------------------------------------
640// Test-only re-exports
641// ---------------------------------------------------------------------------
642
643export const __test = {
644 loadFailedJobs,
645 hasMarker,
646 defaultFindCandidates,
647 normaliseSeverity,
648 truncate,
649 HEAL_MIN_AGE_MS,
650 HEAL_MAX_AGE_MS,
651 HEAL_CAP_PER_TICK,
652};