Commit9336f45unknown_key
feat(ai-ci-healer): autonomous CI failure → root-cause → patch PR loop
3 files changed+1184−09336f45d277026f0af2e7933e44715f941e1c460
3 changed files+1184−0
Addedsrc/__tests__/ai-ci-healer.test.ts+511−0View fileUnifiedSplit
@@ -0,0 +1,511 @@
1/**
2 * Tests for src/lib/ai-ci-healer.ts.
3 *
4 * Two layers:
5 * 1. Pure helpers — buildCiHealPrompt + fixesToFindings + small private
6 * helpers exposed via __test. Always run.
7 * 2. End-to-end with a fake Claude client + a real DB row + bare repo.
8 * Gated on DATABASE_URL via the HAS_DB skipIf pattern used across
9 * the suite.
10 *
11 * The Anthropic client is faked via the public `client` option so we never
12 * touch the network or require ANTHROPIC_API_KEY. We also DI the
13 * patch-generator into `healOneRun` so each test pins its own outcome.
14 */
15
16import { describe, it, expect, beforeAll, afterAll } from "bun:test";
17import { join } from "path";
18import { mkdir, rm } from "fs/promises";
19import { eq, and } from "drizzle-orm";
20import {
21 analyzeFailedWorkflowRun,
22 buildCiHealPrompt,
23 fixesToFindings,
24 healOneRun,
25 runCiHealerTick,
26 __test,
27} from "../lib/ai-ci-healer";
28import { db } from "../db";
29import {
30 auditLog,
31 pullRequests,
32 repositories,
33 users,
34 workflowJobs,
35 workflowRuns,
36 workflows,
37} from "../db/schema";
38import {
39 createOrUpdateFileOnBranch,
40 initBareRepo,
41} from "../git/repository";
42
43const HAS_DB = Boolean(process.env.DATABASE_URL);
44
45const TEST_REPOS = join(
46 import.meta.dir,
47 "../../.test-repos-ai-ci-healer-" + Date.now()
48);
49
50beforeAll(async () => {
51 process.env.GIT_REPOS_PATH = TEST_REPOS;
52 process.env.DATABASE_URL = process.env.DATABASE_URL || "";
53 // Make sure autopilot doesn't short-circuit our tests via env.
54 delete process.env.AUTOPILOT_DISABLED;
55 await rm(TEST_REPOS, { recursive: true, force: true });
56 await mkdir(TEST_REPOS, { recursive: true });
57});
58
59afterAll(async () => {
60 await rm(TEST_REPOS, { recursive: true, force: true });
61});
62
63// ---------------------------------------------------------------------------
64// Pure helpers
65// ---------------------------------------------------------------------------
66
67describe("buildCiHealPrompt", () => {
68 it("embeds repo, sha, yaml, and failed job logs", () => {
69 const prompt = buildCiHealPrompt({
70 repoFullName: "alice/web",
71 commitSha: "abcdef1234567890",
72 workflowYaml: "name: ci\non: [push]\njobs:\n build:\n steps: []",
73 failedJobs: [
74 { name: "build", conclusion: "failure", logs: "TypeError: x is not a function" },
75 ],
76 });
77 expect(prompt).toContain("alice/web");
78 expect(prompt).toContain("abcdef123456");
79 expect(prompt).toContain("name: ci");
80 expect(prompt).toContain("TypeError: x is not a function");
81 // Strict JSON schema cue
82 expect(prompt).toContain('"rootCause"');
83 expect(prompt).toContain('"suggestedFixes"');
84 });
85
86 it("handles the empty-jobs case gracefully", () => {
87 const prompt = buildCiHealPrompt({
88 repoFullName: "x/y",
89 commitSha: "1234567",
90 workflowYaml: "name: ci",
91 failedJobs: [],
92 });
93 expect(prompt).toContain("(no failed-job logs available)");
94 });
95});
96
97describe("fixesToFindings", () => {
98 it("maps SuggestedFix[] onto GateTestFinding[] preserving path + severity", () => {
99 const findings = fixesToFindings(
100 [
101 { path: "src/a.ts", description: "fix imports", severity: "high" },
102 { path: "src/b.ts", description: "guard null", severity: "medium" },
103 ],
104 "deadbeef"
105 );
106 expect(findings.length).toBe(2);
107 expect(findings[0].path).toBe("src/a.ts");
108 expect(findings[0].severity).toBe("high");
109 expect(findings[0].id).toContain("ci-heal-deadbeef");
110 expect(findings[1].path).toBe("src/b.ts");
111 expect(findings[1].severity).toBe("medium");
112 });
113
114 it("drops fixes with no path", () => {
115 const findings = fixesToFindings(
116 [
117 { path: "", description: "noop" },
118 { path: "src/c.ts", description: "ok" },
119 ],
120 "x"
121 );
122 expect(findings.length).toBe(1);
123 expect(findings[0].path).toBe("src/c.ts");
124 });
125
126 it("defaults severity to high when omitted", () => {
127 const findings = fixesToFindings(
128 [{ path: "src/d.ts", description: "no sev" }],
129 "x"
130 );
131 expect(findings[0].severity).toBe("high");
132 });
133});
134
135describe("__test internals", () => {
136 it("normaliseSeverity accepts the canonical levels case-insensitively", () => {
137 expect(__test.normaliseSeverity("HIGH")).toBe("high");
138 expect(__test.normaliseSeverity("Medium")).toBe("medium");
139 expect(__test.normaliseSeverity("critical")).toBe("critical");
140 expect(__test.normaliseSeverity("low")).toBe("low");
141 });
142
143 it("normaliseSeverity rejects garbage", () => {
144 expect(__test.normaliseSeverity("bananas")).toBeUndefined();
145 expect(__test.normaliseSeverity(undefined)).toBeUndefined();
146 expect(__test.normaliseSeverity(null)).toBeUndefined();
147 expect(__test.normaliseSeverity(42)).toBeUndefined();
148 });
149
150 it("truncate caps long strings + appends marker", () => {
151 expect(__test.truncate("abc", 10)).toBe("abc");
152 const long = "x".repeat(20);
153 const t = __test.truncate(long, 5);
154 expect(t.startsWith("xxxxx")).toBe(true);
155 expect(t).toContain("(truncated)");
156 });
157});
158
159// ---------------------------------------------------------------------------
160// Skip when AUTOPILOT_DISABLED: tick must no-op cleanly
161// ---------------------------------------------------------------------------
162
163describe("runCiHealerTick — env gates", () => {
164 it("no-ops when AUTOPILOT_DISABLED=1", async () => {
165 const prev = process.env.AUTOPILOT_DISABLED;
166 process.env.AUTOPILOT_DISABLED = "1";
167 try {
168 const summary = await runCiHealerTick({
169 findCandidates: async () => {
170 throw new Error("should not be called");
171 },
172 });
173 expect(summary).toEqual({
174 considered: 0,
175 healed: 0,
176 gaveUp: 0,
177 skipped: 0,
178 });
179 } finally {
180 if (prev === undefined) delete process.env.AUTOPILOT_DISABLED;
181 else process.env.AUTOPILOT_DISABLED = prev;
182 }
183 });
184
185 it("no-ops when ANTHROPIC_API_KEY is unset", async () => {
186 const prev = process.env.ANTHROPIC_API_KEY;
187 delete process.env.ANTHROPIC_API_KEY;
188 try {
189 const summary = await runCiHealerTick({
190 findCandidates: async () => {
191 throw new Error("should not be called");
192 },
193 });
194 expect(summary.considered).toBe(0);
195 expect(summary.healed).toBe(0);
196 } finally {
197 if (prev !== undefined) process.env.ANTHROPIC_API_KEY = prev;
198 }
199 });
200});
201
202// ---------------------------------------------------------------------------
203// DB-backed end-to-end. Builds a real failed run + jobs + repo + workflow
204// rows, then drives `healOneRun` with a fake Claude client.
205// ---------------------------------------------------------------------------
206
207function fakeClient(responseText: string) {
208 return {
209 messages: {
210 create: async () => ({
211 content: [{ type: "text" as const, text: responseText }],
212 }),
213 },
214 } as any;
215}
216
217interface Fixture {
218 repoId: string;
219 repoName: string;
220 ownerUsername: string;
221 workflowId: string;
222 runId: string;
223 baseSha: string;
224}
225
226async function seedFailedRun(label: string): Promise<Fixture> {
227 const username = `cihealer_${label}_${Date.now()}_${Math.random()
228 .toString(36)
229 .slice(2, 6)}`;
230 const [u] = await db
231 .insert(users)
232 .values({
233 username,
234 email: `${username}@example.com`,
235 passwordHash: "x",
236 })
237 .returning({ id: users.id });
238
239 const repoName = `subject_${label}_${Date.now()}`;
240 const [r] = await db
241 .insert(repositories)
242 .values({
243 ownerId: u.id,
244 name: repoName,
245 diskPath: `/tmp/${username}/${repoName}`,
246 defaultBranch: "main",
247 })
248 .returning({ id: repositories.id, name: repositories.name });
249
250 await initBareRepo(username, repoName);
251 const seeded = await createOrUpdateFileOnBranch({
252 owner: username,
253 name: repoName,
254 branch: "main",
255 filePath: "src/index.ts",
256 bytes: new TextEncoder().encode(
257 "export function broken(){ return undefined.length; }\n"
258 ),
259 message: "seed",
260 authorName: "Seeder",
261 authorEmail: "s@e.com",
262 });
263 if ("error" in seeded) throw new Error("seed failed: " + seeded.error);
264
265 const [w] = await db
266 .insert(workflows)
267 .values({
268 repositoryId: r.id,
269 name: "ci",
270 path: ".gluecron/workflows/ci.yml",
271 yaml: "name: ci\non: [push]\njobs:\n build:\n steps:\n - run: bun test",
272 parsed: JSON.stringify({
273 name: "ci",
274 on: ["push"],
275 jobs: { build: { steps: [{ run: "bun test" }] } },
276 }),
277 })
278 .returning({ id: workflows.id });
279
280 // Insert run as `failure` with createdAt safely in the past so the
281 // candidate finder considers it (HEAL_MIN_AGE_MS = 60s).
282 const oldCreatedAt = new Date(Date.now() - 5 * 60 * 1000);
283 const [run] = await db
284 .insert(workflowRuns)
285 .values({
286 workflowId: w.id,
287 repositoryId: r.id,
288 runNumber: 1,
289 event: "push",
290 ref: "refs/heads/main",
291 commitSha: seeded.commitSha,
292 status: "failure",
293 conclusion: "failure",
294 queuedAt: oldCreatedAt,
295 startedAt: oldCreatedAt,
296 finishedAt: oldCreatedAt,
297 createdAt: oldCreatedAt,
298 })
299 .returning({ id: workflowRuns.id });
300
301 await db.insert(workflowJobs).values({
302 runId: run.id,
303 name: "build",
304 jobOrder: 0,
305 runsOn: "default",
306 status: "failure",
307 conclusion: "failure",
308 exitCode: 1,
309 steps: "[]",
310 logs:
311 "==> bun test\nTypeError: Cannot read properties of undefined (reading 'length')\n at broken (src/index.ts:1:38)\n[exit 1 in 120ms]",
312 startedAt: oldCreatedAt,
313 finishedAt: oldCreatedAt,
314 });
315
316 return {
317 repoId: r.id,
318 repoName,
319 ownerUsername: username,
320 workflowId: w.id,
321 runId: run.id,
322 baseSha: seeded.commitSha,
323 };
324}
325
326describe.skipIf(!HAS_DB)("ai-ci-healer DB-backed E2E", () => {
327 it("opens a patch PR + writes an ai.ci.healed audit row on the happy path", async () => {
328 const fx = await seedFailedRun("happy");
329
330 const cannedClaude = JSON.stringify({
331 rootCause:
332 "broken() references undefined.length, which throws at runtime.",
333 fixable: true,
334 suggestedFixes: [
335 {
336 path: "src/index.ts",
337 description: "Return a numeric literal instead of dereferencing undefined.",
338 severity: "high",
339 },
340 ],
341 });
342
343 const cannedPatch = JSON.stringify({
344 explanation: "Replaced the bad expression with a safe literal.",
345 patches: [
346 {
347 path: "src/index.ts",
348 new_content: "export function broken(){ return 0; }\n",
349 },
350 ],
351 });
352
353 const client = fakeClient(cannedClaude);
354
355 // Use the real patch generator — feed it the same fake client for the
356 // patch call. The generator picks `client` from the opts we forward.
357 const { generatePatchForGateTestFinding } = await import(
358 "../lib/ai-patch-generator"
359 );
360 const out = await healOneRun(fx.runId, {
361 client,
362 // Wrap the real generator so we can swap in the second fake response
363 // for the patch step (it makes a fresh call). The healer hands the
364 // client through, so we wrap to replace the response between steps.
365 generatePatch: async (opts) => {
366 return generatePatchForGateTestFinding({
367 ...opts,
368 client: fakeClient(cannedPatch),
369 branchOverride: `ai-patch/ci-heal-${Date.now()}`,
370 });
371 },
372 });
373
374 expect(out.outcome).toBe("healed");
375 expect(typeof out.prNumber).toBe("number");
376 expect(out.branch).toContain("ai-patch/ci-heal-");
377
378 // PR row exists
379 const prs = await db
380 .select({ number: pullRequests.number, headBranch: pullRequests.headBranch })
381 .from(pullRequests)
382 .where(eq(pullRequests.repositoryId, fx.repoId));
383 expect(prs.length).toBe(1);
384 expect(prs[0].number).toBe(out.prNumber!);
385
386 // Audit marker present
387 const audits = await db
388 .select({ action: auditLog.action })
389 .from(auditLog)
390 .where(
391 and(
392 eq(auditLog.targetType, "workflow_run"),
393 eq(auditLog.targetId, fx.runId)
394 )
395 );
396 expect(audits.some((a) => a.action === "ai.ci.healed")).toBe(true);
397 }, 30_000);
398
399 it("writes ai.ci.gave_up + opens no PR when Claude says unfixable", async () => {
400 const fx = await seedFailedRun("unfix");
401
402 const cannedClaude = JSON.stringify({
403 rootCause: "The npm registry returned 503 mid-install.",
404 fixable: false,
405 suggestedFixes: [],
406 unfixableReason: "External registry outage — retry later.",
407 });
408
409 const client = fakeClient(cannedClaude);
410 const out = await healOneRun(fx.runId, { client });
411 expect(out.outcome).toBe("gave_up");
412
413 // No PR row
414 const prs = await db
415 .select({ number: pullRequests.number })
416 .from(pullRequests)
417 .where(eq(pullRequests.repositoryId, fx.repoId));
418 expect(prs.length).toBe(0);
419
420 // Audit marker present as gave_up
421 const audits = await db
422 .select({ action: auditLog.action })
423 .from(auditLog)
424 .where(
425 and(
426 eq(auditLog.targetType, "workflow_run"),
427 eq(auditLog.targetId, fx.runId)
428 )
429 );
430 expect(audits.some((a) => a.action === "ai.ci.gave_up")).toBe(true);
431 expect(audits.some((a) => a.action === "ai.ci.healed")).toBe(false);
432 }, 30_000);
433
434 it("skips runs that already have a marker (no double-processing)", async () => {
435 const fx = await seedFailedRun("dedupe");
436
437 // Pre-insert a marker
438 await db.insert(auditLog).values({
439 action: "ai.ci.healed",
440 targetType: "workflow_run",
441 targetId: fx.runId,
442 metadata: JSON.stringify({ pre: true }),
443 });
444
445 // Should refuse to act — Claude must NOT be called.
446 let claudeCalled = 0;
447 const client = {
448 messages: {
449 create: async () => {
450 claudeCalled += 1;
451 return { content: [{ type: "text" as const, text: "{}" }] };
452 },
453 },
454 } as any;
455
456 const out = await healOneRun(fx.runId, { client });
457 expect(out.outcome).toBe("skipped");
458 expect(claudeCalled).toBe(0);
459 }, 30_000);
460});
461
462// ---------------------------------------------------------------------------
463// analyzeFailedWorkflowRun direct-call sanity (the public surface the
464// task description explicitly calls out as the entry point).
465// ---------------------------------------------------------------------------
466
467describe.skipIf(!HAS_DB)("analyzeFailedWorkflowRun", () => {
468 it("returns parsed analysis with patchablePaths when Claude says fixable", async () => {
469 const fx = await seedFailedRun("analyze");
470 const canned = JSON.stringify({
471 rootCause: "Null deref in src/index.ts.",
472 fixable: true,
473 suggestedFixes: [
474 { path: "src/index.ts", description: "Add null guard.", severity: "high" },
475 ],
476 });
477 const analysis = await analyzeFailedWorkflowRun(fx.runId, {
478 client: fakeClient(canned),
479 });
480 expect(analysis).not.toBeNull();
481 expect(analysis!.rootCause).toContain("Null deref");
482 expect(analysis!.patchablePaths).toEqual(["src/index.ts"]);
483 expect(analysis!.suggestedFixes.length).toBe(1);
484 }, 30_000);
485
486 it("returns null when Claude says unfixable", async () => {
487 const fx = await seedFailedRun("analyze-unfix");
488 const canned = JSON.stringify({
489 rootCause: "Registry outage.",
490 fixable: false,
491 suggestedFixes: [],
492 });
493 const analysis = await analyzeFailedWorkflowRun(fx.runId, {
494 client: fakeClient(canned),
495 });
496 expect(analysis).toBeNull();
497 }, 30_000);
498
499 it("returns null for non-failure runs", async () => {
500 const fx = await seedFailedRun("non-failure");
501 // Flip the run back to success — analyzer should bail.
502 await db
503 .update(workflowRuns)
504 .set({ status: "success", conclusion: "success" })
505 .where(eq(workflowRuns.id, fx.runId));
506 const analysis = await analyzeFailedWorkflowRun(fx.runId, {
507 client: fakeClient('{"rootCause":"x","fixable":true,"suggestedFixes":[]}'),
508 });
509 expect(analysis).toBeNull();
510 }, 30_000);
511});
Addedsrc/lib/ai-ci-healer.ts+652−0View fileUnifiedSplit
@@ -0,0 +1,652 @@
1/**
2 * AI CI Healer — autonomous failure → root-cause → patch PR loop.
3 *
4 * When a `workflow_runs` row lands in status="failure", this module:
5 * 1. Pulls the run + all its `workflow_jobs` (especially the failed ones'
6 * `logs` column).
7 * 2. Asks Claude to identify the root cause + whether it's fixable from
8 * inside this repo.
9 * 3. If patchable, hands the suggested fixes off to the existing
10 * `generatePatchForGateTestFinding` (the finding-shape maps cleanly
11 * onto its `GateTestFinding[]` API — both surfaces want
12 * `{ path, description, severity }`).
13 * 4. Records an `ai.ci.healed` or `ai.ci.gave_up` audit row keyed on the
14 * run id so the autopilot poller doesn't re-process the same failure
15 * every 5 minutes.
16 *
17 * Degrades silently when ANTHROPIC_API_KEY is unset (autopilot also gates
18 * on the env var, but the lib double-checks so it's safe to call directly).
19 * Everything is wrapped in try/catch — `analyzeFailedWorkflowRun` returns
20 * `null` rather than throwing on any failure.
21 *
22 * Marker convention:
23 * - Successful patch open → audit action `ai.ci.healed`, targetId = runId.
24 * - Claude says unfixable → audit action `ai.ci.gave_up`, targetId = runId.
25 * The autopilot poller skips any run that already has either marker, so we
26 * never retry forever.
27 */
28
29import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
30import type Anthropic from "@anthropic-ai/sdk";
31import { db } from "../db";
32import {
33 auditLog,
34 repositories,
35 workflowJobs,
36 workflowRuns,
37 workflows,
38} from "../db/schema";
39import {
40 MODEL_SONNET,
41 extractText,
42 getAnthropic,
43 isAiAvailable,
44 parseJsonResponse,
45} from "./ai-client";
46import { audit } from "./notify";
47import {
48 generatePatchForGateTestFinding,
49 type GateTestFinding,
50} from "./ai-patch-generator";
51
52// ---------------------------------------------------------------------------
53// Tunables
54// ---------------------------------------------------------------------------
55
56/** Cap per-job log slice we ship to Claude — protect the prompt budget. */
57const LOG_SNIPPET_BYTES = 8 * 1024;
58
59/** Hard cap on runs processed per autopilot tick — runaway protection. */
60const HEAL_CAP_PER_TICK = 5;
61
62/** Only consider runs that finished at least this long ago. Gives the
63 * workflow runner time to flush logs + final job rows before we sample. */
64const HEAL_MIN_AGE_MS = 60 * 1_000;
65
66/** Don't bother healing ancient failures — older than this and we assume
67 * the human has already triaged or the SHA has been rewritten. */
68const HEAL_MAX_AGE_MS = 24 * 60 * 60 * 1_000;
69
70// ---------------------------------------------------------------------------
71// Public surface
72// ---------------------------------------------------------------------------
73
74export interface SuggestedFix {
75 /** Relative path inside the repo that the AI thinks needs touching. */
76 path: string;
77 /** What's wrong / what the fix should look like (one-liner). */
78 description: string;
79 /** Severity inherited from the AI's confidence call. */
80 severity?: "low" | "medium" | "high" | "critical";
81}
82
83export interface CiHealAnalysis {
84 /** Plain-English root cause, 1-3 sentences. */
85 rootCause: string;
86 /** Concrete fix proposals (may be empty if Claude can't pinpoint files). */
87 suggestedFixes: SuggestedFix[];
88 /** Convenience: the paths from `suggestedFixes` deduped. */
89 patchablePaths: string[];
90}
91
92export interface CiHealerTickSummary {
93 considered: number;
94 healed: number;
95 gaveUp: number;
96 skipped: number;
97}
98
99/**
100 * Claude's response shape. Kept loose so we tolerate minor schema drift.
101 */
102interface ClaudeCiResponse {
103 rootCause?: string;
104 fixable?: boolean;
105 suggestedFixes?: Array<{
106 path?: string;
107 description?: string;
108 severity?: string;
109 }>;
110 /** Optional: Claude's reason it gave up. We just persist it for ops. */
111 unfixableReason?: string;
112}
113
114// ---------------------------------------------------------------------------
115// Helpers
116// ---------------------------------------------------------------------------
117
118function truncate(s: string | null | undefined, max: number): string {
119 if (!s) return "";
120 if (s.length <= max) return s;
121 return s.slice(0, max) + "\n…(truncated)";
122}
123
124function normaliseSeverity(s: unknown): SuggestedFix["severity"] | undefined {
125 if (typeof s !== "string") return undefined;
126 const v = s.toLowerCase();
127 if (v === "low" || v === "medium" || v === "high" || v === "critical") {
128 return v;
129 }
130 return undefined;
131}
132
133/**
134 * Map our AI-derived `SuggestedFix` set onto the
135 * `GateTestFinding[]` shape that `generatePatchForGateTestFinding`
136 * already accepts. The patch generator only needs `{path, description,
137 * severity}` — exactly what we already have. Each fix becomes one
138 * candidate finding; the generator stops at the first one that produces a
139 * non-empty patch set so a bad suggestion doesn't drown a good one.
140 */
141export function fixesToFindings(
142 fixes: SuggestedFix[],
143 runShortId: string
144): GateTestFinding[] {
145 return fixes
146 .filter((f) => f.path && f.path.trim().length > 0)
147 .map((f, i) => ({
148 id: `ci-heal-${runShortId}-${i}`,
149 ruleId: "ci-failure",
150 path: f.path.trim(),
151 severity: f.severity || "high",
152 title: "CI failure auto-heal",
153 description: f.description || "CI failure",
154 }));
155}
156
157/**
158 * Build the Claude prompt. Pure function so tests can pin the shape.
159 */
160export function buildCiHealPrompt(args: {
161 repoFullName: string;
162 commitSha: string;
163 workflowYaml: string;
164 failedJobs: Array<{ name: string; conclusion: string | null; logs: string }>;
165}): string {
166 const jobsBlock = args.failedJobs
167 .map(
168 (j) =>
169 `### Job: ${j.name} (${j.conclusion || "failure"})\n\`\`\`\n${truncate(
170 j.logs,
171 LOG_SNIPPET_BYTES
172 )}\n\`\`\``
173 )
174 .join("\n\n");
175 return [
176 "You are GlueCron's CI healer. A workflow run just failed. Decide whether the failure is a code bug fixable in THIS repository, and if so propose concrete file edits.",
177 "",
178 `**Repository:** ${args.repoFullName}`,
179 `**Commit:** ${args.commitSha.slice(0, 12)}`,
180 "",
181 "## Workflow YAML",
182 "```yaml",
183 truncate(args.workflowYaml, 4_000),
184 "```",
185 "",
186 "## Failed job logs",
187 jobsBlock || "(no failed-job logs available)",
188 "",
189 "Respond ONLY with JSON of this exact shape:",
190 "{",
191 ' "rootCause": "1-3 sentence diagnosis (what failed, why)",',
192 ' "fixable": true | false,',
193 ' "suggestedFixes": [',
194 ' { "path": "src/foo.ts", "description": "what to change", "severity": "high" }',
195 " ],",
196 ' "unfixableReason": "(only if fixable=false) why the human must intervene"',
197 "}",
198 "",
199 "Rules:",
200 "- `fixable` MUST be false if the failure is an env/infra/external-service problem (missing secret, registry down, network, GitHub Actions runner image, etc.) — anything you can't fix by editing files in this repo.",
201 "- `suggestedFixes` must be empty when fixable=false.",
202 "- Only suggest paths you can identify with high confidence from the logs or YAML. Do not guess at random files.",
203 "- Keep `description` short — the patch generator will be invoked with this finding to produce the actual diff.",
204 ].join("\n");
205}
206
207interface FailedJobRow {
208 name: string;
209 conclusion: string | null;
210 logs: string;
211}
212
213async function loadFailedJobs(runId: string): Promise<FailedJobRow[]> {
214 try {
215 const rows = await db
216 .select({
217 name: workflowJobs.name,
218 conclusion: workflowJobs.conclusion,
219 logs: workflowJobs.logs,
220 status: workflowJobs.status,
221 })
222 .from(workflowJobs)
223 .where(eq(workflowJobs.runId, runId));
224 return rows
225 .filter((r) => r.status === "failure" || r.conclusion === "failure")
226 .map((r) => ({
227 name: r.name,
228 conclusion: r.conclusion,
229 logs: r.logs || "",
230 }));
231 } catch (err) {
232 console.error("[ai-ci-healer] loadFailedJobs failed:", err);
233 return [];
234 }
235}
236
237/**
238 * Has this run already been processed (success or give-up)? We use the
239 * audit log as the marker store so we don't need a schema change.
240 */
241async function hasMarker(runId: string): Promise<boolean> {
242 try {
243 const [row] = await db
244 .select({ id: auditLog.id })
245 .from(auditLog)
246 .where(
247 and(
248 eq(auditLog.targetType, "workflow_run"),
249 eq(auditLog.targetId, runId),
250 sql`${auditLog.action} IN ('ai.ci.healed', 'ai.ci.gave_up')`
251 )
252 )
253 .limit(1);
254 return !!row;
255 } catch (err) {
256 console.warn("[ai-ci-healer] hasMarker query failed:", err);
257 // Fail closed — if we can't check, skip to avoid duplicate work.
258 return true;
259 }
260}
261
262// ---------------------------------------------------------------------------
263// analyzeFailedWorkflowRun — public entry point #1
264// ---------------------------------------------------------------------------
265
266export interface AnalyzeOptions {
267 /** Test-only Anthropic client injection. */
268 client?: Pick<Anthropic, "messages">;
269}
270
271/**
272 * Diagnose a failed run. Returns `null` when:
273 * - The run doesn't exist or isn't actually a failure.
274 * - ANTHROPIC_API_KEY is unset AND no client was injected.
275 * - Claude says the failure isn't fixable from this repo (env/infra).
276 * - Any DB / network step blows up (logged, swallowed).
277 */
278export async function analyzeFailedWorkflowRun(
279 runId: string,
280 opts: AnalyzeOptions = {}
281): Promise<CiHealAnalysis | null> {
282 // Resolve client lazily — tests inject, production reads env.
283 let client: Pick<Anthropic, "messages">;
284 if (opts.client) {
285 client = opts.client;
286 } else {
287 if (!isAiAvailable()) return null;
288 try {
289 client = getAnthropic();
290 } catch {
291 return null;
292 }
293 }
294
295 // Load run row.
296 let run: typeof workflowRuns.$inferSelect | null = null;
297 try {
298 const [row] = await db
299 .select()
300 .from(workflowRuns)
301 .where(eq(workflowRuns.id, runId))
302 .limit(1);
303 run = row || null;
304 } catch (err) {
305 console.error("[ai-ci-healer] loadRun failed:", err);
306 return null;
307 }
308 if (!run || run.status !== "failure") return null;
309
310 // Load workflow + repo (for YAML + naming context).
311 let workflowYaml = "";
312 let repoFullName = "unknown/unknown";
313 try {
314 const [w] = await db
315 .select({ yaml: workflows.yaml })
316 .from(workflows)
317 .where(eq(workflows.id, run.workflowId))
318 .limit(1);
319 if (w?.yaml) workflowYaml = w.yaml;
320 } catch (err) {
321 console.warn("[ai-ci-healer] load workflow failed:", err);
322 }
323 try {
324 const [r] = await db
325 .select({
326 name: repositories.name,
327 ownerId: repositories.ownerId,
328 })
329 .from(repositories)
330 .where(eq(repositories.id, run.repositoryId))
331 .limit(1);
332 if (r) {
333 repoFullName = `${r.ownerId.slice(0, 8)}/${r.name}`;
334 }
335 } catch (err) {
336 console.warn("[ai-ci-healer] load repo failed:", err);
337 }
338
339 const failedJobs = await loadFailedJobs(runId);
340
341 // Ask Claude.
342 let parsed: ClaudeCiResponse | null = null;
343 try {
344 const message = await client.messages.create({
345 model: MODEL_SONNET,
346 max_tokens: 2048,
347 messages: [
348 {
349 role: "user",
350 content: buildCiHealPrompt({
351 repoFullName,
352 commitSha: run.commitSha || "(unknown)",
353 workflowYaml,
354 failedJobs,
355 }),
356 },
357 ],
358 });
359 parsed = parseJsonResponse<ClaudeCiResponse>(extractText(message));
360 } catch (err) {
361 console.warn(
362 "[ai-ci-healer] Claude call failed:",
363 err instanceof Error ? err.message : err
364 );
365 return null;
366 }
367 if (!parsed) return null;
368
369 const rootCause =
370 typeof parsed.rootCause === "string" && parsed.rootCause.trim()
371 ? parsed.rootCause.trim()
372 : "(no root cause provided)";
373
374 // "Not fixable" branch — return null so the caller can mark `ai.ci.gave_up`.
375 if (parsed.fixable === false) {
376 return null;
377 }
378
379 const rawFixes = Array.isArray(parsed.suggestedFixes)
380 ? parsed.suggestedFixes
381 : [];
382 const suggestedFixes: SuggestedFix[] = rawFixes
383 .filter(
384 (f): f is { path: string; description?: string; severity?: string } =>
385 !!f && typeof f.path === "string" && f.path.trim().length > 0
386 )
387 .map((f) => ({
388 path: f.path.trim(),
389 description:
390 typeof f.description === "string" && f.description.trim()
391 ? f.description.trim()
392 : rootCause,
393 severity: normaliseSeverity(f.severity),
394 }));
395
396 // Claude said fixable but produced zero usable paths → treat as
397 // unfixable so we don't loop.
398 if (suggestedFixes.length === 0) return null;
399
400 const patchablePaths = Array.from(new Set(suggestedFixes.map((f) => f.path)));
401
402 return { rootCause, suggestedFixes, patchablePaths };
403}
404
405// ---------------------------------------------------------------------------
406// healOneRun — drives one run end-to-end. Public so callers (autopilot,
407// tests, manual retrigger) share the same pipeline.
408// ---------------------------------------------------------------------------
409
410export interface HealOneOptions extends AnalyzeOptions {
411 /** Test override — pin the patch generator output. */
412 generatePatch?: typeof generatePatchForGateTestFinding;
413}
414
415export interface HealOneResult {
416 outcome: "healed" | "gave_up" | "skipped";
417 prNumber?: number;
418 branch?: string;
419 reason?: string;
420}
421
422export async function healOneRun(
423 runId: string,
424 opts: HealOneOptions = {}
425): Promise<HealOneResult> {
426 if (process.env.AUTOPILOT_DISABLED === "1") {
427 return { outcome: "skipped", reason: "autopilot disabled" };
428 }
429 if (!opts.client && !isAiAvailable()) {
430 return { outcome: "skipped", reason: "ANTHROPIC_API_KEY missing" };
431 }
432
433 // Skip if already processed (audit marker present).
434 if (await hasMarker(runId)) {
435 return { outcome: "skipped", reason: "already processed" };
436 }
437
438 const analysis = await analyzeFailedWorkflowRun(runId, {
439 client: opts.client,
440 });
441
442 // Look up the run for audit metadata (repo + sha).
443 let repositoryId: string | null = null;
444 let commitSha: string | null = null;
445 try {
446 const [row] = await db
447 .select({
448 repositoryId: workflowRuns.repositoryId,
449 commitSha: workflowRuns.commitSha,
450 })
451 .from(workflowRuns)
452 .where(eq(workflowRuns.id, runId))
453 .limit(1);
454 if (row) {
455 repositoryId = row.repositoryId;
456 commitSha = row.commitSha;
457 }
458 } catch (err) {
459 console.warn("[ai-ci-healer] post-analyze run lookup failed:", err);
460 }
461
462 if (!analysis) {
463 await audit({
464 userId: null,
465 repositoryId,
466 action: "ai.ci.gave_up",
467 targetType: "workflow_run",
468 targetId: runId,
469 metadata: { commitSha, reason: "unfixable or analysis returned null" },
470 });
471 return { outcome: "gave_up", reason: "unfixable" };
472 }
473
474 if (!repositoryId || !commitSha) {
475 // No base sha → can't seed a patch branch. Mark gave_up so we don't loop.
476 await audit({
477 userId: null,
478 repositoryId,
479 action: "ai.ci.gave_up",
480 targetType: "workflow_run",
481 targetId: runId,
482 metadata: {
483 commitSha,
484 reason: "missing repositoryId or commitSha for patch branch",
485 },
486 });
487 return { outcome: "gave_up", reason: "missing base sha" };
488 }
489
490 const generator = opts.generatePatch ?? generatePatchForGateTestFinding;
491 const findings = fixesToFindings(analysis.suggestedFixes, runId.slice(0, 8));
492
493 const patch = await generator({
494 repositoryId,
495 baseSha: commitSha,
496 findings,
497 client: opts.client,
498 });
499
500 if (!patch) {
501 await audit({
502 userId: null,
503 repositoryId,
504 action: "ai.ci.gave_up",
505 targetType: "workflow_run",
506 targetId: runId,
507 metadata: {
508 commitSha,
509 reason: "patch generator returned null",
510 rootCause: analysis.rootCause,
511 patchablePaths: analysis.patchablePaths,
512 },
513 });
514 return { outcome: "gave_up", reason: "patch generator returned null" };
515 }
516
517 await audit({
518 userId: null,
519 repositoryId,
520 action: "ai.ci.healed",
521 targetType: "workflow_run",
522 targetId: runId,
523 metadata: {
524 commitSha,
525 rootCause: analysis.rootCause,
526 patchablePaths: analysis.patchablePaths,
527 prNumber: patch.prNumber,
528 branch: patch.branch,
529 },
530 });
531
532 return {
533 outcome: "healed",
534 prNumber: patch.prNumber,
535 branch: patch.branch,
536 };
537}
538
539// ---------------------------------------------------------------------------
540// runCiHealerTick — public autopilot entry point
541// ---------------------------------------------------------------------------
542
543export interface CiHealerTickDeps {
544 /** Inject a candidate finder for tests. */
545 findCandidates?: (limit: number) => Promise<{ id: string }[]>;
546 /** Inject the per-run handler for tests. */
547 healOne?: (runId: string) => Promise<HealOneResult>;
548 /** Inject the per-tick cap. */
549 cap?: number;
550}
551
552/**
553 * Find failed workflow runs that:
554 * - finished at least HEAL_MIN_AGE_MS ago (let the runner flush logs)
555 * - finished less than HEAL_MAX_AGE_MS ago (don't chase ancient failures)
556 * - do NOT yet have an ai.ci.healed / ai.ci.gave_up audit marker
557 *
558 * The `hasMarker` filter is applied per-row in `healOneRun` rather than
559 * the SQL select — keeps the query simple and the index on (status,
560 * createdAt) doing most of the work.
561 */
562async function defaultFindCandidates(
563 limit: number
564): Promise<{ id: string }[]> {
565 const now = Date.now();
566 const cutoffNew = new Date(now - HEAL_MIN_AGE_MS);
567 const cutoffOld = new Date(now - HEAL_MAX_AGE_MS);
568 try {
569 const rows = await db
570 .select({ id: workflowRuns.id })
571 .from(workflowRuns)
572 .where(
573 and(
574 eq(workflowRuns.status, "failure"),
575 lt(workflowRuns.createdAt, cutoffNew),
576 gte(workflowRuns.createdAt, cutoffOld)
577 )
578 )
579 .orderBy(desc(workflowRuns.createdAt))
580 .limit(limit);
581 return rows;
582 } catch (err) {
583 console.error("[ai-ci-healer] candidate query failed:", err);
584 return [];
585 }
586}
587
588/**
589 * One autopilot tick: scan recent failures, heal what we can. Returns a
590 * counts summary. Never throws.
591 *
592 * No-op when:
593 * - AUTOPILOT_DISABLED=1
594 * - ANTHROPIC_API_KEY is unset
595 */
596export async function runCiHealerTick(
597 deps: CiHealerTickDeps = {}
598): Promise<CiHealerTickSummary> {
599 if (process.env.AUTOPILOT_DISABLED === "1") {
600 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
601 }
602 if (!isAiAvailable()) {
603 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
604 }
605
606 const findCandidates = deps.findCandidates ?? defaultFindCandidates;
607 const healOne = deps.healOne ?? ((id: string) => healOneRun(id));
608 const cap = deps.cap ?? HEAL_CAP_PER_TICK;
609
610 let candidates: { id: string }[] = [];
611 try {
612 candidates = await findCandidates(cap);
613 } catch (err) {
614 console.error("[ai-ci-healer] findCandidates threw:", err);
615 return { considered: 0, healed: 0, gaveUp: 0, skipped: 0 };
616 }
617
618 let healed = 0;
619 let gaveUp = 0;
620 let skipped = 0;
621 for (const cand of candidates) {
622 try {
623 const res = await healOne(cand.id);
624 if (res.outcome === "healed") healed += 1;
625 else if (res.outcome === "gave_up") gaveUp += 1;
626 else skipped += 1;
627 } catch (err) {
628 skipped += 1;
629 console.error(
630 `[ai-ci-healer] per-run failure for run=${cand.id}:`,
631 err instanceof Error ? err.message : err
632 );
633 }
634 }
635
636 return { considered: candidates.length, healed, gaveUp, skipped };
637}
638
639// ---------------------------------------------------------------------------
640// Test-only re-exports
641// ---------------------------------------------------------------------------
642
643export const __test = {
644 loadFailedJobs,
645 hasMarker,
646 defaultFindCandidates,
647 normaliseSeverity,
648 truncate,
649 HEAL_MIN_AGE_MS,
650 HEAL_MAX_AGE_MS,
651 HEAL_CAP_PER_TICK,
652};
Modifiedsrc/lib/autopilot.ts+21−0View fileUnifiedSplit
@@ -55,6 +55,7 @@ import {
5555 type SyntheticCheckResult,
5656} from "./synthetic-monitor";
5757import { aiProactiveMonitorTick } from "./ai-proactive-monitor";
58import { runCiHealerTick } from "./ai-ci-healer";
5859
5960export interface AutopilotTaskResult {
6061 name: string;
@@ -262,6 +263,26 @@ export function defaultTasks(): AutopilotTask[] {
262263 }
263264 },
264265 },
266 {
267 // AI CI Healer — autonomous CI failure → root-cause → patch PR loop.
268 // Polls every tick (5 min) for failed workflow_runs that finished
269 // at least HEAL_MIN_AGE_MS ago and haven't been processed yet.
270 // Skips when ANTHROPIC_API_KEY is unset (handled inside the lib);
271 // AUTOPILOT_DISABLED=1 short-circuits the wrapping startAutopilot
272 // call already, but the lib double-checks for direct callers.
273 name: "ci-healer",
274 run: async () => {
275 if (!process.env.ANTHROPIC_API_KEY) return;
276 try {
277 const summary = await runCiHealerTick();
278 console.log(
279 `[autopilot] ci-healer: considered=${summary.considered} healed=${summary.healed} gaveUp=${summary.gaveUp} skipped=${summary.skipped}`
280 );
281 } catch (err) {
282 console.error("[autopilot] ci-healer: threw:", err);
283 }
284 },
285 },
265286 {
266287 // BLOCK S4 — Synthetic monitor.
267288 //
268289