Blame · Line-by-line history

`ai-ci-healer.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-ci-healer.test.tsBlame511 lines · 1 contributor

Normal view Raw

9336f45Claude	1	/**
	2	* Tests for src/lib/ai-ci-healer.ts.
	3	*
	4	* Two layers:
	5	* 1. Pure helpers — buildCiHealPrompt + fixesToFindings + small private
	6	* helpers exposed via __test. Always run.
	7	* 2. End-to-end with a fake Claude client + a real DB row + bare repo.
	8	* Gated on DATABASE_URL via the HAS_DB skipIf pattern used across
	9	* the suite.
	10	*
	11	* The Anthropic client is faked via the public `client` option so we never
	12	* touch the network or require ANTHROPIC_API_KEY. We also DI the
	13	* patch-generator into `healOneRun` so each test pins its own outcome.
	14	*/
	15
	16	import { describe, it, expect, beforeAll, afterAll } from "bun:test";
	17	import { join } from "path";
	18	import { mkdir, rm } from "fs/promises";
	19	import { eq, and } from "drizzle-orm";
	20	import {
	21	analyzeFailedWorkflowRun,
	22	buildCiHealPrompt,
	23	fixesToFindings,
	24	healOneRun,
	25	runCiHealerTick,
	26	__test,
	27	} from "../lib/ai-ci-healer";
	28	import { db } from "../db";
	29	import {
	30	auditLog,
	31	pullRequests,
	32	repositories,
	33	users,
	34	workflowJobs,
	35	workflowRuns,
	36	workflows,
	37	} from "../db/schema";
	38	import {
	39	createOrUpdateFileOnBranch,
	40	initBareRepo,
	41	} from "../git/repository";
	42
	43	const HAS_DB = Boolean(process.env.DATABASE_URL);
	44
	45	const TEST_REPOS = join(
	46	import.meta.dir,
	47	"../../.test-repos-ai-ci-healer-" + Date.now()
	48	);
	49
	50	beforeAll(async () => {
	51	process.env.GIT_REPOS_PATH = TEST_REPOS;
	52	process.env.DATABASE_URL = process.env.DATABASE_URL \|\| "";
	53	// Make sure autopilot doesn't short-circuit our tests via env.
	54	delete process.env.AUTOPILOT_DISABLED;
	55	await rm(TEST_REPOS, { recursive: true, force: true });
	56	await mkdir(TEST_REPOS, { recursive: true });
	57	});
	58
	59	afterAll(async () => {
	60	await rm(TEST_REPOS, { recursive: true, force: true });
	61	});
	62
	63	// ---------------------------------------------------------------------------
	64	// Pure helpers
	65	// ---------------------------------------------------------------------------
	66
	67	describe("buildCiHealPrompt", () => {
	68	it("embeds repo, sha, yaml, and failed job logs", () => {
	69	const prompt = buildCiHealPrompt({
	70	repoFullName: "alice/web",
	71	commitSha: "abcdef1234567890",
	72	workflowYaml: "name: ci\non: [push]\njobs:\n build:\n steps: []",
	73	failedJobs: [
	74	{ name: "build", conclusion: "failure", logs: "TypeError: x is not a function" },
	75	],
	76	});
	77	expect(prompt).toContain("alice/web");
	78	expect(prompt).toContain("abcdef123456");
	79	expect(prompt).toContain("name: ci");
	80	expect(prompt).toContain("TypeError: x is not a function");
	81	// Strict JSON schema cue
	82	expect(prompt).toContain('"rootCause"');
	83	expect(prompt).toContain('"suggestedFixes"');
	84	});
	85
	86	it("handles the empty-jobs case gracefully", () => {
	87	const prompt = buildCiHealPrompt({
	88	repoFullName: "x/y",
	89	commitSha: "1234567",
	90	workflowYaml: "name: ci",
	91	failedJobs: [],
	92	});
	93	expect(prompt).toContain("(no failed-job logs available)");
	94	});
	95	});
	96
	97	describe("fixesToFindings", () => {
	98	it("maps SuggestedFix[] onto GateTestFinding[] preserving path + severity", () => {
	99	const findings = fixesToFindings(
	100	[
	101	{ path: "src/a.ts", description: "fix imports", severity: "high" },
	102	{ path: "src/b.ts", description: "guard null", severity: "medium" },
	103	],
	104	"deadbeef"
	105	);
	106	expect(findings.length).toBe(2);
	107	expect(findings[0].path).toBe("src/a.ts");
	108	expect(findings[0].severity).toBe("high");
	109	expect(findings[0].id).toContain("ci-heal-deadbeef");
	110	expect(findings[1].path).toBe("src/b.ts");
	111	expect(findings[1].severity).toBe("medium");
	112	});
	113
	114	it("drops fixes with no path", () => {
	115	const findings = fixesToFindings(
	116	[
	117	{ path: "", description: "noop" },
	118	{ path: "src/c.ts", description: "ok" },
	119	],
	120	"x"
	121	);
	122	expect(findings.length).toBe(1);
	123	expect(findings[0].path).toBe("src/c.ts");
	124	});
	125
	126	it("defaults severity to high when omitted", () => {
	127	const findings = fixesToFindings(
	128	[{ path: "src/d.ts", description: "no sev" }],
	129	"x"
	130	);
	131	expect(findings[0].severity).toBe("high");
	132	});
	133	});
	134
	135	describe("__test internals", () => {
	136	it("normaliseSeverity accepts the canonical levels case-insensitively", () => {
	137	expect(__test.normaliseSeverity("HIGH")).toBe("high");
	138	expect(__test.normaliseSeverity("Medium")).toBe("medium");
	139	expect(__test.normaliseSeverity("critical")).toBe("critical");
	140	expect(__test.normaliseSeverity("low")).toBe("low");
	141	});
	142
	143	it("normaliseSeverity rejects garbage", () => {
	144	expect(__test.normaliseSeverity("bananas")).toBeUndefined();
	145	expect(__test.normaliseSeverity(undefined)).toBeUndefined();
	146	expect(__test.normaliseSeverity(null)).toBeUndefined();
	147	expect(__test.normaliseSeverity(42)).toBeUndefined();
	148	});
	149
	150	it("truncate caps long strings + appends marker", () => {
	151	expect(__test.truncate("abc", 10)).toBe("abc");
	152	const long = "x".repeat(20);
	153	const t = __test.truncate(long, 5);
	154	expect(t.startsWith("xxxxx")).toBe(true);
	155	expect(t).toContain("(truncated)");
	156	});
	157	});
	158
	159	// ---------------------------------------------------------------------------
	160	// Skip when AUTOPILOT_DISABLED: tick must no-op cleanly
	161	// ---------------------------------------------------------------------------
	162
	163	describe("runCiHealerTick — env gates", () => {
	164	it("no-ops when AUTOPILOT_DISABLED=1", async () => {
	165	const prev = process.env.AUTOPILOT_DISABLED;
	166	process.env.AUTOPILOT_DISABLED = "1";
	167	try {
	168	const summary = await runCiHealerTick({
	169	findCandidates: async () => {
	170	throw new Error("should not be called");
	171	},
	172	});
	173	expect(summary).toEqual({
	174	considered: 0,
	175	healed: 0,
	176	gaveUp: 0,
	177	skipped: 0,
	178	});
	179	} finally {
	180	if (prev === undefined) delete process.env.AUTOPILOT_DISABLED;
	181	else process.env.AUTOPILOT_DISABLED = prev;
	182	}
	183	});
	184
	185	it("no-ops when ANTHROPIC_API_KEY is unset", async () => {
	186	const prev = process.env.ANTHROPIC_API_KEY;
	187	delete process.env.ANTHROPIC_API_KEY;
	188	try {
	189	const summary = await runCiHealerTick({
	190	findCandidates: async () => {
	191	throw new Error("should not be called");
	192	},
	193	});
	194	expect(summary.considered).toBe(0);
	195	expect(summary.healed).toBe(0);
	196	} finally {
	197	if (prev !== undefined) process.env.ANTHROPIC_API_KEY = prev;
	198	}
	199	});
	200	});
	201
	202	// ---------------------------------------------------------------------------
	203	// DB-backed end-to-end. Builds a real failed run + jobs + repo + workflow
	204	// rows, then drives `healOneRun` with a fake Claude client.
	205	// ---------------------------------------------------------------------------
	206
	207	function fakeClient(responseText: string) {
	208	return {
	209	messages: {
	210	create: async () => ({
	211	content: [{ type: "text" as const, text: responseText }],
	212	}),
	213	},
	214	} as any;
	215	}
	216
	217	interface Fixture {
	218	repoId: string;
	219	repoName: string;
	220	ownerUsername: string;
	221	workflowId: string;
	222	runId: string;
	223	baseSha: string;
	224	}
	225
	226	async function seedFailedRun(label: string): Promise<Fixture> {
	227	const username = `cihealer_${label}_${Date.now()}_${Math.random()
	228	.toString(36)
	229	.slice(2, 6)}`;
	230	const [u] = await db
	231	.insert(users)
	232	.values({
	233	username,
	234	email: `${username}@example.com`,
	235	passwordHash: "x",
	236	})
	237	.returning({ id: users.id });
	238
	239	const repoName = `subject_${label}_${Date.now()}`;
	240	const [r] = await db
	241	.insert(repositories)
	242	.values({
	243	ownerId: u.id,
	244	name: repoName,
	245	diskPath: `/tmp/${username}/${repoName}`,
	246	defaultBranch: "main",
	247	})
	248	.returning({ id: repositories.id, name: repositories.name });
	249
	250	await initBareRepo(username, repoName);
	251	const seeded = await createOrUpdateFileOnBranch({
	252	owner: username,
	253	name: repoName,
	254	branch: "main",
	255	filePath: "src/index.ts",
	256	bytes: new TextEncoder().encode(
	257	"export function broken(){ return undefined.length; }\n"
	258	),
	259	message: "seed",
	260	authorName: "Seeder",
	261	authorEmail: "s@e.com",
	262	});
	263	if ("error" in seeded) throw new Error("seed failed: " + seeded.error);
	264
	265	const [w] = await db
	266	.insert(workflows)
	267	.values({
	268	repositoryId: r.id,
	269	name: "ci",
	270	path: ".gluecron/workflows/ci.yml",
	271	yaml: "name: ci\non: [push]\njobs:\n build:\n steps:\n - run: bun test",
	272	parsed: JSON.stringify({
	273	name: "ci",
	274	on: ["push"],
	275	jobs: { build: { steps: [{ run: "bun test" }] } },
	276	}),
	277	})
	278	.returning({ id: workflows.id });
	279
	280	// Insert run as `failure` with createdAt safely in the past so the
	281	// candidate finder considers it (HEAL_MIN_AGE_MS = 60s).
	282	const oldCreatedAt = new Date(Date.now() - 5 * 60 * 1000);
	283	const [run] = await db
	284	.insert(workflowRuns)
	285	.values({
	286	workflowId: w.id,
	287	repositoryId: r.id,
	288	runNumber: 1,
	289	event: "push",
	290	ref: "refs/heads/main",
	291	commitSha: seeded.commitSha,
	292	status: "failure",
	293	conclusion: "failure",
	294	queuedAt: oldCreatedAt,
	295	startedAt: oldCreatedAt,
	296	finishedAt: oldCreatedAt,
	297	createdAt: oldCreatedAt,
	298	})
	299	.returning({ id: workflowRuns.id });
	300
	301	await db.insert(workflowJobs).values({
	302	runId: run.id,
	303	name: "build",
	304	jobOrder: 0,
	305	runsOn: "default",
	306	status: "failure",
	307	conclusion: "failure",
	308	exitCode: 1,
	309	steps: "[]",
	310	logs:
	311	"==> bun test\nTypeError: Cannot read properties of undefined (reading 'length')\n at broken (src/index.ts:1:38)\n[exit 1 in 120ms]",
	312	startedAt: oldCreatedAt,
	313	finishedAt: oldCreatedAt,
	314	});
	315
	316	return {
	317	repoId: r.id,
	318	repoName,
	319	ownerUsername: username,
	320	workflowId: w.id,
	321	runId: run.id,
	322	baseSha: seeded.commitSha,
	323	};
	324	}
	325
	326	describe.skipIf(!HAS_DB)("ai-ci-healer DB-backed E2E", () => {
	327	it("opens a patch PR + writes an ai.ci.healed audit row on the happy path", async () => {
	328	const fx = await seedFailedRun("happy");
	329
	330	const cannedClaude = JSON.stringify({
	331	rootCause:
	332	"broken() references undefined.length, which throws at runtime.",
	333	fixable: true,
	334	suggestedFixes: [
	335	{
	336	path: "src/index.ts",
	337	description: "Return a numeric literal instead of dereferencing undefined.",
	338	severity: "high",
	339	},
	340	],
	341	});
	342
	343	const cannedPatch = JSON.stringify({
	344	explanation: "Replaced the bad expression with a safe literal.",
	345	patches: [
	346	{
	347	path: "src/index.ts",
	348	new_content: "export function broken(){ return 0; }\n",
	349	},
	350	],
	351	});
	352
	353	const client = fakeClient(cannedClaude);
	354
	355	// Use the real patch generator — feed it the same fake client for the
	356	// patch call. The generator picks `client` from the opts we forward.
	357	const { generatePatchForGateTestFinding } = await import(
	358	"../lib/ai-patch-generator"
	359	);
	360	const out = await healOneRun(fx.runId, {
	361	client,
	362	// Wrap the real generator so we can swap in the second fake response
	363	// for the patch step (it makes a fresh call). The healer hands the
	364	// client through, so we wrap to replace the response between steps.
	365	generatePatch: async (opts) => {
	366	return generatePatchForGateTestFinding({
	367	...opts,
	368	client: fakeClient(cannedPatch),
	369	branchOverride: `ai-patch/ci-heal-${Date.now()}`,
	370	});
	371	},
	372	});
	373
	374	expect(out.outcome).toBe("healed");
	375	expect(typeof out.prNumber).toBe("number");
	376	expect(out.branch).toContain("ai-patch/ci-heal-");
	377
	378	// PR row exists
	379	const prs = await db
	380	.select({ number: pullRequests.number, headBranch: pullRequests.headBranch })
	381	.from(pullRequests)
	382	.where(eq(pullRequests.repositoryId, fx.repoId));
	383	expect(prs.length).toBe(1);
	384	expect(prs[0].number).toBe(out.prNumber!);
	385
	386	// Audit marker present
	387	const audits = await db
	388	.select({ action: auditLog.action })
	389	.from(auditLog)
	390	.where(
	391	and(
	392	eq(auditLog.targetType, "workflow_run"),
	393	eq(auditLog.targetId, fx.runId)
	394	)
	395	);
	396	expect(audits.some((a) => a.action === "ai.ci.healed")).toBe(true);
	397	}, 30_000);
	398
	399	it("writes ai.ci.gave_up + opens no PR when Claude says unfixable", async () => {
	400	const fx = await seedFailedRun("unfix");
	401
	402	const cannedClaude = JSON.stringify({
	403	rootCause: "The npm registry returned 503 mid-install.",
	404	fixable: false,
	405	suggestedFixes: [],
	406	unfixableReason: "External registry outage — retry later.",
	407	});
	408
	409	const client = fakeClient(cannedClaude);
	410	const out = await healOneRun(fx.runId, { client });
	411	expect(out.outcome).toBe("gave_up");
	412
	413	// No PR row
	414	const prs = await db
	415	.select({ number: pullRequests.number })
	416	.from(pullRequests)
	417	.where(eq(pullRequests.repositoryId, fx.repoId));
	418	expect(prs.length).toBe(0);
	419
	420	// Audit marker present as gave_up
	421	const audits = await db
	422	.select({ action: auditLog.action })
	423	.from(auditLog)
	424	.where(
	425	and(
	426	eq(auditLog.targetType, "workflow_run"),
	427	eq(auditLog.targetId, fx.runId)
	428	)
	429	);
	430	expect(audits.some((a) => a.action === "ai.ci.gave_up")).toBe(true);
	431	expect(audits.some((a) => a.action === "ai.ci.healed")).toBe(false);
	432	}, 30_000);
	433
	434	it("skips runs that already have a marker (no double-processing)", async () => {
	435	const fx = await seedFailedRun("dedupe");
	436
	437	// Pre-insert a marker
	438	await db.insert(auditLog).values({
	439	action: "ai.ci.healed",
	440	targetType: "workflow_run",
	441	targetId: fx.runId,
	442	metadata: JSON.stringify({ pre: true }),
	443	});
	444
	445	// Should refuse to act — Claude must NOT be called.
	446	let claudeCalled = 0;
	447	const client = {
	448	messages: {
	449	create: async () => {
	450	claudeCalled += 1;
	451	return { content: [{ type: "text" as const, text: "{}" }] };
	452	},
	453	},
	454	} as any;
	455
	456	const out = await healOneRun(fx.runId, { client });
	457	expect(out.outcome).toBe("skipped");
	458	expect(claudeCalled).toBe(0);
	459	}, 30_000);
	460	});
	461
	462	// ---------------------------------------------------------------------------
	463	// analyzeFailedWorkflowRun direct-call sanity (the public surface the
	464	// task description explicitly calls out as the entry point).
	465	// ---------------------------------------------------------------------------
	466
	467	describe.skipIf(!HAS_DB)("analyzeFailedWorkflowRun", () => {
	468	it("returns parsed analysis with patchablePaths when Claude says fixable", async () => {
	469	const fx = await seedFailedRun("analyze");
	470	const canned = JSON.stringify({
	471	rootCause: "Null deref in src/index.ts.",
	472	fixable: true,
	473	suggestedFixes: [
	474	{ path: "src/index.ts", description: "Add null guard.", severity: "high" },
	475	],
	476	});
	477	const analysis = await analyzeFailedWorkflowRun(fx.runId, {
	478	client: fakeClient(canned),
	479	});
	480	expect(analysis).not.toBeNull();
	481	expect(analysis!.rootCause).toContain("Null deref");
	482	expect(analysis!.patchablePaths).toEqual(["src/index.ts"]);
	483	expect(analysis!.suggestedFixes.length).toBe(1);
	484	}, 30_000);
	485
	486	it("returns null when Claude says unfixable", async () => {
	487	const fx = await seedFailedRun("analyze-unfix");
	488	const canned = JSON.stringify({
	489	rootCause: "Registry outage.",
	490	fixable: false,
	491	suggestedFixes: [],
	492	});
	493	const analysis = await analyzeFailedWorkflowRun(fx.runId, {
	494	client: fakeClient(canned),
	495	});
	496	expect(analysis).toBeNull();
	497	}, 30_000);
	498
	499	it("returns null for non-failure runs", async () => {
	500	const fx = await seedFailedRun("non-failure");
	501	// Flip the run back to success — analyzer should bail.
	502	await db
	503	.update(workflowRuns)
	504	.set({ status: "success", conclusion: "success" })
	505	.where(eq(workflowRuns.id, fx.runId));
	506	const analysis = await analyzeFailedWorkflowRun(fx.runId, {
	507	client: fakeClient('{"rootCause":"x","fixable":true,"suggestedFixes":[]}'),
	508	});
	509	expect(analysis).toBeNull();
	510	}, 30_000);
	511	});