Blame · Line-by-line history

`ai-test-generator.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-test-generator.test.tsBlame611 lines · 1 contributor

Normal view Raw

232d974Claude	1	/**
	2	* Tests for src/lib/ai-test-generator.ts.
	3	*
	4	* Two layers:
	5	*
	6	* 1. Pure helpers — no DB, no Claude. Always run.
	7	* - isCandidateSourceFile filters test files / docs / configs.
	8	* - buildTestsForPrPrompt embeds the source file + framework hint.
	9	* - testsBranchName respects an override and otherwise embeds the
	10	* PR number + timestamp.
	11	* - looksLikeSpecPr detects spec-implementation PR bodies.
	12	* - renderFollowUpPrBody includes the marker, label, and file list.
	13	* - autopilot dispatch counts on injected stubs.
	14	*
	15	* 2. End-to-end generateTestsForPr — injected fake Claude client + a
	16	* real bare repo on disk. DB-backed steps (PR insert, label upsert,
	17	* audit) are gated on HAS_DB so the DB-less sandbox still exercises
	18	* the git-side write path.
	19	*/
	20
	21	import { describe, it, expect, beforeAll, afterAll } from "bun:test";
	22	import { join } from "path";
	23	import { rm, mkdir } from "fs/promises";
	24	import { eq } from "drizzle-orm";
	25	import {
	26	AI_TESTS_LABEL,
	27	AI_TESTS_MARKER,
	28	buildTestsForPrPrompt,
	29	generateTestsForPr,
	30	isCandidateSourceFile,
	31	looksLikeSpecPr,
	32	renderFollowUpPrBody,
	33	testsBranchName,
	34	} from "../lib/ai-test-generator";
	35	import {
	36	runPrTestGeneratorTaskOnce,
	37	type PrTestGenCandidate,
	38	} from "../lib/autopilot-pr-test-generator";
	39	import {
	40	initBareRepo,
	41	createOrUpdateFileOnBranch,
	42	getBlob,
	43	refExists,
	44	resolveRef,
	45	} from "../git/repository";
	46	import { db } from "../db";
	47	import {
	48	prComments,
	49	pullRequests,
	50	repositories,
	51	users,
	52	} from "../db/schema";
	53
	54	const HAS_DB = Boolean(process.env.DATABASE_URL);
	55
	56	const TEST_REPOS = join(
	57	import.meta.dir,
	58	"../../.test-repos-ai-test-gen-" + Date.now()
	59	);
	60
	61	beforeAll(async () => {
	62	process.env.GIT_REPOS_PATH = TEST_REPOS;
	63	await rm(TEST_REPOS, { recursive: true, force: true });
	64	await mkdir(TEST_REPOS, { recursive: true });
	65	});
	66
	67	afterAll(async () => {
	68	await rm(TEST_REPOS, { recursive: true, force: true });
	69	});
	70
	71	// ---------------------------------------------------------------------------
	72	// Pure helpers
	73	// ---------------------------------------------------------------------------
	74
	75	describe("isCandidateSourceFile", () => {
	76	it("accepts ordinary TypeScript / JS / Python / Go sources", () => {
	77	expect(isCandidateSourceFile("src/lib/math.ts")).toBe(true);
	78	expect(isCandidateSourceFile("pkg/widget.py")).toBe(true);
	79	expect(isCandidateSourceFile("cmd/main.go")).toBe(true);
	80	expect(isCandidateSourceFile("lib/util.js")).toBe(true);
	81	});
	82
	83	it("rejects test files", () => {
	84	expect(isCandidateSourceFile("src/__tests__/math.test.ts")).toBe(false);
	85	expect(isCandidateSourceFile("tests/test_widget.py")).toBe(false);
	86	expect(isCandidateSourceFile("foo_test.go")).toBe(false);
	87	expect(isCandidateSourceFile("src/lib/math.spec.ts")).toBe(false);
	88	});
	89
	90	it("rejects docs and configs", () => {
	91	expect(isCandidateSourceFile("README.md")).toBe(false);
	92	expect(isCandidateSourceFile("package.json")).toBe(false);
	93	expect(isCandidateSourceFile("vitest.config.ts")).toBe(true); // .ts is code-shaped
	94	expect(isCandidateSourceFile("public/logo.png")).toBe(false);
	95	expect(isCandidateSourceFile("docs/intro.md")).toBe(false);
	96	});
	97
	98	it("rejects build / dependency directories", () => {
	99	expect(isCandidateSourceFile("node_modules/foo/index.js")).toBe(false);
	100	expect(isCandidateSourceFile("dist/bundle.js")).toBe(false);
	101	expect(isCandidateSourceFile(".github/workflows/ci.yml")).toBe(false);
	102	});
	103
	104	it("rejects path-traversal attempts", () => {
	105	expect(isCandidateSourceFile("../etc/passwd")).toBe(false);
	106	expect(isCandidateSourceFile("src/../../oops.ts")).toBe(false);
	107	});
	108	});
	109
	110	describe("buildTestsForPrPrompt", () => {
	111	it("embeds path, framework, language, and source code", () => {
	112	const out = buildTestsForPrPrompt({
	113	filePath: "src/lib/math.ts",
	114	language: "typescript",
	115	framework: "bun:test",
	116	sourceCode: "export function add(a: number, b: number) { return a + b; }",
	117	prTitle: "Add math helpers",
	118	});
	119	expect(out).toContain("src/lib/math.ts");
	120	expect(out).toContain("bun:test");
	121	expect(out).toContain("typescript");
	122	expect(out).toContain("export function add");
	123	expect(out).toContain("Add math helpers");
	124	// JSON schema cue MUST be present so the model returns parseable output.
	125	expect(out).toContain('"patches"');
	126	expect(out).toContain('"new_content"');
	127	});
	128
	129	it("truncates very large source files", () => {
	130	const huge = "a".repeat(50_000);
	131	const out = buildTestsForPrPrompt({
	132	filePath: "big.ts",
	133	language: "typescript",
	134	framework: "bun:test",
	135	sourceCode: huge,
	136	prTitle: "huge file",
	137	});
	138	expect(out.length).toBeLessThan(huge.length + 4_000);
	139	expect(out).toContain("truncated");
	140	});
	141	});
	142
	143	describe("testsBranchName", () => {
	144	it("honours an override", () => {
	145	expect(testsBranchName(42, "custom/branch")).toBe("custom/branch");
	146	});
	147
	148	it("uses ai-tests/pr-<n>-<timestamp> by default", () => {
	149	const name = testsBranchName(42);
	150	expect(name.startsWith("ai-tests/pr-42-")).toBe(true);
	151	const ts = name.slice("ai-tests/pr-42-".length);
	152	expect(/^\d+$/.test(ts)).toBe(true);
	153	});
	154	});
	155
	156	describe("looksLikeSpecPr", () => {
	157	it("detects the spec-implementation marker", () => {
	158	expect(
	159	looksLikeSpecPr("<!-- gluecron:ai-spec-implementation:v1 -->\nblah")
	160	).toBe(true);
	161	});
	162
	163	it("detects the spec label citation", () => {
	164	expect(looksLikeSpecPr("Label: `ai:spec-implementation`")).toBe(true);
	165	});
	166
	167	it("returns false for empty / null / unrelated bodies", () => {
	168	expect(looksLikeSpecPr(null)).toBe(false);
	169	expect(looksLikeSpecPr("")).toBe(false);
	170	expect(looksLikeSpecPr("Normal PR body")).toBe(false);
	171	});
	172	});
	173
	174	describe("renderFollowUpPrBody", () => {
	175	it("includes the marker, label, branch, and file list", () => {
	176	const body = renderFollowUpPrBody({
	177	originalPrNumber: 7,
	178	branch: "ai-tests/pr-7-1234",
	179	written: ["src/__tests__/math.test.ts"],
	180	});
	181	expect(body).toContain(AI_TESTS_MARKER);
	182	expect(body).toContain("for PR #7");
	183	expect(body).toContain(AI_TESTS_LABEL);
	184	expect(body).toContain("ai-tests/pr-7-1234");
	185	expect(body).toContain("src/__tests__/math.test.ts");
	186	});
	187
	188	it("renders cleanly with no files", () => {
	189	const body = renderFollowUpPrBody({
	190	originalPrNumber: 1,
	191	branch: "ai-tests/pr-1-0",
	192	written: [],
	193	});
	194	expect(body).toContain(AI_TESTS_MARKER);
	195	expect(body).toContain("_(none)_");
	196	});
	197	});
	198
	199	// ---------------------------------------------------------------------------
	200	// Autopilot task — DI seam, no DB / no Claude
	201	// ---------------------------------------------------------------------------
	202
	203	describe("runPrTestGeneratorTaskOnce", () => {
	204	const originalKey = process.env.ANTHROPIC_API_KEY;
	205	const originalDisabled = process.env.AUTOPILOT_DISABLED;
	206
	207	function restoreEnv() {
	208	if (originalKey === undefined) delete process.env.ANTHROPIC_API_KEY;
	209	else process.env.ANTHROPIC_API_KEY = originalKey;
	210	if (originalDisabled === undefined) delete process.env.AUTOPILOT_DISABLED;
	211	else process.env.AUTOPILOT_DISABLED = originalDisabled;
	212	}
	213
	214	it("no-ops cleanly when AUTOPILOT_DISABLED=1", async () => {
	215	process.env.ANTHROPIC_API_KEY = "x";
	216	process.env.AUTOPILOT_DISABLED = "1";
	217	let calls = 0;
	218	const out = await runPrTestGeneratorTaskOnce({
	219	findCandidates: async () => {
	220	calls += 1;
	221	return [];
	222	},
	223	dispatcher: async () => ({ ok: true }),
	224	});
	225	expect(calls).toBe(0);
	226	expect(out).toEqual({ considered: 0, dispatched: 0, skipped: 0, failed: 0 });
	227	restoreEnv();
	228	});
	229
	230	it("no-ops cleanly when ANTHROPIC_API_KEY is missing", async () => {
	231	delete process.env.AUTOPILOT_DISABLED;
	232	delete process.env.ANTHROPIC_API_KEY;
	233	let calls = 0;
	234	const out = await runPrTestGeneratorTaskOnce({
	235	findCandidates: async () => {
	236	calls += 1;
	237	return [];
	238	},
	239	dispatcher: async () => ({ ok: true }),
	240	});
	241	expect(calls).toBe(0);
	242	expect(out.dispatched).toBe(0);
	243	restoreEnv();
	244	});
	245
	246	it("counts dispatched / skipped / failed correctly", async () => {
	247	delete process.env.AUTOPILOT_DISABLED;
	248	process.env.ANTHROPIC_API_KEY = "x";
	249
	250	const candidates: PrTestGenCandidate[] = [
	251	{ prId: "pr-ok", prNumber: 1, repositoryId: "r", body: "ok" },
	252	{ prId: "pr-already", prNumber: 2, repositoryId: "r", body: "ok" },
	253	{ prId: "pr-no-files", prNumber: 3, repositoryId: "r", body: "ok" },
	254	{ prId: "pr-fail", prNumber: 4, repositoryId: "r", body: "ok" },
	255	];
	256
	257	const out = await runPrTestGeneratorTaskOnce({
	258	findCandidates: async () => candidates,
	259	dispatcher: async ({ prId }) => {
	260	if (prId === "pr-ok") return { ok: true, written: 1, branch: "x" };
	261	if (prId === "pr-already") return { ok: true, alreadyDone: true };
	262	if (prId === "pr-no-files")
	263	return { ok: false, error: "No candidate source files in diff" };
	264	return { ok: false, error: "real failure" };
	265	},
	266	});
	267
	268	expect(out.considered).toBe(4);
	269	expect(out.dispatched).toBe(1);
	270	expect(out.skipped).toBe(2); // alreadyDone + no-candidate
	271	expect(out.failed).toBe(1);
	272	restoreEnv();
	273	});
	274
	275	it("classifies AI-generated-PR refusals as skipped, not failed", async () => {
	276	delete process.env.AUTOPILOT_DISABLED;
	277	process.env.ANTHROPIC_API_KEY = "x";
	278
	279	const out = await runPrTestGeneratorTaskOnce({
	280	findCandidates: async () => [
	281	{ prId: "pr-spec", prNumber: 9, repositoryId: "r", body: null },
	282	],
	283	dispatcher: async () => ({
	284	ok: false,
	285	error: "PR is AI-generated (ai:spec-implementation); skipping",
	286	}),
	287	});
	288	expect(out.skipped).toBe(1);
	289	expect(out.failed).toBe(0);
	290	restoreEnv();
	291	});
	292	});
	293
	294	// ---------------------------------------------------------------------------
	295	// Injected-Claude end-to-end
	296	// ---------------------------------------------------------------------------
	297
	298	function fakeClient(responseText: string) {
	299	return {
	300	messages: {
	301	create: async () => ({
	302	content: [{ type: "text" as const, text: responseText }],
	303	}),
	304	},
	305	} as any;
	306	}
	307
	308	describe("generateTestsForPr — guards (no DB needed)", () => {
	309	it("returns ok:false when ANTHROPIC_API_KEY is missing and no client injected", async () => {
	310	const originalKey = process.env.ANTHROPIC_API_KEY;
	311	delete process.env.ANTHROPIC_API_KEY;
	312	const out = await generateTestsForPr({
	313	prId: "00000000-0000-0000-0000-000000000000",
	314	mode: "append-commit",
	315	});
	316	expect(out.ok).toBe(false);
	317	expect(out.error).toContain("ANTHROPIC_API_KEY");
	318	if (originalKey !== undefined) process.env.ANTHROPIC_API_KEY = originalKey;
	319	});
	320
	321	it("returns ok:false when the PR id does not resolve", async () => {
	322	const out = await generateTestsForPr({
	323	prId: "00000000-0000-0000-0000-000000000000",
	324	mode: "append-commit",
	325	client: fakeClient('{"patches":[]}'),
	326	});
	327	expect(out.ok).toBe(false);
	328	// "PR not found" when DB is reachable; "PR not found" still surfaces
	329	// because loadPrFacts swallows DB errors and returns null.
	330	expect((out.error \|\| "").toLowerCase()).toContain("not found");
	331	});
	332	});
	333
	334	describe.skipIf(!HAS_DB)(
	335	"generateTestsForPr — end-to-end with fake Claude",
	336	() => {
	337	/**
	338	* Seed a fresh user + repo + bare git repo with a base commit and a
	339	* head branch that adds a single source file. Returns identifiers
	340	* the per-test cases can plug into a PR row.
	341	*/
	342	async function seed(): Promise<{
	343	userId: string;
	344	repoId: string;
	345	ownerName: string;
	346	repoName: string;
	347	headBranch: string;
	348	}> {
	349	const username = `aitests_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
	350	const [u] = await db
	351	.insert(users)
	352	.values({
	353	username,
	354	email: `${username}@example.com`,
	355	passwordHash: "x",
	356	})
	357	.returning({ id: users.id });
	358
	359	const repoName = `subject_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
	360	const [r] = await db
	361	.insert(repositories)
	362	.values({
	363	ownerId: u.id,
	364	name: repoName,
	365	diskPath: `/tmp/${username}/${repoName}`,
	366	defaultBranch: "main",
	367	})
	368	.returning({ id: repositories.id });
	369
	370	await initBareRepo(username, repoName);
	371
	372	// Base commit on main.
	373	const baseSeed = await createOrUpdateFileOnBranch({
	374	owner: username,
	375	name: repoName,
	376	branch: "main",
	377	filePath: "README.md",
	378	bytes: new TextEncoder().encode("# subject\n"),
	379	message: "seed",
	380	authorName: "Seeder",
	381	authorEmail: "s@e.com",
	382	});
	383	if ("error" in baseSeed) throw new Error("base seed failed");
	384
	385	// Head branch adds a source file.
	386	const headBranch = "feature/add-math";
	387	const headSeed = await createOrUpdateFileOnBranch({
	388	owner: username,
	389	name: repoName,
	390	branch: headBranch,
	391	filePath: "src/lib/math.ts",
	392	bytes: new TextEncoder().encode(
	393	"export function add(a: number, b: number): number { return a + b; }\n"
	394	),
	395	message: "add math",
	396	authorName: "Dev",
	397	authorEmail: "d@e.com",
	398	});
	399	if ("error" in headSeed) throw new Error("head seed failed");
	400
	401	return {
	402	userId: u.id,
	403	repoId: r.id,
	404	ownerName: username,
	405	repoName,
	406	headBranch,
	407	};
	408	}
	409
	410	async function insertPr(args: {
	411	repoId: string;
	412	authorId: string;
	413	headBranch: string;
	414	title?: string;
	415	body?: string \| null;
	416	}): Promise<{ id: string; number: number }> {
	417	const [pr] = await db
	418	.insert(pullRequests)
	419	.values({
	420	repositoryId: args.repoId,
	421	authorId: args.authorId,
	422	title: args.title \|\| "Add math helpers",
	423	body: args.body ?? "Adds math helpers.",
	424	baseBranch: "main",
	425	headBranch: args.headBranch,
	426	isDraft: false,
	427	})
	428	.returning({ id: pullRequests.id, number: pullRequests.number });
	429	return { id: pr.id, number: pr.number };
	430	}
	431
	432	const cannedPatch = JSON.stringify({
	433	patches: [
	434	{
	435	path: "src/__tests__/math.test.ts",
	436	new_content:
	437	"import { add } from '../lib/math';\nimport { describe, it, expect } from 'bun:test';\ndescribe('add', () => {\n it('adds two numbers', () => {\n expect(add(2, 3)).toBe(5);\n });\n});\n",
	438	},
	439	],
	440	});
	441
	442	it(
	443	"append-commit mode pushes a test commit onto the PR's head branch",
	444	async () => {
	445	const fixt = await seed();
	446	const pr = await insertPr({
	447	repoId: fixt.repoId,
	448	authorId: fixt.userId,
	449	headBranch: fixt.headBranch,
	450	});
	451
	452	const beforeHead = await resolveRef(
	453	fixt.ownerName,
	454	fixt.repoName,
	455	fixt.headBranch
	456	);
	457
	458	const result = await generateTestsForPr({
	459	prId: pr.id,
	460	mode: "append-commit",
	461	client: fakeClient(cannedPatch),
	462	});
	463
	464	expect(result.ok).toBe(true);
	465	expect(result.written).toBe(1);
	466	expect(result.branch).toBe(fixt.headBranch);
	467	expect(result.prNumber).toBeUndefined(); // append-commit doesn't open a PR
	468
	469	// The head branch should have advanced.
	470	const afterHead = await resolveRef(
	471	fixt.ownerName,
	472	fixt.repoName,
	473	fixt.headBranch
	474	);
	475	expect(afterHead).not.toBe(beforeHead);
	476
	477	// The new test file is on the head branch.
	478	const blob = await getBlob(
	479	fixt.ownerName,
	480	fixt.repoName,
	481	fixt.headBranch,
	482	"src/__tests__/math.test.ts"
	483	);
	484	expect(blob).not.toBeNull();
	485	expect(blob!.content).toContain("expect(add(2, 3)).toBe(5)");
	486
	487	// The original source file should be untouched.
	488	const sourceBlob = await getBlob(
	489	fixt.ownerName,
	490	fixt.repoName,
	491	fixt.headBranch,
	492	"src/lib/math.ts"
	493	);
	494	expect(sourceBlob!.content).toContain("export function add");
	495
	496	// Marker comment landed on the PR.
	497	const markerComments = await db
	498	.select({ body: prComments.body })
	499	.from(prComments)
	500	.where(eq(prComments.pullRequestId, pr.id));
	501	const hasMarker = markerComments.some((row) =>
	502	(row.body \|\| "").includes(AI_TESTS_MARKER)
	503	);
	504	expect(hasMarker).toBe(true);
	505
	506	// Dedupe: a second run must NOT re-write anything.
	507	const second = await generateTestsForPr({
	508	prId: pr.id,
	509	mode: "append-commit",
	510	client: fakeClient(cannedPatch),
	511	});
	512	expect(second.ok).toBe(true);
	513	expect(second.alreadyDone).toBe(true);
	514	},
	515	30_000
	516	);
	517
	518	it(
	519	"follow-up-pr mode opens a new PR against the head branch",
	520	async () => {
	521	const fixt = await seed();
	522	const pr = await insertPr({
	523	repoId: fixt.repoId,
	524	authorId: fixt.userId,
	525	headBranch: fixt.headBranch,
	526	});
	527
	528	const result = await generateTestsForPr({
	529	prId: pr.id,
	530	mode: "follow-up-pr",
	531	client: fakeClient(cannedPatch),
	532	});
	533
	534	expect(result.ok).toBe(true);
	535	expect(result.branch).toBeDefined();
	536	expect(result.branch!.startsWith("ai-tests/pr-")).toBe(true);
	537	expect(typeof result.prNumber).toBe("number");
	538
	539	// New branch exists and carries the test file.
	540	expect(
	541	await refExists(fixt.ownerName, fixt.repoName, `refs/heads/${result.branch}`)
	542	).toBe(true);
	543	const blob = await getBlob(
	544	fixt.ownerName,
	545	fixt.repoName,
	546	result.branch!,
	547	"src/__tests__/math.test.ts"
	548	);
	549	expect(blob).not.toBeNull();
	550
	551	// Follow-up PR row exists with the right base/head.
	552	const followUps = await db
	553	.select({
	554	number: pullRequests.number,
	555	title: pullRequests.title,
	556	baseBranch: pullRequests.baseBranch,
	557	headBranch: pullRequests.headBranch,
	558	body: pullRequests.body,
	559	})
	560	.from(pullRequests)
	561	.where(eq(pullRequests.repositoryId, fixt.repoId));
	562
	563	const followUp = followUps.find((p) => p.number === result.prNumber);
	564	expect(followUp).toBeDefined();
	565	expect(followUp!.headBranch).toBe(result.branch!);
	566	expect(followUp!.baseBranch).toBe(fixt.headBranch);
	567	expect(followUp!.title).toContain(`+tests for #${pr.number}`);
	568	expect(followUp!.body).toContain(AI_TESTS_MARKER);
	569	expect(followUp!.body).toContain(AI_TESTS_LABEL);
	570
	571	// Dedupe: a second run must NOT open another follow-up.
	572	const second = await generateTestsForPr({
	573	prId: pr.id,
	574	mode: "follow-up-pr",
	575	client: fakeClient(cannedPatch),
	576	});
	577	expect(second.ok).toBe(true);
	578	expect(second.alreadyDone).toBe(true);
	579
	580	// Still only one follow-up PR exists.
	581	const followUpsAfter = await db
	582	.select({ number: pullRequests.number, title: pullRequests.title })
	583	.from(pullRequests)
	584	.where(eq(pullRequests.repositoryId, fixt.repoId));
	585	const aiOnes = followUpsAfter.filter((p) =>
	586	(p.title \|\| "").startsWith("[tests] +tests for")
	587	);
	588	expect(aiOnes.length).toBe(1);
	589	},
	590	30_000
	591	);
	592
	593	it("refuses a spec-implementation PR (recursion guard)", async () => {
	594	const fixt = await seed();
	595	const pr = await insertPr({
	596	repoId: fixt.repoId,
	597	authorId: fixt.userId,
	598	headBranch: fixt.headBranch,
	599	body: "<!-- gluecron:ai-spec-implementation:v1 -->\nSpec body.",
	600	});
	601
	602	const result = await generateTestsForPr({
	603	prId: pr.id,
	604	mode: "append-commit",
	605	client: fakeClient(cannedPatch),
	606	});
	607	expect(result.ok).toBe(false);
	608	expect((result.error \|\| "").toLowerCase()).toContain("ai-generated");
	609	});
	610	}
	611	);