Commit232d974unknown_key
feat(ai-tests): auto-generate tests on PR open — opt-in autopilot task + manual button
feat(ai-tests): auto-generate tests on PR open — opt-in autopilot task + manual button Adds the test suite for src/lib/ai-test-generator.ts and src/lib/autopilot-pr-test-generator.ts. The generator + autopilot task + UI + repo-settings toggle landed in 0c3eee5 / 1d4ff60; this commit completes the feature with the missing test coverage: - Pure helpers: isCandidateSourceFile / buildTestsForPrPrompt / testsBranchName / looksLikeSpecPr / renderFollowUpPrBody - Autopilot DI: dispatched/skipped/failed bucketing, AUTOPILOT_DISABLED + no-ANTHROPIC_API_KEY no-ops, spec-PR refusal classified as skip - End-to-end (HAS_DB-gated): append-commit mode lands tests on the PR's head branch, follow-up-pr mode opens a new PR against the head branch with the right base/title/body, idempotent dedupe via the ai:added-tests marker comment, recursion guard refuses spec PRs 20 pass / 3 HAS_DB-skip / 0 fail. bunx tsc --noEmit clean. https://claude.ai/code/session_01KdpgsaGpmEExJAz3yhFLvx
1 file changed+611−0232d97455b97451c2c82720b4df68a6bd551bf87
1 changed file+611−0
Addedsrc/__tests__/ai-test-generator.test.ts+611−0View fileUnifiedSplit
@@ -0,0 +1,611 @@
1/**
2 * Tests for src/lib/ai-test-generator.ts.
3 *
4 * Two layers:
5 *
6 * 1. Pure helpers — no DB, no Claude. Always run.
7 * - isCandidateSourceFile filters test files / docs / configs.
8 * - buildTestsForPrPrompt embeds the source file + framework hint.
9 * - testsBranchName respects an override and otherwise embeds the
10 * PR number + timestamp.
11 * - looksLikeSpecPr detects spec-implementation PR bodies.
12 * - renderFollowUpPrBody includes the marker, label, and file list.
13 * - autopilot dispatch counts on injected stubs.
14 *
15 * 2. End-to-end generateTestsForPr — injected fake Claude client + a
16 * real bare repo on disk. DB-backed steps (PR insert, label upsert,
17 * audit) are gated on HAS_DB so the DB-less sandbox still exercises
18 * the git-side write path.
19 */
20
21import { describe, it, expect, beforeAll, afterAll } from "bun:test";
22import { join } from "path";
23import { rm, mkdir } from "fs/promises";
24import { eq } from "drizzle-orm";
25import {
26 AI_TESTS_LABEL,
27 AI_TESTS_MARKER,
28 buildTestsForPrPrompt,
29 generateTestsForPr,
30 isCandidateSourceFile,
31 looksLikeSpecPr,
32 renderFollowUpPrBody,
33 testsBranchName,
34} from "../lib/ai-test-generator";
35import {
36 runPrTestGeneratorTaskOnce,
37 type PrTestGenCandidate,
38} from "../lib/autopilot-pr-test-generator";
39import {
40 initBareRepo,
41 createOrUpdateFileOnBranch,
42 getBlob,
43 refExists,
44 resolveRef,
45} from "../git/repository";
46import { db } from "../db";
47import {
48 prComments,
49 pullRequests,
50 repositories,
51 users,
52} from "../db/schema";
53
54const HAS_DB = Boolean(process.env.DATABASE_URL);
55
56const TEST_REPOS = join(
57 import.meta.dir,
58 "../../.test-repos-ai-test-gen-" + Date.now()
59);
60
61beforeAll(async () => {
62 process.env.GIT_REPOS_PATH = TEST_REPOS;
63 await rm(TEST_REPOS, { recursive: true, force: true });
64 await mkdir(TEST_REPOS, { recursive: true });
65});
66
67afterAll(async () => {
68 await rm(TEST_REPOS, { recursive: true, force: true });
69});
70
71// ---------------------------------------------------------------------------
72// Pure helpers
73// ---------------------------------------------------------------------------
74
75describe("isCandidateSourceFile", () => {
76 it("accepts ordinary TypeScript / JS / Python / Go sources", () => {
77 expect(isCandidateSourceFile("src/lib/math.ts")).toBe(true);
78 expect(isCandidateSourceFile("pkg/widget.py")).toBe(true);
79 expect(isCandidateSourceFile("cmd/main.go")).toBe(true);
80 expect(isCandidateSourceFile("lib/util.js")).toBe(true);
81 });
82
83 it("rejects test files", () => {
84 expect(isCandidateSourceFile("src/__tests__/math.test.ts")).toBe(false);
85 expect(isCandidateSourceFile("tests/test_widget.py")).toBe(false);
86 expect(isCandidateSourceFile("foo_test.go")).toBe(false);
87 expect(isCandidateSourceFile("src/lib/math.spec.ts")).toBe(false);
88 });
89
90 it("rejects docs and configs", () => {
91 expect(isCandidateSourceFile("README.md")).toBe(false);
92 expect(isCandidateSourceFile("package.json")).toBe(false);
93 expect(isCandidateSourceFile("vitest.config.ts")).toBe(true); // .ts is code-shaped
94 expect(isCandidateSourceFile("public/logo.png")).toBe(false);
95 expect(isCandidateSourceFile("docs/intro.md")).toBe(false);
96 });
97
98 it("rejects build / dependency directories", () => {
99 expect(isCandidateSourceFile("node_modules/foo/index.js")).toBe(false);
100 expect(isCandidateSourceFile("dist/bundle.js")).toBe(false);
101 expect(isCandidateSourceFile(".github/workflows/ci.yml")).toBe(false);
102 });
103
104 it("rejects path-traversal attempts", () => {
105 expect(isCandidateSourceFile("../etc/passwd")).toBe(false);
106 expect(isCandidateSourceFile("src/../../oops.ts")).toBe(false);
107 });
108});
109
110describe("buildTestsForPrPrompt", () => {
111 it("embeds path, framework, language, and source code", () => {
112 const out = buildTestsForPrPrompt({
113 filePath: "src/lib/math.ts",
114 language: "typescript",
115 framework: "bun:test",
116 sourceCode: "export function add(a: number, b: number) { return a + b; }",
117 prTitle: "Add math helpers",
118 });
119 expect(out).toContain("src/lib/math.ts");
120 expect(out).toContain("bun:test");
121 expect(out).toContain("typescript");
122 expect(out).toContain("export function add");
123 expect(out).toContain("Add math helpers");
124 // JSON schema cue MUST be present so the model returns parseable output.
125 expect(out).toContain('"patches"');
126 expect(out).toContain('"new_content"');
127 });
128
129 it("truncates very large source files", () => {
130 const huge = "a".repeat(50_000);
131 const out = buildTestsForPrPrompt({
132 filePath: "big.ts",
133 language: "typescript",
134 framework: "bun:test",
135 sourceCode: huge,
136 prTitle: "huge file",
137 });
138 expect(out.length).toBeLessThan(huge.length + 4_000);
139 expect(out).toContain("truncated");
140 });
141});
142
143describe("testsBranchName", () => {
144 it("honours an override", () => {
145 expect(testsBranchName(42, "custom/branch")).toBe("custom/branch");
146 });
147
148 it("uses ai-tests/pr-<n>-<timestamp> by default", () => {
149 const name = testsBranchName(42);
150 expect(name.startsWith("ai-tests/pr-42-")).toBe(true);
151 const ts = name.slice("ai-tests/pr-42-".length);
152 expect(/^\d+$/.test(ts)).toBe(true);
153 });
154});
155
156describe("looksLikeSpecPr", () => {
157 it("detects the spec-implementation marker", () => {
158 expect(
159 looksLikeSpecPr("<!-- gluecron:ai-spec-implementation:v1 -->\nblah")
160 ).toBe(true);
161 });
162
163 it("detects the spec label citation", () => {
164 expect(looksLikeSpecPr("Label: `ai:spec-implementation`")).toBe(true);
165 });
166
167 it("returns false for empty / null / unrelated bodies", () => {
168 expect(looksLikeSpecPr(null)).toBe(false);
169 expect(looksLikeSpecPr("")).toBe(false);
170 expect(looksLikeSpecPr("Normal PR body")).toBe(false);
171 });
172});
173
174describe("renderFollowUpPrBody", () => {
175 it("includes the marker, label, branch, and file list", () => {
176 const body = renderFollowUpPrBody({
177 originalPrNumber: 7,
178 branch: "ai-tests/pr-7-1234",
179 written: ["src/__tests__/math.test.ts"],
180 });
181 expect(body).toContain(AI_TESTS_MARKER);
182 expect(body).toContain("for PR #7");
183 expect(body).toContain(AI_TESTS_LABEL);
184 expect(body).toContain("ai-tests/pr-7-1234");
185 expect(body).toContain("src/__tests__/math.test.ts");
186 });
187
188 it("renders cleanly with no files", () => {
189 const body = renderFollowUpPrBody({
190 originalPrNumber: 1,
191 branch: "ai-tests/pr-1-0",
192 written: [],
193 });
194 expect(body).toContain(AI_TESTS_MARKER);
195 expect(body).toContain("_(none)_");
196 });
197});
198
199// ---------------------------------------------------------------------------
200// Autopilot task — DI seam, no DB / no Claude
201// ---------------------------------------------------------------------------
202
203describe("runPrTestGeneratorTaskOnce", () => {
204 const originalKey = process.env.ANTHROPIC_API_KEY;
205 const originalDisabled = process.env.AUTOPILOT_DISABLED;
206
207 function restoreEnv() {
208 if (originalKey === undefined) delete process.env.ANTHROPIC_API_KEY;
209 else process.env.ANTHROPIC_API_KEY = originalKey;
210 if (originalDisabled === undefined) delete process.env.AUTOPILOT_DISABLED;
211 else process.env.AUTOPILOT_DISABLED = originalDisabled;
212 }
213
214 it("no-ops cleanly when AUTOPILOT_DISABLED=1", async () => {
215 process.env.ANTHROPIC_API_KEY = "x";
216 process.env.AUTOPILOT_DISABLED = "1";
217 let calls = 0;
218 const out = await runPrTestGeneratorTaskOnce({
219 findCandidates: async () => {
220 calls += 1;
221 return [];
222 },
223 dispatcher: async () => ({ ok: true }),
224 });
225 expect(calls).toBe(0);
226 expect(out).toEqual({ considered: 0, dispatched: 0, skipped: 0, failed: 0 });
227 restoreEnv();
228 });
229
230 it("no-ops cleanly when ANTHROPIC_API_KEY is missing", async () => {
231 delete process.env.AUTOPILOT_DISABLED;
232 delete process.env.ANTHROPIC_API_KEY;
233 let calls = 0;
234 const out = await runPrTestGeneratorTaskOnce({
235 findCandidates: async () => {
236 calls += 1;
237 return [];
238 },
239 dispatcher: async () => ({ ok: true }),
240 });
241 expect(calls).toBe(0);
242 expect(out.dispatched).toBe(0);
243 restoreEnv();
244 });
245
246 it("counts dispatched / skipped / failed correctly", async () => {
247 delete process.env.AUTOPILOT_DISABLED;
248 process.env.ANTHROPIC_API_KEY = "x";
249
250 const candidates: PrTestGenCandidate[] = [
251 { prId: "pr-ok", prNumber: 1, repositoryId: "r", body: "ok" },
252 { prId: "pr-already", prNumber: 2, repositoryId: "r", body: "ok" },
253 { prId: "pr-no-files", prNumber: 3, repositoryId: "r", body: "ok" },
254 { prId: "pr-fail", prNumber: 4, repositoryId: "r", body: "ok" },
255 ];
256
257 const out = await runPrTestGeneratorTaskOnce({
258 findCandidates: async () => candidates,
259 dispatcher: async ({ prId }) => {
260 if (prId === "pr-ok") return { ok: true, written: 1, branch: "x" };
261 if (prId === "pr-already") return { ok: true, alreadyDone: true };
262 if (prId === "pr-no-files")
263 return { ok: false, error: "No candidate source files in diff" };
264 return { ok: false, error: "real failure" };
265 },
266 });
267
268 expect(out.considered).toBe(4);
269 expect(out.dispatched).toBe(1);
270 expect(out.skipped).toBe(2); // alreadyDone + no-candidate
271 expect(out.failed).toBe(1);
272 restoreEnv();
273 });
274
275 it("classifies AI-generated-PR refusals as skipped, not failed", async () => {
276 delete process.env.AUTOPILOT_DISABLED;
277 process.env.ANTHROPIC_API_KEY = "x";
278
279 const out = await runPrTestGeneratorTaskOnce({
280 findCandidates: async () => [
281 { prId: "pr-spec", prNumber: 9, repositoryId: "r", body: null },
282 ],
283 dispatcher: async () => ({
284 ok: false,
285 error: "PR is AI-generated (ai:spec-implementation); skipping",
286 }),
287 });
288 expect(out.skipped).toBe(1);
289 expect(out.failed).toBe(0);
290 restoreEnv();
291 });
292});
293
294// ---------------------------------------------------------------------------
295// Injected-Claude end-to-end
296// ---------------------------------------------------------------------------
297
298function fakeClient(responseText: string) {
299 return {
300 messages: {
301 create: async () => ({
302 content: [{ type: "text" as const, text: responseText }],
303 }),
304 },
305 } as any;
306}
307
308describe("generateTestsForPr — guards (no DB needed)", () => {
309 it("returns ok:false when ANTHROPIC_API_KEY is missing and no client injected", async () => {
310 const originalKey = process.env.ANTHROPIC_API_KEY;
311 delete process.env.ANTHROPIC_API_KEY;
312 const out = await generateTestsForPr({
313 prId: "00000000-0000-0000-0000-000000000000",
314 mode: "append-commit",
315 });
316 expect(out.ok).toBe(false);
317 expect(out.error).toContain("ANTHROPIC_API_KEY");
318 if (originalKey !== undefined) process.env.ANTHROPIC_API_KEY = originalKey;
319 });
320
321 it("returns ok:false when the PR id does not resolve", async () => {
322 const out = await generateTestsForPr({
323 prId: "00000000-0000-0000-0000-000000000000",
324 mode: "append-commit",
325 client: fakeClient('{"patches":[]}'),
326 });
327 expect(out.ok).toBe(false);
328 // "PR not found" when DB is reachable; "PR not found" still surfaces
329 // because loadPrFacts swallows DB errors and returns null.
330 expect((out.error || "").toLowerCase()).toContain("not found");
331 });
332});
333
334describe.skipIf(!HAS_DB)(
335 "generateTestsForPr — end-to-end with fake Claude",
336 () => {
337 /**
338 * Seed a fresh user + repo + bare git repo with a base commit and a
339 * head branch that adds a single source file. Returns identifiers
340 * the per-test cases can plug into a PR row.
341 */
342 async function seed(): Promise<{
343 userId: string;
344 repoId: string;
345 ownerName: string;
346 repoName: string;
347 headBranch: string;
348 }> {
349 const username = `aitests_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
350 const [u] = await db
351 .insert(users)
352 .values({
353 username,
354 email: `${username}@example.com`,
355 passwordHash: "x",
356 })
357 .returning({ id: users.id });
358
359 const repoName = `subject_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
360 const [r] = await db
361 .insert(repositories)
362 .values({
363 ownerId: u.id,
364 name: repoName,
365 diskPath: `/tmp/${username}/${repoName}`,
366 defaultBranch: "main",
367 })
368 .returning({ id: repositories.id });
369
370 await initBareRepo(username, repoName);
371
372 // Base commit on main.
373 const baseSeed = await createOrUpdateFileOnBranch({
374 owner: username,
375 name: repoName,
376 branch: "main",
377 filePath: "README.md",
378 bytes: new TextEncoder().encode("# subject\n"),
379 message: "seed",
380 authorName: "Seeder",
381 authorEmail: "s@e.com",
382 });
383 if ("error" in baseSeed) throw new Error("base seed failed");
384
385 // Head branch adds a source file.
386 const headBranch = "feature/add-math";
387 const headSeed = await createOrUpdateFileOnBranch({
388 owner: username,
389 name: repoName,
390 branch: headBranch,
391 filePath: "src/lib/math.ts",
392 bytes: new TextEncoder().encode(
393 "export function add(a: number, b: number): number { return a + b; }\n"
394 ),
395 message: "add math",
396 authorName: "Dev",
397 authorEmail: "d@e.com",
398 });
399 if ("error" in headSeed) throw new Error("head seed failed");
400
401 return {
402 userId: u.id,
403 repoId: r.id,
404 ownerName: username,
405 repoName,
406 headBranch,
407 };
408 }
409
410 async function insertPr(args: {
411 repoId: string;
412 authorId: string;
413 headBranch: string;
414 title?: string;
415 body?: string | null;
416 }): Promise<{ id: string; number: number }> {
417 const [pr] = await db
418 .insert(pullRequests)
419 .values({
420 repositoryId: args.repoId,
421 authorId: args.authorId,
422 title: args.title || "Add math helpers",
423 body: args.body ?? "Adds math helpers.",
424 baseBranch: "main",
425 headBranch: args.headBranch,
426 isDraft: false,
427 })
428 .returning({ id: pullRequests.id, number: pullRequests.number });
429 return { id: pr.id, number: pr.number };
430 }
431
432 const cannedPatch = JSON.stringify({
433 patches: [
434 {
435 path: "src/__tests__/math.test.ts",
436 new_content:
437 "import { add } from '../lib/math';\nimport { describe, it, expect } from 'bun:test';\ndescribe('add', () => {\n it('adds two numbers', () => {\n expect(add(2, 3)).toBe(5);\n });\n});\n",
438 },
439 ],
440 });
441
442 it(
443 "append-commit mode pushes a test commit onto the PR's head branch",
444 async () => {
445 const fixt = await seed();
446 const pr = await insertPr({
447 repoId: fixt.repoId,
448 authorId: fixt.userId,
449 headBranch: fixt.headBranch,
450 });
451
452 const beforeHead = await resolveRef(
453 fixt.ownerName,
454 fixt.repoName,
455 fixt.headBranch
456 );
457
458 const result = await generateTestsForPr({
459 prId: pr.id,
460 mode: "append-commit",
461 client: fakeClient(cannedPatch),
462 });
463
464 expect(result.ok).toBe(true);
465 expect(result.written).toBe(1);
466 expect(result.branch).toBe(fixt.headBranch);
467 expect(result.prNumber).toBeUndefined(); // append-commit doesn't open a PR
468
469 // The head branch should have advanced.
470 const afterHead = await resolveRef(
471 fixt.ownerName,
472 fixt.repoName,
473 fixt.headBranch
474 );
475 expect(afterHead).not.toBe(beforeHead);
476
477 // The new test file is on the head branch.
478 const blob = await getBlob(
479 fixt.ownerName,
480 fixt.repoName,
481 fixt.headBranch,
482 "src/__tests__/math.test.ts"
483 );
484 expect(blob).not.toBeNull();
485 expect(blob!.content).toContain("expect(add(2, 3)).toBe(5)");
486
487 // The original source file should be untouched.
488 const sourceBlob = await getBlob(
489 fixt.ownerName,
490 fixt.repoName,
491 fixt.headBranch,
492 "src/lib/math.ts"
493 );
494 expect(sourceBlob!.content).toContain("export function add");
495
496 // Marker comment landed on the PR.
497 const markerComments = await db
498 .select({ body: prComments.body })
499 .from(prComments)
500 .where(eq(prComments.pullRequestId, pr.id));
501 const hasMarker = markerComments.some((row) =>
502 (row.body || "").includes(AI_TESTS_MARKER)
503 );
504 expect(hasMarker).toBe(true);
505
506 // Dedupe: a second run must NOT re-write anything.
507 const second = await generateTestsForPr({
508 prId: pr.id,
509 mode: "append-commit",
510 client: fakeClient(cannedPatch),
511 });
512 expect(second.ok).toBe(true);
513 expect(second.alreadyDone).toBe(true);
514 },
515 30_000
516 );
517
518 it(
519 "follow-up-pr mode opens a new PR against the head branch",
520 async () => {
521 const fixt = await seed();
522 const pr = await insertPr({
523 repoId: fixt.repoId,
524 authorId: fixt.userId,
525 headBranch: fixt.headBranch,
526 });
527
528 const result = await generateTestsForPr({
529 prId: pr.id,
530 mode: "follow-up-pr",
531 client: fakeClient(cannedPatch),
532 });
533
534 expect(result.ok).toBe(true);
535 expect(result.branch).toBeDefined();
536 expect(result.branch!.startsWith("ai-tests/pr-")).toBe(true);
537 expect(typeof result.prNumber).toBe("number");
538
539 // New branch exists and carries the test file.
540 expect(
541 await refExists(fixt.ownerName, fixt.repoName, `refs/heads/${result.branch}`)
542 ).toBe(true);
543 const blob = await getBlob(
544 fixt.ownerName,
545 fixt.repoName,
546 result.branch!,
547 "src/__tests__/math.test.ts"
548 );
549 expect(blob).not.toBeNull();
550
551 // Follow-up PR row exists with the right base/head.
552 const followUps = await db
553 .select({
554 number: pullRequests.number,
555 title: pullRequests.title,
556 baseBranch: pullRequests.baseBranch,
557 headBranch: pullRequests.headBranch,
558 body: pullRequests.body,
559 })
560 .from(pullRequests)
561 .where(eq(pullRequests.repositoryId, fixt.repoId));
562
563 const followUp = followUps.find((p) => p.number === result.prNumber);
564 expect(followUp).toBeDefined();
565 expect(followUp!.headBranch).toBe(result.branch!);
566 expect(followUp!.baseBranch).toBe(fixt.headBranch);
567 expect(followUp!.title).toContain(`+tests for #${pr.number}`);
568 expect(followUp!.body).toContain(AI_TESTS_MARKER);
569 expect(followUp!.body).toContain(AI_TESTS_LABEL);
570
571 // Dedupe: a second run must NOT open another follow-up.
572 const second = await generateTestsForPr({
573 prId: pr.id,
574 mode: "follow-up-pr",
575 client: fakeClient(cannedPatch),
576 });
577 expect(second.ok).toBe(true);
578 expect(second.alreadyDone).toBe(true);
579
580 // Still only one follow-up PR exists.
581 const followUpsAfter = await db
582 .select({ number: pullRequests.number, title: pullRequests.title })
583 .from(pullRequests)
584 .where(eq(pullRequests.repositoryId, fixt.repoId));
585 const aiOnes = followUpsAfter.filter((p) =>
586 (p.title || "").startsWith("[tests] +tests for")
587 );
588 expect(aiOnes.length).toBe(1);
589 },
590 30_000
591 );
592
593 it("refuses a spec-implementation PR (recursion guard)", async () => {
594 const fixt = await seed();
595 const pr = await insertPr({
596 repoId: fixt.repoId,
597 authorId: fixt.userId,
598 headBranch: fixt.headBranch,
599 body: "<!-- gluecron:ai-spec-implementation:v1 -->\nSpec body.",
600 });
601
602 const result = await generateTestsForPr({
603 prId: pr.id,
604 mode: "append-commit",
605 client: fakeClient(cannedPatch),
606 });
607 expect(result.ok).toBe(false);
608 expect((result.error || "").toLowerCase()).toContain("ai-generated");
609 });
610 }
611);
0612