Blame · Line-by-line history

`autopilot-ai-tasks.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

autopilot-ai-tasks.test.tsBlame438 lines · 1 contributor

Normal view Raw

2b9055eClaude	1	/**
	2	* Block K3 — autopilot AI-driver task tests.
	3	*
	4	* Uses the dependency-injection seams added in `src/lib/autopilot.ts`
	5	* (`runAutoMergeSweep`) and `src/lib/ai-build-tasks.ts`
	6	* (`runAiBuildTaskOnce`) so we never hit the DB or the AI client.
	7	*
	8	* Covers the contract called out in the K3 spec:
	9	* - auto-merge-sweep skips drafts
	10	* - auto-merge-sweep skips archived repos
	11	* - auto-merge-sweep invokes merge exactly once per `merge:true` PR
	12	* - ai-build dispatches against `ai:build`-labelled issues
	13	* - ai-build skips an issue with a marker comment already present
	14	* - both tasks no-op cleanly when AUTOPILOT_DISABLED=1
	15	*/
	16
	17	import { describe, it, expect, beforeEach, afterEach } from "bun:test";
	18	import {
	19	startAutopilot,
	20	runAutoMergeSweep,
	21	type AutoMergeSweepDeps,
	22	type AutopilotTask,
	23	} from "../lib/autopilot";
	24	import {
	25	runAiBuildTaskOnce,
	26	AI_BUILD_MARKER,
	27	type AiBuildCandidate,
	28	} from "../lib/ai-build-tasks";
	29	import type {
	30	AutoMergeContext,
	31	AutoMergeDecision,
	32	} from "../lib/auto-merge";
	33	import type { PerformMergeResult } from "../lib/pr-merge";
	34
	35	// ---------------------------------------------------------------------------
	36	// Test fixtures
	37	// ---------------------------------------------------------------------------
	38
	39	type Candidate = Parameters<NonNullable<AutoMergeSweepDeps["merge"]>>[0];
	40
	41	function makeCandidate(overrides: Partial<Candidate> = {}): Candidate {
	42	return {
	43	prId: "pr-1",
	44	prNumber: 42,
	45	prTitle: "Test PR",
	46	prBody: "Closes #99",
	47	baseBranch: "main",
	48	headBranch: "feature",
	49	isDraft: false,
	50	repositoryId: "repo-1",
	51	authorUserId: "user-1",
	52	ownerUsername: "alice",
	53	repoName: "demo",
	54	state: "open",
	55	...overrides,
	56	};
	57	}
	58
	59	const okMerge: PerformMergeResult = {
	60	ok: true,
	61	closedIssueNumbers: [],
	62	resolvedFiles: [],
	63	};
	64
	65	const allowDecision: AutoMergeDecision = {
	66	merge: true,
	67	reason: "All auto-merge conditions met for 'main'.",
	68	};
	69
	70	const blockDecision: AutoMergeDecision = {
	71	merge: false,
	72	reason: "blocked",
	73	blocking: ["draft"],
	74	};
	75
	76	// ---------------------------------------------------------------------------
	77	// auto-merge-sweep
	78	// ---------------------------------------------------------------------------
	79
	80	describe("auto-merge-sweep", () => {
	81	it("skips drafts at the candidate-finder layer (drafts must never enter the loop)", async () => {
	82	// The default findCandidates filters drafts via SQL. We model that
	83	// contract here: when the caller supplies drafts, the sweep STILL
	84	// delegates the merge() call ONLY for non-draft PRs whose decision is
	85	// merge:true. Drafts that somehow leak through must surface as blocked
	86	// (decideAutoMerge will reject them).
	87	const draft = makeCandidate({ prId: "draft-1", isDraft: true });
	88	const open = makeCandidate({ prId: "open-1", isDraft: false });
	89	const mergeCalls: string[] = [];
	90
	91	const summary = await runAutoMergeSweep({
	92	findCandidates: async () => [draft, open],
	93	// Simulate evaluateAutoMerge: drafts get blocked, opens get allowed.
	94	evaluate: async (ctx: AutoMergeContext) =>
	95	ctx.isDraft ? blockDecision : allowDecision,
	96	merge: async (cand) => {
	97	mergeCalls.push(cand.prId);
	98	return okMerge;
	99	},
	100	recordAttempt: async () => {},
	101	onMerged: async () => {},
	102	onMergeFailed: async () => {},
	103	shouldShortCircuitAi: async () => false,
	104	});
	105
	106	expect(mergeCalls).toEqual(["open-1"]);
	107	expect(summary.evaluated).toBe(2);
	108	expect(summary.merged).toBe(1);
	109	expect(summary.blocked).toBe(1);
	110	});
	111
	112	it("skips archived repos (no candidates surfaced from the finder)", async () => {
	113	// The default finder excludes archived repos in SQL. We simulate by
	114	// supplying an empty result and confirming a clean no-op summary.
	115	let findCalled = false;
	116	let mergeCalled = 0;
	117	const summary = await runAutoMergeSweep({
	118	findCandidates: async () => {
	119	findCalled = true;
	120	return [];
	121	},
	122	evaluate: async () => allowDecision,
	123	merge: async () => {
	124	mergeCalled += 1;
	125	return okMerge;
	126	},
	127	recordAttempt: async () => {},
	128	onMerged: async () => {},
	129	onMergeFailed: async () => {},
	130	shouldShortCircuitAi: async () => false,
	131	});
	132	expect(findCalled).toBe(true);
	133	expect(mergeCalled).toBe(0);
	134	expect(summary).toEqual({ evaluated: 0, merged: 0, blocked: 0 });
	135	});
	136
	137	it("invokes the merge function exactly once per merge:true PR and records an evaluated audit per PR", async () => {
	138	const candidates = [
	139	makeCandidate({ prId: "a" }),
	140	makeCandidate({ prId: "b" }),
	141	makeCandidate({ prId: "c" }),
	142	];
	143	const mergeCalls: string[] = [];
	144	const evaluatedAuditCalls: string[] = [];
	145	const mergedSuccessCalls: string[] = [];
	146
	147	const summary = await runAutoMergeSweep({
	148	findCandidates: async () => candidates,
	149	// 'a' and 'c' are allowed, 'b' is blocked.
	150	evaluate: async (ctx) =>
	151	ctx.pullRequestId === "b" ? blockDecision : allowDecision,
	152	merge: async (cand) => {
	153	mergeCalls.push(cand.prId);
	154	return okMerge;
	155	},
	156	recordAttempt: async (_repoId, prId) => {
	157	evaluatedAuditCalls.push(prId);
	158	},
	159	onMerged: async (cand) => {
	160	mergedSuccessCalls.push(cand.prId);
	161	},
	162	onMergeFailed: async () => {},
	163	shouldShortCircuitAi: async () => false,
	164	});
	165
	166	expect(mergeCalls).toEqual(["a", "c"]);
	167	expect(mergedSuccessCalls).toEqual(["a", "c"]);
	168	// recordAttempt fires for EVERY evaluation, not just successes.
	169	expect(evaluatedAuditCalls.sort()).toEqual(["a", "b", "c"]);
	170	expect(summary).toEqual({ evaluated: 3, merged: 2, blocked: 1 });
	171	});
	172
	173	it("emits the merge_failed audit when performMerge returns ok=false (and does NOT emit merged audit)", async () => {
	174	const cand = makeCandidate({ prId: "x" });
	175	let mergedHits = 0;
	176	let failedHits = 0;
	177
	178	const summary = await runAutoMergeSweep({
	179	findCandidates: async () => [cand],
	180	evaluate: async () => allowDecision,
	181	merge: async () => ({
	182	ok: false,
	183	error: "git update-ref failed: not a fast-forward",
	184	closedIssueNumbers: [],
	185	resolvedFiles: [],
	186	}),
	187	recordAttempt: async () => {},
	188	onMerged: async () => {
	189	mergedHits += 1;
	190	},
	191	onMergeFailed: async () => {
	192	failedHits += 1;
	193	},
	194	shouldShortCircuitAi: async () => false,
	195	});
	196
	197	expect(mergedHits).toBe(0);
	198	expect(failedHits).toBe(1);
	199	expect(summary).toEqual({ evaluated: 1, merged: 0, blocked: 1 });
	200	});
	201
	202	it("short-circuits AI-required rules when ANTHROPIC_API_KEY is unset (logged as blocked, not error)", async () => {
	203	const cand = makeCandidate({ prId: "ai-1" });
	204	let evaluateHits = 0;
	205	let mergeHits = 0;
	206
	207	const summary = await runAutoMergeSweep({
	208	findCandidates: async () => [cand],
	209	shouldShortCircuitAi: async () => true,
	210	evaluate: async () => {
	211	evaluateHits += 1;
	212	return allowDecision;
	213	},
	214	merge: async () => {
	215	mergeHits += 1;
	216	return okMerge;
	217	},
	218	recordAttempt: async () => {},
	219	onMerged: async () => {},
	220	onMergeFailed: async () => {},
	221	});
	222	expect(evaluateHits).toBe(0); // never invoked
	223	expect(mergeHits).toBe(0); // never invoked
	224	expect(summary).toEqual({ evaluated: 1, merged: 0, blocked: 1 });
	225	});
	226
	227	it("never throws even when the candidate-finder throws — returns zero summary", async () => {
	228	const summary = await runAutoMergeSweep({
	229	findCandidates: async () => {
	230	throw new Error("db blew up");
	231	},
	232	evaluate: async () => allowDecision,
	233	merge: async () => okMerge,
	234	recordAttempt: async () => {},
	235	onMerged: async () => {},
	236	onMergeFailed: async () => {},
	237	shouldShortCircuitAi: async () => false,
	238	});
	239	expect(summary).toEqual({ evaluated: 0, merged: 0, blocked: 0 });
	240	});
	241
	242	it("isolates per-PR failures — a thrown merge() doesn't stop later PRs", async () => {
	243	const candidates = [
	244	makeCandidate({ prId: "first" }),
	245	makeCandidate({ prId: "second" }),
	246	];
	247	const mergeCalls: string[] = [];
	248	const summary = await runAutoMergeSweep({
	249	findCandidates: async () => candidates,
	250	evaluate: async () => allowDecision,
	251	merge: async (cand) => {
	252	mergeCalls.push(cand.prId);
	253	if (cand.prId === "first") throw new Error("kaboom");
	254	return okMerge;
	255	},
	256	recordAttempt: async () => {},
	257	onMerged: async () => {},
	258	onMergeFailed: async () => {},
	259	shouldShortCircuitAi: async () => false,
	260	});
	261	expect(mergeCalls).toEqual(["first", "second"]);
	262	expect(summary.evaluated).toBe(2);
	263	expect(summary.merged).toBe(1);
	264	expect(summary.blocked).toBe(1);
	265	});
	266	});
	267
	268	// ---------------------------------------------------------------------------
	269	// ai-build-from-issues
	270	// ---------------------------------------------------------------------------
	271
	272	function makeAiBuildCandidate(
	273	overrides: Partial<AiBuildCandidate> = {}
	274	): AiBuildCandidate {
	275	return {
	276	issueId: "issue-1",
	277	issueNumber: 7,
	278	issueTitle: "Add a sparkle button",
	279	issueBody: "Should sparkle when clicked.",
	280	repositoryId: "repo-1",
	281	authorUserId: "user-1",
	282	ownerUsername: "alice",
	283	repoName: "demo",
	284	defaultBranch: "main",
	285	...overrides,
	286	};
	287	}
	288
	289	describe("ai-build-from-issues", () => {
	290	it("dispatches the spec-to-PR pipeline for an ai:build-labelled issue", async () => {
	291	const dispatched: Array<{ repoId: string; spec: string; baseRef: string }> =
	292	[];
	293	const markersPosted: string[] = [];
	294
	295	const summary = await runAiBuildTaskOnce({
	296	findCandidates: async () => [makeAiBuildCandidate()],
	297	hasDispatchMarker: async () => false,
	298	hasOpenLinkedPr: async () => false,
	299	dispatcher: async (args) => {
	300	dispatched.push({
	301	repoId: args.repoId,
	302	spec: args.spec,
	303	baseRef: args.baseRef,
	304	});
	305	return { ok: true, prNumber: 123 };
	306	},
	307	postMarkerComment: async (issueId) => {
	308	markersPosted.push(issueId);
	309	},
	310	});
	311
	312	expect(dispatched.length).toBe(1);
	313	expect(dispatched[0].repoId).toBe("repo-1");
	314	expect(dispatched[0].baseRef).toBe("main");
	315	// buildSpecFromIssue prefixes with "Implement:" and includes "Closes #N".
	316	expect(dispatched[0].spec).toContain("Implement: Add a sparkle button");
	317	expect(dispatched[0].spec).toContain("Closes #7");
	318	expect(markersPosted).toEqual(["issue-1"]);
	319	expect(summary).toEqual({ queued: 1, skipped: 0 });
	320	});
	321
	322	it("skips an issue that already has the dispatch marker comment", async () => {
	323	const dispatched: string[] = [];
	324	const markersPosted: string[] = [];
	325
	326	const summary = await runAiBuildTaskOnce({
	327	findCandidates: async () => [makeAiBuildCandidate({ issueId: "i-already" })],
	328	hasDispatchMarker: async (id) => id === "i-already",
	329	hasOpenLinkedPr: async () => false,
	330	dispatcher: async (args) => {
	331	dispatched.push(args.repoId);
	332	return { ok: true, prNumber: 1 };
	333	},
	334	postMarkerComment: async (id) => {
	335	markersPosted.push(id);
	336	},
	337	});
	338
	339	expect(dispatched).toEqual([]);
	340	expect(markersPosted).toEqual([]);
	341	expect(summary).toEqual({ queued: 0, skipped: 1 });
	342	});
	343
	344	it("skips an issue that already has an open PR closing it via close-keywords", async () => {
	345	let dispatcherHits = 0;
	346	const summary = await runAiBuildTaskOnce({
	347	findCandidates: async () => [makeAiBuildCandidate({ issueNumber: 42 })],
	348	hasDispatchMarker: async () => false,
	349	hasOpenLinkedPr: async (_repoId, n) => n === 42,
	350	dispatcher: async () => {
	351	dispatcherHits += 1;
	352	return { ok: true, prNumber: 1 };
	353	},
	354	postMarkerComment: async () => {},
	355	});
	356	expect(dispatcherHits).toBe(0);
	357	expect(summary).toEqual({ queued: 0, skipped: 1 });
	358	});
	359
	360	it("swallows dispatcher failures without crashing the tick", async () => {
	361	let markerPosted = false;
	362	const summary = await runAiBuildTaskOnce({
	363	findCandidates: async () => [makeAiBuildCandidate()],
	364	hasDispatchMarker: async () => false,
	365	hasOpenLinkedPr: async () => false,
	366	dispatcher: async () => {
	367	throw new Error("anthropic 500");
	368	},
	369	postMarkerComment: async () => {
	370	markerPosted = true;
	371	},
	372	});
	373	// Marker still posted (idempotency takes priority over success).
	374	expect(markerPosted).toBe(true);
	375	// The issue counts as queued because we did our part — we don't retry.
	376	expect(summary.queued).toBe(1);
	377	});
	378
	379	it("returns zero summary if findCandidates throws", async () => {
	380	const summary = await runAiBuildTaskOnce({
	381	findCandidates: async () => {
	382	throw new Error("db down");
	383	},
	384	});
	385	expect(summary).toEqual({ queued: 0, skipped: 0 });
	386	});
	387
	388	it("uses the AI_BUILD_MARKER constant in the marker body", async () => {
	389	let receivedBody = "";
	390	await runAiBuildTaskOnce({
	391	findCandidates: async () => [makeAiBuildCandidate()],
	392	hasDispatchMarker: async () => false,
	393	hasOpenLinkedPr: async () => false,
	394	dispatcher: async () => ({ ok: true, prNumber: 1 }),
	395	postMarkerComment: async (_id, _author, body) => {
	396	receivedBody = body;
	397	},
	398	});
	399	expect(receivedBody.includes(AI_BUILD_MARKER)).toBe(true);
	400	});
	401	});
	402
	403	// ---------------------------------------------------------------------------
	404	// AUTOPILOT_DISABLED=1 — both tasks no-op when the parent loop never runs.
	405	// ---------------------------------------------------------------------------
	406
	407	describe("autopilot disabled — neither AI task fires", () => {
	408	const originalDisabled = process.env.AUTOPILOT_DISABLED;
	409	afterEach(() => {
	410	if (originalDisabled === undefined) delete process.env.AUTOPILOT_DISABLED;
	411	else process.env.AUTOPILOT_DISABLED = originalDisabled;
	412	});
	413
	414	it("startAutopilot with AUTOPILOT_DISABLED=1 doesn't run injected AI tasks", async () => {
	415	process.env.AUTOPILOT_DISABLED = "1";
	416	let autoMergeRan = 0;
	417	let aiBuildRan = 0;
	418	const tasks: AutopilotTask[] = [
	419	{
	420	name: "auto-merge-sweep",
	421	run: async () => {
	422	autoMergeRan += 1;
	423	},
	424	},
	425	{
	426	name: "ai-build-from-issues",
	427	run: async () => {
	428	aiBuildRan += 1;
	429	},
	430	},
	431	];
	432	const { stop } = startAutopilot({ intervalMs: 5, tasks });
	433	await new Promise((r) => setTimeout(r, 40));
	434	stop();
	435	expect(autoMergeRan).toBe(0);
	436	expect(aiBuildRan).toBe(0);
	437	});
	438	});