Blame · Line-by-line history

`spec-ai.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

spec-ai.test.tsBlame254 lines · 1 contributor

Normal view Raw

23d1a81Claude	1	/**
	2	* Tests for spec-to-PR v2, part 2 — the Claude call + response parser.
	3	*
	4	* The Anthropic SDK captures `globalThis.fetch` at client construction, so
	5	* every test that installs a stub must first call `_resetClientForTests()`.
	6	*/
	7
	8	import { afterEach, beforeEach, describe, expect, it } from "bun:test";
	9	import {
	10	_resetClientForTests,
	11	generateSpecEdits,
	12	isForbiddenPath,
	13	parseAiJsonResponse,
	14	validateEdit,
	15	} from "../lib/spec-ai";
	16
	17	const origFetch = globalThis.fetch;
	18	const origKey = process.env.ANTHROPIC_API_KEY;
	19
	20	/**
	21	* Install a fake fetch that returns a single Anthropic-shaped messages.create
	22	* response with the provided text body.
	23	*/
	24	function installAnthropicFetch(textBody: string \| (() => string)): void {
	25	// @ts-expect-error — override global fetch for the duration of the test
	26	globalThis.fetch = async (
	27	_input: RequestInfo \| URL,
	28	_init: RequestInit = {}
	29	): Promise<Response> => {
	30	const text = typeof textBody === "function" ? textBody() : textBody;
	31	const payload = {
	32	id: "msg_test",
	33	type: "message",
	34	role: "assistant",
	35	model: "claude-sonnet-4-6",
	36	content: [{ type: "text", text }],
	37	stop_reason: "end_turn",
	38	stop_sequence: null,
	39	usage: { input_tokens: 1, output_tokens: 1 },
	40	};
	41	return new Response(JSON.stringify(payload), {
	42	status: 200,
	43	headers: { "content-type": "application/json" },
	44	});
	45	};
	46	}
	47
	48	function restoreEnv(): void {
	49	globalThis.fetch = origFetch;
	50	if (origKey === undefined) {
	51	delete process.env.ANTHROPIC_API_KEY;
	52	} else {
	53	process.env.ANTHROPIC_API_KEY = origKey;
	54	}
	55	_resetClientForTests();
	56	}
	57
	58	// ---------------------------------------------------------------------------
	59	// Pure helpers — quick sanity checks alongside the main 4 tests.
	60	// ---------------------------------------------------------------------------
	61
	62	describe("lib/spec-ai — pure helpers", () => {
	63	it("isForbiddenPath flags locked files", () => {
	64	expect(isForbiddenPath("BUILD_BIBLE.md")).toBe(true);
	65	expect(isForbiddenPath("src/views/layout.tsx")).toBe(true);
	66	expect(isForbiddenPath("drizzle/0001_init.sql")).toBe(true);
	67	expect(isForbiddenPath("legal/terms.md")).toBe(true);
	68	expect(isForbiddenPath("LICENSE")).toBe(true);
	69	expect(isForbiddenPath(".github/workflows/ci.yml")).toBe(true);
	70	});
	71
	72	it("isForbiddenPath lets ordinary source paths through", () => {
	73	expect(isForbiddenPath("src/lib/foo.ts")).toBe(false);
	74	expect(isForbiddenPath("src/routes/api.ts")).toBe(false);
	75	});
	76
	77	it("validateEdit rejects unsafe paths", () => {
	78	expect(validateEdit({ action: "edit", path: "/etc/passwd", content: "" })).toBe(false);
	79	expect(validateEdit({ action: "edit", path: "../foo", content: "" })).toBe(false);
	80	expect(validateEdit({ action: "edit", path: "", content: "" })).toBe(false);
	81	});
	82
	83	it("validateEdit accepts a well-formed edit", () => {
	84	expect(
	85	validateEdit({ action: "edit", path: "src/lib/foo.ts", content: "x" })
	86	).toBe(true);
	87	expect(
	88	validateEdit({ action: "delete", path: "src/old.ts" })
	89	).toBe(true);
	90	});
	91
	92	it("parseAiJsonResponse strips ```json fences", () => {
	93	const parsed = parseAiJsonResponse('```json\n{"a":1}\n```');
	94	expect(parsed).toEqual({ a: 1 });
	95	});
	96
	97	it("parseAiJsonResponse parses raw JSON", () => {
	98	const parsed = parseAiJsonResponse('{"b":2}');
	99	expect(parsed).toEqual({ b: 2 });
	100	});
	101
	102	it("parseAiJsonResponse returns null on garbage", () => {
	103	expect(parseAiJsonResponse("not json at all")).toBeNull();
	104	});
	105	});
	106
	107	// ---------------------------------------------------------------------------
	108	// The 4 required tests.
	109	// ---------------------------------------------------------------------------
	110
	111	describe("lib/spec-ai — generateSpecEdits", () => {
	112	beforeEach(() => {
	113	_resetClientForTests();
	114	});
	115
	116	afterEach(() => {
	117	restoreEnv();
	118	});
	119
	120	it("returns ok:false when ANTHROPIC_API_KEY missing", async () => {
	121	delete process.env.ANTHROPIC_API_KEY;
	122	const result = await generateSpecEdits({
	123	spec: "add a greeting function",
	124	fileList: ["src/index.ts"],
	125	relevantFiles: [{ path: "src/index.ts", content: "// hi" }],
	126	defaultBranch: "main",
	127	});
	128	expect(result.ok).toBe(false);
	129	if (result.ok === false) {
	130	expect(result.error).toContain("ANTHROPIC_API_KEY");
	131	}
	132	});
	133
	134	it("parses a well-formed response", async () => {
	135	process.env.ANTHROPIC_API_KEY = "test-key";
	136	installAnthropicFetch(
	137	JSON.stringify({
	138	summary: "add greet()",
	139	edits: [
	140	{
	141	action: "create",
	142	path: "src/lib/greet.ts",
	143	content: "export const greet = () => 'hi';",
	144	},
	145	{
	146	action: "edit",
	147	path: "src/index.ts",
	148	content: "import { greet } from './lib/greet';\ngreet();",
	149	},
	150	{ action: "delete", path: "src/old.ts" },
	151	],
	152	})
	153	);
	154
	155	const result = await generateSpecEdits({
	156	spec: "add a greeting",
	157	fileList: ["src/index.ts"],
	158	relevantFiles: [{ path: "src/index.ts", content: "// hi" }],
	159	defaultBranch: "main",
	160	});
	161
	162	expect(result.ok).toBe(true);
	163	if (result.ok === true) {
	164	expect(result.summary).toBe("add greet()");
	165	expect(result.edits).toHaveLength(3);
	166	expect(result.edits[0]).toEqual({
	167	action: "create",
	168	path: "src/lib/greet.ts",
	169	content: "export const greet = () => 'hi';",
	170	});
	171	expect(result.edits[2]).toEqual({ action: "delete", path: "src/old.ts" });
	172	}
	173	});
	174
	175	// We chose "drop the forbidden edit, keep the ok:true envelope" — the
	176	// caller can compare input vs output length if they want to detect this.
	177	// If Claude proposes ONLY forbidden edits, the caller receives
	178	// `{ok:true, edits:[], summary:"AI proposed no changes"}`.
	179	it("rejects edits targeting forbidden paths (silently dropped)", async () => {
	180	process.env.ANTHROPIC_API_KEY = "test-key";
	181	installAnthropicFetch(
	182	JSON.stringify({
	183	summary: "mixed forbidden + allowed",
	184	edits: [
	185	{
	186	action: "edit",
	187	path: "BUILD_BIBLE.md",
	188	content: "should be dropped",
	189	},
	190	{
	191	action: "edit",
	192	path: "src/views/layout.tsx",
	193	content: "should also be dropped",
	194	},
	195	{
	196	action: "edit",
	197	path: "drizzle/0001_init.sql",
	198	content: "dropped",
	199	},
	200	{
	201	action: "edit",
	202	path: "LICENSE",
	203	content: "dropped",
	204	},
	205	{
	206	action: "edit",
	207	path: ".github/workflows/ci.yml",
	208	content: "dropped",
	209	},
	210	{
	211	action: "create",
	212	path: "src/lib/ok.ts",
	213	content: "export const ok = 1;",
	214	},
	215	],
	216	})
	217	);
	218
	219	const result = await generateSpecEdits({
	220	spec: "touch forbidden files",
	221	fileList: ["BUILD_BIBLE.md", "LICENSE"],
	222	relevantFiles: [],
	223	defaultBranch: "main",
	224	});
	225
	226	expect(result.ok).toBe(true);
	227	if (result.ok === true) {
	228	// Exactly one allowed edit survives.
	229	expect(result.edits).toHaveLength(1);
	230	expect(result.edits[0].path).toBe("src/lib/ok.ts");
	231	// No edit points at a forbidden path.
	232	for (const e of result.edits) {
	233	expect(isForbiddenPath(e.path)).toBe(false);
	234	}
	235	}
	236	});
	237
	238	it("handles malformed JSON response", async () => {
	239	process.env.ANTHROPIC_API_KEY = "test-key";
	240	installAnthropicFetch("this is not JSON, sorry");
	241
	242	const result = await generateSpecEdits({
	243	spec: "whatever",
	244	fileList: [],
	245	relevantFiles: [],
	246	defaultBranch: "main",
	247	});
	248
	249	expect(result.ok).toBe(false);
	250	if (result.ok === false) {
	251	expect(result.error).toBe("AI returned invalid JSON");
	252	}
	253	});
	254	});