Blame · Line-by-line history

`ai-release-notes.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-release-notes.test.tsBlame519 lines · 1 contributor

Normal view Raw

424eb72Claude	1	/**
	2	* Tests for src/lib/ai-release-notes.ts.
	3	*
	4	* Two layers:
	5	*
	6	* 1. Pure helpers — bucketing, prompt assembly, markdown rendering.
	7	* No DB / no Claude. Always run.
	8	*
	9	* 2. End-to-end through `generateReleaseNotes` with a fake Claude
	10	* client + a real bare repo on disk. DB-touching assertions are
	11	* gated on DATABASE_URL via the HAS_DB skipIf pattern used across
	12	* the rest of the suite.
	13	*/
	14
	15	import { describe, it, expect, beforeAll, afterAll } from "bun:test";
	16	import { join } from "path";
	17	import { rm, mkdir } from "fs/promises";
	18	import {
	19	bucketPrs,
	20	buildReleaseNotesPrompt,
	21	classifyPr,
	22	generateReleaseNotes,
	23	isSemverTag,
	24	mergeClaudeSections,
	25	prNumberFromCommitMessage,
	26	renderPrBullet,
	27	renderSectionsToMarkdown,
	28	resolvePrsForCommits,
	29	type ResolvedPullRequest,
	30	type ReleaseSections,
	31	} from "../lib/ai-release-notes";
	32	import {
	33	initBareRepo,
	34	createOrUpdateFileOnBranch,
	35	createTag,
	36	resolveRef,
	37	} from "../git/repository";
	38	import { db } from "../db";
	39	import { eq } from "drizzle-orm";
	40	import {
	41	pullRequests,
	42	repositories,
	43	users,
	44	} from "../db/schema";
	45
	46	const HAS_DB = Boolean(process.env.DATABASE_URL);
	47
	48	const TEST_REPOS = join(
	49	import.meta.dir,
	50	"../../.test-repos-ai-release-notes-" + Date.now()
	51	);
	52
	53	beforeAll(async () => {
	54	process.env.GIT_REPOS_PATH = TEST_REPOS;
	55	await rm(TEST_REPOS, { recursive: true, force: true });
	56	await mkdir(TEST_REPOS, { recursive: true });
	57	});
	58
	59	afterAll(async () => {
	60	await rm(TEST_REPOS, { recursive: true, force: true });
	61	});
	62
	63	// ---------------------------------------------------------------------------
	64	// Pure helpers
	65	// ---------------------------------------------------------------------------
	66
	67	function pr(overrides: Partial<ResolvedPullRequest>): ResolvedPullRequest {
	68	return {
	69	number: 1,
	70	title: "feat: thing",
	71	body: null,
	72	authorUsername: "alice",
	73	headBranch: "feature/x",
	74	mergedAt: new Date("2026-01-01T00:00:00Z"),
	75	labels: [],
	76	autoMergedByAi: false,
	77	...overrides,
	78	};
	79	}
	80
	81	describe("classifyPr", () => {
	82	it("routes auto-merged PRs to ai_changes", () => {
	83	expect(classifyPr(pr({ autoMergedByAi: true }))).toBe("ai_changes");
	84	});
	85
	86	it("routes by label first", () => {
	87	expect(classifyPr(pr({ title: "anything", labels: ["security"] }))).toBe(
	88	"security"
	89	);
	90	expect(classifyPr(pr({ title: "anything", labels: ["bug"] }))).toBe(
	91	"fixes"
	92	);
	93	expect(classifyPr(pr({ title: "anything", labels: ["ai:feature"] }))).toBe(
	94	"features"
	95	);
	96	expect(classifyPr(pr({ title: "anything", labels: ["ai:auto-merge"] }))).toBe(
	97	"ai_changes"
	98	);
	99	});
	100
	101	it("falls back to conventional-commit prefix on the title", () => {
	102	expect(classifyPr(pr({ title: "feat: new x" }))).toBe("features");
	103	expect(classifyPr(pr({ title: "feat(api): scoped" }))).toBe("features");
	104	expect(classifyPr(pr({ title: "fix: bug" }))).toBe("fixes");
	105	expect(classifyPr(pr({ title: "perf: faster" }))).toBe("perf");
	106	expect(classifyPr(pr({ title: "docs: readme" }))).toBe("docs");
	107	expect(classifyPr(pr({ title: "security: sanitise" }))).toBe("security");
	108	});
	109
	110	it("dumps anything unclassified into 'other'", () => {
	111	expect(classifyPr(pr({ title: "tidy up" }))).toBe("other");
	112	expect(classifyPr(pr({ title: "refactor: split file" }))).toBe("other");
	113	});
	114	});
	115
	116	describe("renderPrBullet", () => {
	117	it("strips the conventional-commit prefix and adds attribution", () => {
	118	const out = renderPrBullet(pr({ number: 42, title: "feat(api): scoped thing", authorUsername: "bob" }));
	119	expect(out).toBe("scoped thing (#42) — @bob");
	120	});
	121	});
	122
	123	describe("bucketPrs", () => {
	124	it("groups PRs into the right sections", () => {
	125	const sections = bucketPrs([
	126	pr({ number: 1, title: "feat: A" }),
	127	pr({ number: 2, title: "fix: B" }),
	128	pr({ number: 3, title: "perf: C" }),
	129	pr({ number: 4, title: "anything", autoMergedByAi: true }),
	130	pr({ number: 5, title: "refactor: D" }),
	131	]);
	132	expect(sections.features.bullets.length).toBe(1);
	133	expect(sections.fixes.bullets.length).toBe(1);
	134	expect(sections.perf.bullets.length).toBe(1);
	135	expect(sections.ai_changes.bullets.length).toBe(1);
	136	expect(sections.other.bullets.length).toBe(1);
	137	expect(sections.security.bullets.length).toBe(0);
	138	expect(sections.docs.bullets.length).toBe(0);
	139	});
	140	});
	141
	142	describe("prNumberFromCommitMessage", () => {
	143	it("recognises GitHub-style merge commits", () => {
	144	expect(prNumberFromCommitMessage("Merge pull request #42 from foo/bar")).toBe(42);
	145	});
	146	it("recognises squash subjects", () => {
	147	expect(prNumberFromCommitMessage("feat: stuff (#7)")).toBe(7);
	148	});
	149	it("returns null on missing", () => {
	150	expect(prNumberFromCommitMessage("plain commit")).toBeNull();
	151	});
	152	});
	153
	154	describe("buildReleaseNotesPrompt", () => {
	155	it("embeds repo, range, grouped sections, and raw PRs", () => {
	156	const sections: ReleaseSections = {
	157	features: { bullets: ["A (#1) — @alice"] },
	158	fixes: { bullets: [] },
	159	perf: { bullets: [] },
	160	docs: { bullets: [] },
	161	security: { bullets: [] },
	162	ai_changes: { bullets: [] },
	163	other: { bullets: [] },
	164	};
	165	const prompt = buildReleaseNotesPrompt({
	166	repoFullName: "alice/demo",
	167	fromTag: "v1.0.0",
	168	toTag: "v1.1.0",
	169	sections,
	170	prs: [pr({ number: 1, title: "feat: A" })],
	171	});
	172	expect(prompt).toContain("alice/demo");
	173	expect(prompt).toContain("v1.0.0");
	174	expect(prompt).toContain("v1.1.0");
	175	expect(prompt).toContain("features:");
	176	expect(prompt).toContain("A (#1) — @alice");
	177	expect(prompt).toContain('"sections"');
	178	expect(prompt).toContain('"headline"');
	179	});
	180
	181	it("notes the initial release case when fromTag is null", () => {
	182	const prompt = buildReleaseNotesPrompt({
	183	repoFullName: "alice/demo",
	184	fromTag: null,
	185	toTag: "v0.1.0",
	186	sections: {
	187	features: { bullets: [] },
	188	fixes: { bullets: [] },
	189	perf: { bullets: [] },
	190	docs: { bullets: [] },
	191	security: { bullets: [] },
	192	ai_changes: { bullets: [] },
	193	other: { bullets: [] },
	194	},
	195	prs: [],
	196	});
	197	expect(prompt).toContain("(initial)");
	198	});
	199	});
	200
	201	describe("mergeClaudeSections", () => {
	202	const deterministic: ReleaseSections = {
	203	features: { bullets: ["thing (#1) — @alice"] },
	204	fixes: { bullets: ["bug (#2) — @bob"] },
	205	perf: { bullets: [] },
	206	docs: { bullets: [] },
	207	security: { bullets: [] },
	208	ai_changes: { bullets: [] },
	209	other: { bullets: [] },
	210	};
	211
	212	it("returns deterministic when Claude is null", () => {
	213	expect(mergeClaudeSections(deterministic, null)).toEqual(deterministic);
	214	});
	215
	216	it("prefers Claude wording when it covers all bullets", () => {
	217	const merged = mergeClaudeSections(deterministic, {
	218	sections: {
	219	features: { bullets: ["Add the thing (#1) — @alice"] },
	220	},
	221	});
	222	expect(merged.features.bullets).toEqual(["Add the thing (#1) — @alice"]);
	223	// Untouched bucket stays deterministic.
	224	expect(merged.fixes.bullets).toEqual(["bug (#2) — @bob"]);
	225	});
	226
	227	it("keeps deterministic when Claude drops bullets", () => {
	228	const merged = mergeClaudeSections(deterministic, {
	229	sections: {
	230	// Claude returned fewer items than we know about — distrust.
	231	fixes: { bullets: [] },
	232	},
	233	});
	234	expect(merged.fixes.bullets).toEqual(["bug (#2) — @bob"]);
	235	});
	236	});
	237
	238	describe("renderSectionsToMarkdown", () => {
	239	it("emits a header, headline, summary, and ordered sections", () => {
	240	const md = renderSectionsToMarkdown({
	241	repoFullName: "alice/demo",
	242	fromTag: "v1.0.0",
	243	toTag: "v1.1.0",
	244	headline: "Stability + speed.",
	245	summary: "This release tightens the merge queue.",
	246	sections: {
	247	features: { bullets: ["A (#1) — @alice"] },
	248	fixes: { bullets: ["B (#2) — @bob"] },
	249	perf: { bullets: [] },
	250	docs: { bullets: [] },
	251	security: { bullets: ["sanitise (#3) — @carol"] },
	252	ai_changes: { bullets: [] },
	253	other: { bullets: [] },
	254	},
	255	});
	256	expect(md).toContain("## v1.1.0 (since v1.0.0)");
	257	expect(md).toContain("Stability + speed.");
	258	expect(md).toContain("This release tightens the merge queue.");
	259	// Features comes before fixes, which comes before security.
	260	const feat = md.indexOf("### Features");
	261	const fixIdx = md.indexOf("### Bug fixes");
	262	const sec = md.indexOf("### Security");
	263	expect(feat).toBeGreaterThan(0);
	264	expect(fixIdx).toBeGreaterThan(feat);
	265	expect(sec).toBeGreaterThan(fixIdx);
	266	expect(md).toContain("- A (#1) — @alice");
	267	expect(md).toContain("- B (#2) — @bob");
	268	// Empty sections are skipped.
	269	expect(md).not.toContain("### Performance");
	270	expect(md).not.toContain("### Documentation");
	271	});
	272
	273	it("skips the headline + summary lines when both are empty", () => {
	274	const md = renderSectionsToMarkdown({
	275	repoFullName: "alice/demo",
	276	fromTag: null,
	277	toTag: "v0.1.0",
	278	headline: "",
	279	summary: "",
	280	sections: {
	281	features: { bullets: [] },
	282	fixes: { bullets: [] },
	283	perf: { bullets: [] },
	284	docs: { bullets: [] },
	285	security: { bullets: [] },
	286	ai_changes: { bullets: [] },
	287	other: { bullets: [] },
	288	},
	289	});
	290	expect(md).toContain("## v0.1.0");
	291	expect(md).not.toContain("**");
	292	expect(md).toContain("(initial)...v0.1.0");
	293	});
	294	});
	295
	296	describe("isSemverTag", () => {
	297	it("accepts vX.Y.Z and X.Y.Z", () => {
	298	expect(isSemverTag("v1.2.3")).toBe(true);
	299	expect(isSemverTag("1.2.3")).toBe(true);
	300	expect(isSemverTag("v0.1.0-beta.1")).toBe(true);
	301	});
	302	it("rejects garbage", () => {
	303	expect(isSemverTag("release-2024-01")).toBe(false);
	304	expect(isSemverTag("v1")).toBe(false);
	305	expect(isSemverTag("nightly")).toBe(false);
	306	});
	307	});
	308
	309	// ---------------------------------------------------------------------------
	310	// End-to-end with fake Claude + real bare repo. DB-backed.
	311	// ---------------------------------------------------------------------------
	312
	313	/**
	314	* Fake Anthropic client returning a canned JSON envelope. Matches the
	315	* shape ai-release-notes's parser expects.
	316	*/
	317	function fakeClient(responseText: string) {
	318	return {
	319	messages: {
	320	create: async () => ({
	321	content: [{ type: "text" as const, text: responseText }],
	322	}),
	323	},
	324	} as any;
	325	}
	326
	327	describe.skipIf(!HAS_DB)("generateReleaseNotes — end-to-end with fake Claude", () => {
	328	const OWNER = `relnotes_${Date.now().toString(36)}`;
	329	const REPO = "subject";
	330	let repositoryId = "";
	331	let userId = "";
	332	let toSha = "";
	333
	334	beforeAll(async () => {
	335	// Seed user + repo + bare git repo on disk with two commits and two tags.
	336	const [u] = await db
	337	.insert(users)
	338	.values({
	339	username: OWNER,
	340	email: `${OWNER}@example.com`,
	341	passwordHash: "x",
	342	})
	343	.returning({ id: users.id });
	344	userId = u.id;
	345
	346	const [r] = await db
	347	.insert(repositories)
	348	.values({
	349	ownerId: userId,
	350	name: REPO,
	351	diskPath: `/tmp/${OWNER}/${REPO}`,
	352	defaultBranch: "main",
	353	})
	354	.returning({ id: repositories.id });
	355	repositoryId = r.id;
	356
	357	await initBareRepo(OWNER, REPO);
	358
	359	const seed = await createOrUpdateFileOnBranch({
	360	owner: OWNER,
	361	name: REPO,
	362	branch: "main",
	363	filePath: "README.md",
	364	bytes: new TextEncoder().encode("# v1\n"),
	365	message: "initial commit",
	366	authorName: "Seeder",
	367	authorEmail: "seed@example.com",
	368	});
	369	if ("error" in seed) throw new Error(`seed failed: ${seed.error}`);
	370	await createTag(OWNER, REPO, "v1.0.0", seed.commitSha, "v1.0.0");
	371
	372	// Commit referencing PR #1 — landed as a squash subject.
	373	const c1 = await createOrUpdateFileOnBranch({
	374	owner: OWNER,
	375	name: REPO,
	376	branch: "main",
	377	filePath: "src/feature.ts",
	378	bytes: new TextEncoder().encode("export const f = 1;\n"),
	379	message: "feat: add feature (#1)",
	380	authorName: "Alice",
	381	authorEmail: "alice@example.com",
	382	});
	383	if ("error" in c1) throw new Error(`c1 failed: ${c1.error}`);
	384
	385	// Commit referencing PR #2 — a fix.
	386	const c2 = await createOrUpdateFileOnBranch({
	387	owner: OWNER,
	388	name: REPO,
	389	branch: "main",
	390	filePath: "src/fix.ts",
	391	bytes: new TextEncoder().encode("export const fixed = true;\n"),
	392	message: "fix: handle null case (#2)",
	393	authorName: "Bob",
	394	authorEmail: "bob@example.com",
	395	});
	396	if ("error" in c2) throw new Error(`c2 failed: ${c2.error}`);
	397
	398	await createTag(OWNER, REPO, "v1.1.0", c2.commitSha, "v1.1.0");
	399	toSha = c2.commitSha;
	400
	401	// Insert two merged PR rows so cross-referencing succeeds.
	402	await db.insert(pullRequests).values({
	403	repositoryId,
	404	authorId: userId,
	405	title: "feat: add feature",
	406	body: "Adds the feature.",
	407	state: "merged",
	408	baseBranch: "main",
	409	headBranch: "feature/x",
	410	mergedAt: new Date(),
	411	number: 1,
	412	});
	413	await db.insert(pullRequests).values({
	414	repositoryId,
	415	authorId: userId,
	416	title: "fix: handle null case",
	417	body: "Crash fix.",
	418	state: "merged",
	419	baseBranch: "main",
	420	headBranch: "fix/y",
	421	mergedAt: new Date(),
	422	number: 2,
	423	});
	424	});
	425
	426	afterAll(async () => {
	427	// Cleanup — cascades handle PRs / releases.
	428	if (repositoryId) {
	429	await db.delete(repositories).where(eq(repositories.id, repositoryId));
	430	}
	431	if (userId) {
	432	await db.delete(users).where(eq(users.id, userId));
	433	}
	434	});
	435
	436	it("cross-references PRs from the commit range", async () => {
	437	const fromSha = await resolveRef(OWNER, REPO, "v1.0.0");
	438	expect(fromSha).toBeTruthy();
	439	expect(toSha).toBeTruthy();
	440	const { commitsBetween } = await import("../git/repository");
	441	const commits = await commitsBetween(OWNER, REPO, fromSha, toSha);
	442	expect(commits.length).toBe(2);
	443
	444	const prs = await resolvePrsForCommits(repositoryId, commits);
	445	expect(prs.length).toBe(2);
	446	const nums = prs.map((p) => p.number).sort();
	447	expect(nums).toEqual([1, 2]);
	448	});
	449
	450	it("renders structured output into markdown with section grouping", async () => {
	451	const claudeOut = JSON.stringify({
	452	headline: "Feature + fix.",
	453	summary: "Adds the new feature and patches the null crash.",
	454	sections: {
	455	features: { bullets: ["Add feature (#1) — @" + OWNER] },
	456	fixes: { bullets: ["Handle null (#2) — @" + OWNER] },
	457	},
	458	});
	459
	460	const result = await generateReleaseNotes({
	461	repositoryId,
	462	fromTag: "v1.0.0",
	463	toTag: "v1.1.0",
	464	client: fakeClient(claudeOut),
	465	});
	466
	467	expect(result.aiUsed).toBe(true);
	468	expect(result.prCount).toBe(2);
	469	expect(result.headline).toBe("Feature + fix.");
	470	expect(result.summary).toContain("Adds the new feature");
	471	expect(result.sections.features.bullets.length).toBe(1);
	472	expect(result.sections.fixes.bullets.length).toBe(1);
	473	expect(result.markdown).toContain("## v1.1.0 (since v1.0.0)");
	474	expect(result.markdown).toContain("Feature + fix.");
	475	expect(result.markdown).toContain("### Features");
	476	expect(result.markdown).toContain("### Bug fixes");
	477	expect(result.markdown).toContain("Add feature (#1)");
	478	});
	479
	480	it("falls back to deterministic markdown when Claude returns garbage", async () => {
	481	const result = await generateReleaseNotes({
	482	repositoryId,
	483	fromTag: "v1.0.0",
	484	toTag: "v1.1.0",
	485	client: fakeClient("not json at all"),
	486	});
	487	expect(result.aiUsed).toBe(false);
	488	// Even without Claude, sections must be populated from the PR cross-ref.
	489	expect(result.sections.features.bullets.length).toBeGreaterThan(0);
	490	expect(result.sections.fixes.bullets.length).toBeGreaterThan(0);
	491	expect(result.markdown).toContain("### Features");
	492	expect(result.markdown).toContain("### Bug fixes");
	493	});
	494
	495	it("returns a non-empty markdown even when no PRs match", async () => {
	496	// Generate notes for the initial-tag → initial-tag range (no commits).
	497	const result = await generateReleaseNotes({
	498	repositoryId,
	499	fromTag: "v1.0.0",
	500	toTag: "v1.0.0",
	501	client: fakeClient("{}"),
	502	});
	503	expect(result.markdown).toContain("v1.0.0");
	504	expect(result.prCount).toBe(0);
	505	});
	506	});
	507
	508	describe("generateReleaseNotes — repo not found", () => {
	509	it("returns a graceful fallback for an unknown repository id", async () => {
	510	const result = await generateReleaseNotes({
	511	repositoryId: "00000000-0000-0000-0000-000000000000",
	512	fromTag: null,
	513	toTag: "v1.0.0",
	514	});
	515	expect(result.prCount).toBe(0);
	516	expect(result.aiUsed).toBe(false);
	517	expect(result.markdown).toContain("v1.0.0");
	518	});
	519	});