Blame · Line-by-line history

`test-gaps.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

test-gaps.tsBlame379 lines · 1 contributor

Normal view Raw

53299fbClaude	1	/**
	2	* Test Gap Detector — identify source files with no test coverage, ranked by risk.
	3	*
	4	* Steps:
	5	* 1. git ls-tree to enumerate source + test files
	6	* 2. Match source files against known test files to find uncovered candidates
	7	* 3. Read high-risk candidate content via getBlob
	8	* 4. Single Claude call to score each candidate's functions by risk
	9	* 5. git grep callsite counts in parallel
	10	*
	11	* Cached per-repo with a 2h TTL (in-memory Map).
	12	*/
	13
	14	import { getRepoPath, getBlob } from "../git/repository";
	15	import {
	16	getAnthropic,
	17	isAiAvailable,
	18	MODEL_SONNET,
	19	extractText,
	20	parseJsonResponse,
	21	} from "./ai-client";
	22
	23	// ─── Types ────────────────────────────────────────────────────────────────────
	24
	25	export interface TestGap {
	26	filePath: string;
	27	functionName: string;
	28	riskScore: number;
	29	riskReason: string;
	30	suggestedTestPath: string;
	31	calledByCount: number;
	32	}
	33
	34	export interface TestGapReport {
	35	repoId: string;
	36	totalSourceFiles: number;
	37	totalTestFiles: number;
	38	coverageEstimate: number;
	39	gaps: TestGap[];
	40	analyzedAt: Date;
	41	}
	42
	43	// ─── In-memory cache ──────────────────────────────────────────────────────────
	44
	45	interface CacheEntry {
	46	report: TestGapReport;
	47	expiresAt: number;
	48	}
	49
	50	const cache = new Map<string, CacheEntry>();
	51
	52	const CACHE_TTL_MS = 2 * 60 * 60 * 1000; // 2 hours
	53
	54	// ─── Helpers ──────────────────────────────────────────────────────────────────
	55
	56	async function execInRepo(
	57	args: string[],
	58	repoDir: string
	59	): Promise<{ stdout: string; exitCode: number }> {
	60	const proc = Bun.spawn(args, {
	61	cwd: repoDir,
	62	stdout: "pipe",
	63	stderr: "pipe",
	64	});
	65	const stdout = await new Response(proc.stdout).text();
	66	const exitCode = await proc.exited;
	67	return { stdout, exitCode };
	68	}
	69
	70	const SOURCE_EXTS = /\.(ts\|tsx\|js\|jsx\|py\|go\|rs)$/;
	71	const TEST_PATTERN = /(\.(test\|spec)\.(ts\|tsx\|js\|jsx)$\|__tests__\/)/;
	72	const SKIP_DIRS = /(node_modules\|dist\|build\|\.next\|vendor\|__pycache__)/;
	73
	74	function isSourceFile(p: string): boolean {
	75	return SOURCE_EXTS.test(p) && !TEST_PATTERN.test(p) && !SKIP_DIRS.test(p);
	76	}
	77
	78	function isTestFile(p: string): boolean {
	79	return TEST_PATTERN.test(p) && !SKIP_DIRS.test(p);
	80	}
	81
	82	/**
	83	* Given a source file path, return the set of test-path patterns we'd expect
	84	* to find for it (basename.test.ts, basename.spec.ts, __tests__/basename.test.ts).
	85	*/
	86	function expectedTestPaths(filePath: string): string[] {
	87	const withoutExt = filePath.replace(/\.[^.]+$/, "");
	88	const basename = withoutExt.split("/").pop() ?? withoutExt;
	89	return [
	90	`${withoutExt}.test.ts`,
	91	`${withoutExt}.test.tsx`,
	92	`${withoutExt}.spec.ts`,
	93	`${withoutExt}.spec.tsx`,
	94	`${withoutExt}.test.js`,
	95	`${withoutExt}.spec.js`,
	96	`__tests__/${basename}.test.ts`,
	97	`__tests__/${basename}.test.tsx`,
	98	`__tests__/${basename}.spec.ts`,
	99	`${basename}.test.ts`,
	100	`${basename}.spec.ts`,
	101	];
	102	}
	103
	104	/**
	105	* Check if a source file has any associated test file in the known test set.
	106	*/
	107	function hasTestCoverage(filePath: string, testFileSet: Set<string>): boolean {
	108	// 1. Direct path match
	109	for (const pattern of expectedTestPaths(filePath)) {
	110	if (testFileSet.has(pattern)) return true;
	111	}
	112	// 2. Basename appears in any test file path
	113	const basename = filePath.split("/").pop()?.replace(/\.[^.]+$/, "") ?? "";
	114	if (basename.length < 3) return false; // too generic, skip
	115	for (const testFile of testFileSet) {
	116	if (testFile.toLowerCase().includes(basename.toLowerCase())) return true;
	117	}
	118	return false;
	119	}
	120
	121	const EXPORT_REGEX =
	122	/(export\s+(?:async\s+)?function\s+(\w+)\|export\s+const\s+(\w+)\s*=)/gm;
	123
	124	function extractExportedNames(content: string): string[] {
	125	const names: string[] = [];
	126	let match: RegExpExecArray \| null;
	127	while ((match = EXPORT_REGEX.exec(content)) !== null) {
	128	const name = match[2] ?? match[3];
	129	if (name) names.push(name);
	130	}
	131	return [...new Set(names)];
	132	}
	133
	134	function suggestTestPath(filePath: string): string {
	135	const dir = filePath.includes("/")
	136	? filePath.substring(0, filePath.lastIndexOf("/"))
	137	: "src";
	138	const basename = (filePath.split("/").pop() ?? filePath).replace(
	139	/\.[^.]+$/,
	140	""
	141	);
	142	if (filePath.startsWith("src/")) {
	143	return `src/__tests__/${basename}.test.ts`;
	144	}
	145	return `${dir}/__tests__/${basename}.test.ts`;
	146	}
	147
	148	// ─── Core detector ───────────────────────────────────────────────────────────
	149
	150	export async function detectTestGaps(
	151	ownerName: string,
	152	repoName: string,
	153	repoId: string
	154	): Promise<TestGapReport> {
	155	const repoDir = getRepoPath(ownerName, repoName);
	156
	157	// ── Step 1: get file tree ──────────────────────────────────────────────────
	158	const { stdout: lsOutput, exitCode } = await execInRepo(
	159	["git", "ls-tree", "-r", "--name-only", "HEAD"],
	160	repoDir
	161	);
	162
	163	if (exitCode !== 0) {
	164	return {
	165	repoId,
	166	totalSourceFiles: 0,
	167	totalTestFiles: 0,
	168	coverageEstimate: 0,
	169	gaps: [],
	170	analyzedAt: new Date(),
	171	};
	172	}
	173
	174	const allFiles = lsOutput.trim().split("\n").filter(Boolean);
	175	let sourceFiles = allFiles.filter(isSourceFile).slice(0, 200);
	176	const testFiles = allFiles.filter(isTestFile);
	177	const testFileSet = new Set(testFiles);
	178
	179	// ── Step 2: find uncovered source files ───────────────────────────────────
	180	const candidates = sourceFiles
	181	.filter((f) => !hasTestCoverage(f, testFileSet))
	182	.slice(0, 50);
	183
	184	const coveredCount = sourceFiles.length - candidates.length;
	185	const coverageEstimate =
	186	sourceFiles.length > 0
	187	? Math.round((coveredCount / sourceFiles.length) * 100)
	188	: 100;
	189
	190	if (candidates.length === 0) {
	191	return {
	192	repoId,
	193	totalSourceFiles: sourceFiles.length,
	194	totalTestFiles: testFiles.length,
	195	coverageEstimate,
	196	gaps: [],
	197	analyzedAt: new Date(),
	198	};
	199	}
	200
	201	// ── Step 3: read candidate content (cap 4KB each, 40KB total) ─────────────
	202	interface CandidateInfo {
	203	filePath: string;
	204	exports: string[];
	205	content: string;
	206	}
	207
	208	const candidateInfos: CandidateInfo[] = [];
	209	let totalBytes = 0;
	210
	211	for (const filePath of candidates) {
	212	if (totalBytes >= 40_000) break;
	213	try {
	214	const blob = await getBlob(ownerName, repoName, "HEAD", filePath);
	215	if (!blob \|\| blob.isBinary \|\| !blob.content) continue;
	216	const truncated = blob.content.slice(0, 4_096);
	217	const exports = extractExportedNames(truncated);
	218	if (exports.length === 0) {
	219	// Still include with a placeholder so we can score the file overall
	220	exports.push("<default>");
	221	}
	222	candidateInfos.push({ filePath, exports, content: truncated });
	223	totalBytes += truncated.length;
	224	} catch {
	225	// skip unreadable blobs
	226	}
	227	}
	228
	229	if (candidateInfos.length === 0) {
	230	return {
	231	repoId,
	232	totalSourceFiles: sourceFiles.length,
	233	totalTestFiles: testFiles.length,
	234	coverageEstimate,
	235	gaps: [],
	236	analyzedAt: new Date(),
	237	};
	238	}
	239
	240	// ── Step 4: risk scoring via Claude (or fallback) ─────────────────────────
	241	interface AiGap {
	242	filePath: string;
	243	functionName: string;
	244	riskScore: number;
	245	riskReason: string;
	246	suggestedTestPath: string;
	247	}
	248
	249	interface AiResponse {
	250	gaps: AiGap[];
	251	coverageEstimate?: number;
	252	}
	253
	254	let aiGaps: AiGap[] = [];
	255
	256	if (isAiAvailable()) {
	257	const fileList = candidateInfos
	258	.map((c) => `${c.filePath}: ${c.exports.join(", ")}`)
	259	.join("\n");
	260
	261	try {
	262	const anthropic = getAnthropic();
	263	const message = await anthropic.messages.create({
	264	model: MODEL_SONNET,
	265	max_tokens: 2048,
	266	system:
	267	"You are a senior engineer identifying test coverage risks. Rate each untested function by how critical it is to test (0-100 risk score). Higher scores for: auth/security logic, DB mutations, payment handling, public API endpoints, complex business logic. Lower for: pure utilities, string formatters, simple getters.",
	268	messages: [
	269	{
	270	role: "user",
	271	content: `Repository: ${repoName}
	272	Untested files and their exported functions:
	273	${fileList}
	274
	275	Return JSON:
	276	{"gaps": [{"filePath": "...", "functionName": "...", "riskScore": 0, "riskReason": "...", "suggestedTestPath": "..."}], "coverageEstimate": 0}
	277
	278	Rules:
	279	- One gap entry per function per file (or one per file if no named exports).
	280	- riskScore: integer 0-100.
	281	- suggestedTestPath: where a new test file should live (e.g. "src/__tests__/auth.test.ts").
	282	- Keep gaps list ≤ 30 items total, prioritising highest risk.`,
	283	},
	284	],
	285	});
	286
	287	const text = extractText(message);
	288	const parsed = parseJsonResponse<AiResponse>(text);
	289	if (parsed && Array.isArray(parsed.gaps)) {
	290	aiGaps = parsed.gaps;
	291	}
	292	} catch {
	293	// fall through to fallback below
	294	}
	295	}
	296
	297	// Fallback if AI unavailable or failed
	298	if (aiGaps.length === 0) {
	299	for (const c of candidateInfos) {
	300	for (const fn of c.exports.slice(0, 3)) {
	301	aiGaps.push({
	302	filePath: c.filePath,
	303	functionName: fn === "<default>" ? c.filePath.split("/").pop() ?? c.filePath : fn,
	304	riskScore: 50,
	305	riskReason: "No test file found for this module",
	306	suggestedTestPath: suggestTestPath(c.filePath),
	307	});
	308	}
	309	}
	310	}
	311
	312	// ── Step 5: callsite counts via git grep ──────────────────────────────────
	313	const gapsFull: TestGap[] = await Promise.all(
	314	aiGaps.slice(0, 30).map(async (g) => {
	315	let calledByCount = 0;
	316	const fnName = g.functionName.replace(/[^a-zA-Z0-9_]/g, "");
	317	if (fnName && fnName !== "default") {
	318	try {
	319	const { stdout: grepOut } = await execInRepo(
	320	["git", "grep", "-l", fnName],
	321	repoDir
	322	);
	323	const matchedFiles = grepOut
	324	.trim()
	325	.split("\n")
	326	.filter(Boolean);
	327	// Subtract 1 for the definition file itself
	328	calledByCount = Math.max(0, matchedFiles.length - 1);
	329	} catch {
	330	calledByCount = 0;
	331	}
	332	}
	333	return {
	334	filePath: g.filePath,
	335	functionName: g.functionName,
	336	riskScore: Math.min(100, Math.max(0, Math.round(g.riskScore))),
	337	riskReason: g.riskReason,
	338	suggestedTestPath: g.suggestedTestPath \|\| suggestTestPath(g.filePath),
	339	calledByCount,
	340	};
	341	})
	342	);
	343
	344	// Sort by riskScore desc
	345	gapsFull.sort((a, b) => b.riskScore - a.riskScore);
	346
	347	return {
	348	repoId,
	349	totalSourceFiles: sourceFiles.length,
	350	totalTestFiles: testFiles.length,
	351	coverageEstimate,
	352	gaps: gapsFull,
	353	analyzedAt: new Date(),
	354	};
	355	}
	356
	357	// ─── Cached wrapper ───────────────────────────────────────────────────────────
	358
	359	export async function getTestGaps(
	360	ownerName: string,
	361	repoName: string,
	362	repoId: string
	363	): Promise<TestGapReport> {
	364	const cached = cache.get(repoId);
	365	if (cached && cached.expiresAt > Date.now()) {
	366	return cached.report;
	367	}
	368
	369	const report = await detectTestGaps(ownerName, repoName, repoId);
	370	cache.set(repoId, {
	371	report,
	372	expiresAt: Date.now() + CACHE_TTL_MS,
	373	});
	374	return report;
	375	}
	376
	377	export function clearTestGapsCache(repoId: string): void {
	378	cache.delete(repoId);
	379	}