Blame · Line-by-line history

`semantic-search.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

semantic-search.test.tsBlame274 lines · 1 contributor

Normal view Raw

3cbe3d6Claude	1	/**
	2	* Tests for Block D1 — semantic code search.
	3	*
	4	* Covers the pure helpers (tokenize, hashEmbed, cosine, isCodeFile,
	5	* chunkFile) plus a route-level smoke test that the /:owner/:repo/search/semantic
	6	* page always resolves — even for a nonexistent repo, where it must render
	7	* a 404 Layout rather than blowing up.
	8	*
	9	* These tests deliberately avoid Voyage and the DB. The fallback embedder
	10	* is pure math; the route falls through to the global 404 path when the
	11	* repo doesn't exist.
	12	*/
	13
	14	import { describe, it, expect } from "bun:test";
	15	import app from "../app";
	16	import {
	17	tokenize,
	18	hashEmbed,
	19	cosine,
	20	isCodeFile,
	21	chunkFile,
	22	isEmbeddingsProviderAvailable,
	23	__test,
	24	} from "../lib/semantic-search";
	25
	26	describe("tokenize", () => {
	27	it("splits on non-word boundaries", () => {
	28	const t = tokenize("hello, world!");
	29	expect(t).toContain("hello");
	30	expect(t).toContain("world");
	31	});
	32
	33	it("splits camelCase into fragments", () => {
	34	const t = tokenize("getUserName");
	35	expect(t).toContain("get");
	36	expect(t).toContain("user");
	37	expect(t).toContain("name");
	38	});
	39
	40	it("splits PascalCase and ALLCAPS runs", () => {
	41	const t = tokenize("XMLParser");
	42	expect(t).toContain("xml");
	43	expect(t).toContain("parser");
	44	});
	45
	46	it("splits snake_case into fragments", () => {
	47	const t = tokenize("snake_case_name");
	48	expect(t).toContain("snake");
	49	expect(t).toContain("case");
	50	expect(t).toContain("name");
	51	});
	52
	53	it("lowercases everything", () => {
	54	const t = tokenize("FooBar");
	55	for (const tok of t) {
	56	expect(tok).toBe(tok.toLowerCase());
	57	}
	58	});
	59
	60	it("drops single-character tokens and pure numeric tokens", () => {
	61	const t = tokenize("a b foo 123 bar2");
	62	expect(t).not.toContain("a");
	63	expect(t).not.toContain("b");
	64	expect(t).not.toContain("123");
	65	expect(t).toContain("foo");
	66	expect(t).toContain("bar2");
	67	});
	68
	69	it("returns [] for empty input", () => {
	70	expect(tokenize("")).toEqual([]);
	71	});
	72	});
	73
	74	describe("hashEmbed", () => {
	75	it("returns a vector of the requested dimension", () => {
	76	const v = hashEmbed(["hello", "world"], 512);
	77	expect(v.length).toBe(512);
	78	});
	79
	80	it("produces an L2-normalized vector (sum of squares ≈ 1)", () => {
	81	const v = hashEmbed(tokenize("const user = getUserById(id);"), 512);
	82	let sq = 0;
	83	for (const x of v) sq += x * x;
	84	expect(sq).toBeGreaterThan(0.99);
	85	expect(sq).toBeLessThan(1.01);
	86	});
	87
	88	it("returns an all-zero vector for empty token list", () => {
	89	const v = hashEmbed([], 512);
	90	expect(v.length).toBe(512);
	91	expect(v.every((x) => x === 0)).toBe(true);
	92	});
	93
	94	it("is deterministic across calls", () => {
	95	const v1 = hashEmbed(["foo", "bar", "baz"]);
	96	const v2 = hashEmbed(["foo", "bar", "baz"]);
	97	expect(v1).toEqual(v2);
	98	});
	99
	100	it("honours a custom dimension", () => {
	101	const v = hashEmbed(["x"], 128);
	102	expect(v.length).toBe(128);
	103	});
	104	});
	105
	106	describe("cosine", () => {
	107	it("returns ~1 for identical non-zero vectors", () => {
	108	const v = hashEmbed(tokenize("const answer = 42;"));
	109	const s = cosine(v, v);
	110	expect(s).toBeGreaterThan(0.999);
	111	expect(s).toBeLessThan(1.001);
	112	});
	113
	114	it("returns 0 for orthogonal vectors", () => {
	115	const a = [1, 0, 0, 0];
	116	const b = [0, 1, 0, 0];
	117	expect(cosine(a, b)).toBe(0);
	118	});
	119
	120	it("returns 0 when either vector is all zeros", () => {
	121	const z = [0, 0, 0];
	122	const v = [1, 2, 3];
	123	expect(cosine(z, v)).toBe(0);
	124	expect(cosine(v, z)).toBe(0);
	125	});
	126
	127	it("ranks a close match higher than an unrelated one", () => {
	128	const query = hashEmbed(tokenize("parse JSON response"));
	129	const close = hashEmbed(
	130	tokenize("function parseJsonResponse(text) { return JSON.parse(text); }")
	131	);
	132	const far = hashEmbed(
	133	tokenize("draw a red rectangle on the canvas context")
	134	);
	135	expect(cosine(query, close)).toBeGreaterThan(cosine(query, far));
	136	});
	137	});
	138
	139	describe("isCodeFile", () => {
	140	it("accepts common source extensions", () => {
	141	expect(isCodeFile("src/index.ts")).toBe(true);
	142	expect(isCodeFile("app.tsx")).toBe(true);
	143	expect(isCodeFile("main.py")).toBe(true);
	144	expect(isCodeFile("lib.rs")).toBe(true);
	145	expect(isCodeFile("server.go")).toBe(true);
	146	expect(isCodeFile("style.css")).toBe(true);
	147	expect(isCodeFile("README.md")).toBe(true);
	148	expect(isCodeFile("config.yaml")).toBe(true);
	149	expect(isCodeFile("config.yml")).toBe(true);
	150	expect(isCodeFile("tsconfig.json")).toBe(true);
	151	});
	152
	153	it("rejects lock files", () => {
	154	expect(isCodeFile("package-lock.json")).toBe(false);
	155	expect(isCodeFile("yarn.lock")).toBe(false);
	156	expect(isCodeFile("bun.lockb")).toBe(false);
	157	expect(isCodeFile("bun.lock")).toBe(false);
	158	expect(isCodeFile("pnpm-lock.yaml")).toBe(false);
	159	expect(isCodeFile("poetry.lock")).toBe(false);
	160	expect(isCodeFile("Cargo.lock")).toBe(false);
	161	});
	162
	163	it("rejects binary / image extensions", () => {
	164	expect(isCodeFile("logo.png")).toBe(false);
	165	expect(isCodeFile("photo.jpg")).toBe(false);
	166	expect(isCodeFile("anim.gif")).toBe(false);
	167	expect(isCodeFile("dump.bin")).toBe(false);
	168	expect(isCodeFile("a.exe")).toBe(false);
	169	expect(isCodeFile("font.woff2")).toBe(false);
	170	});
	171
	172	it("rejects files with no extension", () => {
	173	expect(isCodeFile("Makefile")).toBe(false);
	174	expect(isCodeFile("Dockerfile")).toBe(false);
	175	expect(isCodeFile("LICENSE")).toBe(false);
	176	});
	177
	178	it("rejects empty path", () => {
	179	expect(isCodeFile("")).toBe(false);
	180	});
	181	});
	182
	183	describe("chunkFile", () => {
	184	it("returns [] for non-code paths", () => {
	185	expect(chunkFile("image.png", "whatever")).toEqual([]);
	186	expect(chunkFile("package-lock.json", "{}" )).toEqual([]);
	187	});
	188
	189	it("returns [] for empty content", () => {
	190	expect(chunkFile("foo.ts", "")).toEqual([]);
	191	});
	192
	193	it("emits a single chunk for short files", () => {
	194	const content = Array.from({ length: 10 }, (_, i) => `line${i + 1}`).join("\n");
	195	const chunks = chunkFile("short.ts", content, 40);
	196	expect(chunks.length).toBe(1);
	197	expect(chunks[0].startLine).toBe(1);
	198	expect(chunks[0].endLine).toBe(10);
	199	expect(chunks[0].path).toBe("short.ts");
	200	});
	201
	202	it("produces overlapping chunks with expected start/end lines", () => {
	203	// 100 lines, chunkSize 40, overlap 5 → step 35
	204	const content = Array.from({ length: 100 }, (_, i) => `L${i + 1}`).join("\n");
	205	const chunks = chunkFile("big.ts", content, 40);
	206	expect(chunks.length).toBeGreaterThan(1);
	207
	208	// First chunk: lines 1..40
	209	expect(chunks[0].startLine).toBe(1);
	210	expect(chunks[0].endLine).toBe(40);
	211
	212	// Second chunk starts 35 lines later (40 - 5 overlap), i.e. line 36
	213	expect(chunks[1].startLine).toBe(36);
	214	expect(chunks[1].endLine).toBe(75);
	215
	216	// Overlap: last 5 lines of chunk[0] equal first 5 of chunk[1].
	217	const c0Lines = chunks[0].content.split("\n");
	218	const c1Lines = chunks[1].content.split("\n");
	219	expect(c0Lines.slice(-5)).toEqual(c1Lines.slice(0, 5));
	220
	221	// Last chunk must end at the final line exactly.
	222	const last = chunks[chunks.length - 1];
	223	expect(last.endLine).toBe(100);
	224	});
	225
	226	it("preserves the exact path", () => {
	227	const chunks = chunkFile("src/deep/path/file.ts", "a\nb\nc", 40);
	228	expect(chunks[0].path).toBe("src/deep/path/file.ts");
	229	});
	230	});
	231
	232	describe("isEmbeddingsProviderAvailable", () => {
	233	it("always reports fallback: true", () => {
	234	const p = isEmbeddingsProviderAvailable();
	235	expect(p.fallback).toBe(true);
	236	expect(typeof p.voyage).toBe("boolean");
	237	});
	238	});
	239
	240	describe("__test bundle", () => {
	241	it("exports the pure helpers without DB dependency", () => {
	242	expect(typeof __test.tokenize).toBe("function");
	243	expect(typeof __test.hashEmbed).toBe("function");
	244	expect(typeof __test.cosine).toBe("function");
	245	expect(typeof __test.isCodeFile).toBe("function");
	246	expect(typeof __test.chunkFile).toBe("function");
	247	});
	248
	249	it("hashes deterministically via __test.fnv1a", () => {
	250	expect(__test.fnv1a("hello")).toBe(__test.fnv1a("hello"));
	251	expect(__test.fnv1a("hello")).not.toBe(__test.fnv1a("world"));
	252	});
	253	});
	254
	255	describe("semantic-search route — smoke", () => {
	256	it("GET /:owner/:repo/search/semantic for a nonexistent repo returns 404 HTML", async () => {
	257	const res = await app.request(
	258	"/nonexistent-user-xyz/nonexistent-repo-xyz/search/semantic"
	259	);
	260	// Either our own 404 (repo not found) or the global 404 if the router
	261	// isn't mounted yet — both are acceptable.
	262	expect([200, 404, 503]).toContain(res.status);
	263	const html = await res.text();
	264	// Layout marker — the response should be a full HTML shell, not JSON.
	265	expect(html.toLowerCase()).toContain("<html");
	266	});
	267
	268	it("GET without query string renders the Layout (no crash on empty q)", async () => {
	269	const res = await app.request(
	270	"/nonexistent-user-xyz/nonexistent-repo-xyz/search/semantic?q="
	271	);
	272	expect([200, 404, 503]).toContain(res.status);
	273	});
	274	});