Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

semantic-search.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/semantic-search.ts
semantic-search.tsBlame578 lines · 1 contributor
Normal viewRaw
3cbe3d6Claude1/**
2 * Block D1 — Semantic code search.
3 *
4 * Two embedding backends:
5 * 1. Voyage AI (`voyage-code-3`, 1024-dim) if VOYAGE_API_KEY is set.
6 * 2. Lexical fallback: 512-dim hashing bag-of-words, L2-normalised.
7 * Deterministic, no network, good-enough baseline for tests + graceful
8 * degradation when no API key is configured.
9 *
10 * Embeddings are stored in `code_chunks.embedding` as a JSON-encoded number
11 * array (schema = text) so we don't depend on pgvector. Cosine similarity
12 * is computed in JS. If/when scale demands it, swap the column type to
13 * `vector(1024)` and push cosine into Postgres.
14 */
15
16import { eq } from "drizzle-orm";
17import { db } from "../db";
18import { codeChunks } from "../db/schema";
19import {
20 getTree,
21 getBlob,
22 type GitTreeEntry,
23} from "../git/repository";
24
25// ---------------------------------------------------------------------------
26// Pure helpers
27// ---------------------------------------------------------------------------
28
29/**
30 * Split code into identifier fragments. Splits on non-word boundaries, then
31 * further splits `camelCase` and `snake_case` / kebab-case tokens into their
32 * constituent pieces. All lowercase. Drops single-character tokens and pure
33 * numeric tokens to keep the feature space meaningful.
34 */
35export function tokenize(code: string): string[] {
36 if (!code) return [];
37 const out: string[] = [];
38 // First pass: split on non-alphanumeric (keep underscores for snake_case detection)
39 const rough = code.split(/[^A-Za-z0-9_]+/).filter(Boolean);
40 for (const tok of rough) {
41 // Split snake_case / kebab (kebab already gone; underscores split here)
42 const underscoreParts = tok.split(/_+/).filter(Boolean);
43 for (const part of underscoreParts) {
44 // Split camelCase / PascalCase: insert boundary before each uppercase
45 // letter that follows a lowercase or digit, and before the last upper
46 // in a run of uppers followed by a lower (XMLParser -> XML, Parser).
47 const camelParts = part
48 .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
49 .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
50 .split(/\s+/)
51 .filter(Boolean);
52 for (const cp of camelParts) {
53 const lower = cp.toLowerCase();
54 if (lower.length < 2) continue;
55 if (/^\d+$/.test(lower)) continue;
56 out.push(lower);
57 }
58 }
59 }
60 return out;
61}
62
63/**
64 * FNV-1a 32-bit hash of a string. Deterministic + fast enough for the tiny
65 * token volumes we throw at it.
66 */
67function fnv1a(s: string): number {
68 let h = 0x811c9dc5;
69 for (let i = 0; i < s.length; i++) {
70 h ^= s.charCodeAt(i);
71 // 32-bit FNV prime multiplication via Math.imul
72 h = Math.imul(h, 0x01000193);
73 }
74 return h >>> 0;
75}
76
77/**
78 * Feature-hashing embedding with the sign trick. Maps each token into one of
79 * `dim` slots; a second hash decides whether to add +1 or -1. Finally L2-
80 * normalises so cosine ≈ dot product.
81 */
82export function hashEmbed(tokens: string[], dim = 512): number[] {
83 const v = new Array<number>(dim).fill(0);
84 if (!tokens.length) return v;
85 for (const tok of tokens) {
86 const h = fnv1a(tok);
87 const slot = h % dim;
88 // Sign from a perturbed second hash so it's not correlated with slot
89 const signHash = fnv1a("\x00" + tok);
90 const sign = signHash & 1 ? 1 : -1;
91 v[slot] += sign;
92 }
93 let sumsq = 0;
94 for (let i = 0; i < dim; i++) sumsq += v[i] * v[i];
95 if (sumsq === 0) return v;
96 const inv = 1 / Math.sqrt(sumsq);
97 for (let i = 0; i < dim; i++) v[i] *= inv;
98 return v;
99}
100
101/** Cosine similarity. Assumes a, b are same length. Handles zero vectors. */
102export function cosine(a: number[], b: number[]): number {
103 if (!a || !b) return 0;
104 const n = Math.min(a.length, b.length);
105 let dot = 0;
106 let na = 0;
107 let nb = 0;
108 for (let i = 0; i < n; i++) {
109 dot += a[i] * b[i];
110 na += a[i] * a[i];
111 nb += b[i] * b[i];
112 }
113 if (na === 0 || nb === 0) return 0;
114 return dot / (Math.sqrt(na) * Math.sqrt(nb));
115}
116
117const CODE_EXTS = new Set([
118 "ts",
119 "tsx",
120 "js",
121 "jsx",
122 "mjs",
123 "cjs",
124 "py",
125 "go",
126 "rs",
127 "java",
128 "rb",
129 "php",
130 "c",
131 "cpp",
132 "cc",
133 "h",
134 "hpp",
135 "md",
136 "mdx",
137 "yaml",
138 "yml",
139 "json",
140 "css",
141 "html",
142 "htm",
143]);
144
145const SKIP_FILES = new Set([
146 "package-lock.json",
147 "yarn.lock",
148 "pnpm-lock.yaml",
149 "bun.lockb",
150 "bun.lock",
151 "poetry.lock",
152 "cargo.lock",
153 "composer.lock",
154 "gemfile.lock",
155]);
156
157/**
158 * Is the given path a code-like file we should index? Rejects lock files,
159 * known binary extensions, images, and anything without an extension we
160 * recognise.
161 */
162export function isCodeFile(path: string): boolean {
163 if (!path) return false;
164 const base = path.split("/").pop() || "";
165 const lower = base.toLowerCase();
166 if (SKIP_FILES.has(lower)) return false;
167 const dot = lower.lastIndexOf(".");
168 if (dot === -1) return false;
169 const ext = lower.slice(dot + 1);
170 if (!CODE_EXTS.has(ext)) return false;
171 return true;
172}
173
174/**
175 * Split a file's content into overlapping chunks of ~maxLines lines with
176 * a 5-line overlap. Skips non-code files. Returns [] for empty / binary-
177 * looking content.
178 */
179export function chunkFile(
180 path: string,
181 content: string,
182 maxLines = 40
183): Array<{ path: string; startLine: number; endLine: number; content: string }> {
184 if (!isCodeFile(path)) return [];
185 if (!content) return [];
186 if (content.includes("\0")) return []; // binary blob, skip
187 const lines = content.split("\n");
188 if (lines.length === 0) return [];
189
190 const overlap = 5;
191 const step = Math.max(1, maxLines - overlap);
192 const out: Array<{
193 path: string;
194 startLine: number;
195 endLine: number;
196 content: string;
197 }> = [];
198
199 // For short files, emit a single chunk.
200 if (lines.length <= maxLines) {
201 out.push({
202 path,
203 startLine: 1,
204 endLine: lines.length,
205 content: lines.join("\n"),
206 });
207 return out;
208 }
209
210 for (let start = 0; start < lines.length; start += step) {
211 const end = Math.min(lines.length, start + maxLines);
212 out.push({
213 path,
214 startLine: start + 1,
215 endLine: end,
216 content: lines.slice(start, end).join("\n"),
217 });
218 if (end >= lines.length) break;
219 }
220 return out;
221}
222
223// ---------------------------------------------------------------------------
224// Provider detection
225// ---------------------------------------------------------------------------
226
227export function isEmbeddingsProviderAvailable(): {
228 voyage: boolean;
229 fallback: true;
230} {
231 return {
232 voyage: !!process.env.VOYAGE_API_KEY,
233 fallback: true,
234 };
235}
236
237const VOYAGE_MODEL = "voyage-code-3";
238const FALLBACK_MODEL = "gluecron-hash-512";
239const VOYAGE_BATCH = 128;
240
241/**
242 * Embed a batch of texts. Uses Voyage AI when VOYAGE_API_KEY is set, and
243 * falls back to `hashEmbed(tokenize(...))` per text otherwise — or if the
244 * Voyage request fails for any reason.
245 *
246 * Never throws. Always returns the same number of vectors as inputs.
247 */
248export async function embedBatch(
249 texts: string[],
250 inputType: "document" | "query"
251): Promise<{ vectors: number[][]; model: string }> {
252 if (!texts.length) return { vectors: [], model: FALLBACK_MODEL };
253
254 const apiKey = process.env.VOYAGE_API_KEY;
255 if (apiKey) {
256 const all: number[][] = [];
257 let ok = true;
258 for (let i = 0; i < texts.length && ok; i += VOYAGE_BATCH) {
259 const slice = texts.slice(i, i + VOYAGE_BATCH);
260 try {
261 const resp = await fetch("https://api.voyageai.com/v1/embeddings", {
262 method: "POST",
263 headers: {
264 "content-type": "application/json",
265 authorization: `Bearer ${apiKey}`,
266 },
267 body: JSON.stringify({
268 input: slice,
269 model: VOYAGE_MODEL,
270 input_type: inputType,
271 }),
272 });
273 if (!resp.ok) {
274 ok = false;
275 break;
276 }
277 const json: any = await resp.json();
278 const data = Array.isArray(json?.data) ? json.data : null;
279 if (!data || data.length !== slice.length) {
280 ok = false;
281 break;
282 }
283 for (const row of data) {
284 const emb = row?.embedding;
285 if (!Array.isArray(emb)) {
286 ok = false;
287 break;
288 }
289 all.push(emb as number[]);
290 }
291 } catch {
292 ok = false;
293 break;
294 }
295 }
296 if (ok && all.length === texts.length) {
297 return { vectors: all, model: VOYAGE_MODEL };
298 }
299 // fall through to fallback for the entire batch
300 }
301
302 const vectors = texts.map((t) => hashEmbed(tokenize(t), 512));
303 return { vectors, model: FALLBACK_MODEL };
304}
305
306// ---------------------------------------------------------------------------
307// Tree walking
308// ---------------------------------------------------------------------------
309
310const MAX_CHUNKS_PER_REPO = 2000;
311const MAX_BLOB_BYTES = 256 * 1024; // 256KB — skip anything larger
312
313async function walkCodePaths(
314 owner: string,
315 repo: string,
316 ref: string,
317 maxFiles = 5000
318): Promise<string[]> {
319 const out: string[] = [];
320 const queue: string[] = [""];
321 while (queue.length && out.length < maxFiles) {
322 const dir = queue.shift()!;
323 let entries: GitTreeEntry[] = [];
324 try {
325 entries = await getTree(owner, repo, ref, dir);
326 } catch {
327 continue;
328 }
329 for (const e of entries) {
330 const p = dir ? `${dir}/${e.name}` : e.name;
331 if (e.type === "tree") {
332 // Skip common noise directories.
333 const base = e.name.toLowerCase();
334 if (
335 base === "node_modules" ||
336 base === ".git" ||
337 base === "dist" ||
338 base === "build" ||
339 base === "vendor" ||
340 base === ".next" ||
341 base === ".turbo" ||
342 base === "target" ||
343 base === "__pycache__"
344 ) {
345 continue;
346 }
347 queue.push(p);
348 } else if (e.type === "blob") {
349 if (!isCodeFile(p)) continue;
350 if (e.size !== undefined && e.size > MAX_BLOB_BYTES) continue;
351 out.push(p);
352 if (out.length >= maxFiles) break;
353 }
354 }
355 }
356 return out;
357}
358
359// ---------------------------------------------------------------------------
360// Indexing
361// ---------------------------------------------------------------------------
362
363export interface IndexResult {
364 chunksIndexed: number;
365 model: string;
366}
367
368/**
369 * Walk the repo tree at the given commit sha, chunk every code file, embed
370 * the chunks in batches, and replace any previous index rows for this repo.
371 *
372 * Caps total chunks at `MAX_CHUNKS_PER_REPO` (logs + stops). Never throws —
373 * returns `{ chunksIndexed: 0, model }` on any failure.
374 */
375export async function indexRepository(args: {
376 owner: string;
377 repo: string;
378 repositoryId: string;
379 commitSha: string;
380}): Promise<IndexResult> {
381 const { owner, repo, repositoryId, commitSha } = args;
382
383 let chunks: Array<{
384 path: string;
385 startLine: number;
386 endLine: number;
387 content: string;
388 }> = [];
389
390 try {
391 const paths = await walkCodePaths(owner, repo, commitSha);
392 for (const p of paths) {
393 if (chunks.length >= MAX_CHUNKS_PER_REPO) {
394 console.warn(
395 `[semantic-search] chunk cap hit (${MAX_CHUNKS_PER_REPO}) for ${owner}/${repo} @ ${commitSha}; truncating`
396 );
397 break;
398 }
399 let blob;
400 try {
401 blob = await getBlob(owner, repo, commitSha, p);
402 } catch {
403 continue;
404 }
405 if (!blob || blob.isBinary) continue;
406 const fileChunks = chunkFile(p, blob.content, 40);
407 for (const ch of fileChunks) {
408 if (chunks.length >= MAX_CHUNKS_PER_REPO) break;
409 chunks.push(ch);
410 }
411 }
412 } catch (err) {
413 console.error(
414 `[semantic-search] tree walk failed for ${owner}/${repo}:`,
415 err
416 );
417 return { chunksIndexed: 0, model: FALLBACK_MODEL };
418 }
419
420 if (!chunks.length) {
421 // Still wipe old rows so stale indexes don't linger.
422 try {
423 await db.delete(codeChunks).where(eq(codeChunks.repositoryId, repositoryId));
424 } catch {}
425 return { chunksIndexed: 0, model: FALLBACK_MODEL };
426 }
427
428 // Embed in batches sized for Voyage's 128/request limit.
429 let model = FALLBACK_MODEL;
430 const vectors: number[][] = [];
431 try {
432 for (let i = 0; i < chunks.length; i += VOYAGE_BATCH) {
433 const slice = chunks.slice(i, i + VOYAGE_BATCH);
434 const { vectors: vs, model: m } = await embedBatch(
435 slice.map((c) => `${c.path}\n${c.content}`),
436 "document"
437 );
438 model = m;
439 for (const v of vs) vectors.push(v);
440 }
441 } catch (err) {
442 console.error(`[semantic-search] embed failed for ${owner}/${repo}:`, err);
443 return { chunksIndexed: 0, model };
444 }
445
446 if (vectors.length !== chunks.length) {
447 console.error(
448 `[semantic-search] vector/chunk length mismatch (${vectors.length} vs ${chunks.length})`
449 );
450 return { chunksIndexed: 0, model };
451 }
452
453 try {
454 await db.delete(codeChunks).where(eq(codeChunks.repositoryId, repositoryId));
455
456 // Batch-insert to avoid one-row-per-roundtrip.
457 const INSERT_BATCH = 100;
458 for (let i = 0; i < chunks.length; i += INSERT_BATCH) {
459 const slice = chunks.slice(i, i + INSERT_BATCH);
460 const rows = slice.map((c, j) => ({
461 repositoryId,
462 commitSha,
463 path: c.path,
464 startLine: c.startLine,
465 endLine: c.endLine,
466 content: c.content,
467 embedding: JSON.stringify(vectors[i + j]),
468 embeddingModel: model,
469 }));
470 await db.insert(codeChunks).values(rows);
471 }
472 } catch (err) {
473 console.error(`[semantic-search] DB write failed for ${owner}/${repo}:`, err);
474 return { chunksIndexed: 0, model };
475 }
476
477 return { chunksIndexed: chunks.length, model };
478}
479
480// ---------------------------------------------------------------------------
481// Search
482// ---------------------------------------------------------------------------
483
484export interface SearchHit {
485 path: string;
486 startLine: number;
487 endLine: number;
488 content: string;
489 score: number;
490}
491
492/**
493 * Embed `query`, load all chunk embeddings for this repo, rank by cosine,
494 * return the top `limit`. Empty array if the repo has no indexed chunks or
495 * anything fails.
496 */
497export async function searchRepository(args: {
498 repositoryId: string;
499 query: string;
500 limit?: number;
501}): Promise<SearchHit[]> {
502 const { repositoryId, query } = args;
503 const limit = args.limit ?? 20;
504 const q = (query || "").trim();
505 if (!q) return [];
506
507 let rows: Array<{
508 path: string;
509 startLine: number;
510 endLine: number;
511 content: string;
512 embedding: string | null;
513 embeddingModel: string | null;
514 }>;
515 try {
516 rows = await db
517 .select({
518 path: codeChunks.path,
519 startLine: codeChunks.startLine,
520 endLine: codeChunks.endLine,
521 content: codeChunks.content,
522 embedding: codeChunks.embedding,
523 embeddingModel: codeChunks.embeddingModel,
524 })
525 .from(codeChunks)
526 .where(eq(codeChunks.repositoryId, repositoryId));
527 } catch {
528 return [];
529 }
530
531 if (!rows.length) return [];
532
533 // Assume all rows use the same model (indexRepository rewrites them in bulk).
534 const model = rows[0].embeddingModel || FALLBACK_MODEL;
535
536 let queryVec: number[];
537 if (model === VOYAGE_MODEL && process.env.VOYAGE_API_KEY) {
538 const { vectors } = await embedBatch([q], "query");
539 queryVec = vectors[0];
540 } else {
541 queryVec = hashEmbed(tokenize(q), 512);
542 }
543
544 const scored: SearchHit[] = [];
545 for (const r of rows) {
546 if (!r.embedding) continue;
547 let v: number[];
548 try {
549 v = JSON.parse(r.embedding);
550 } catch {
551 continue;
552 }
553 if (!Array.isArray(v) || v.length === 0) continue;
554 const s = cosine(queryVec, v);
555 scored.push({
556 path: r.path,
557 startLine: r.startLine,
558 endLine: r.endLine,
559 content: r.content,
560 score: s,
561 });
562 }
563 scored.sort((a, b) => b.score - a.score);
564 return scored.slice(0, limit);
565}
566
567// ---------------------------------------------------------------------------
568// Test-only exports — pure helpers, no DB dependency.
569// ---------------------------------------------------------------------------
570
571export const __test = {
572 tokenize,
573 hashEmbed,
574 cosine,
575 isCodeFile,
576 chunkFile,
577 fnv1a,
578};