Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

semantic-index.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/semantic-index.ts
semantic-index.tsBlame468 lines · 1 contributor
Normal viewRaw
a686079Claude1/**
2 * Continuous semantic index — per-push embeddings.
3 *
4 * Foundation for spec-to-PR, AI rubber-duck, and the v2 /search endpoint.
5 *
6 * Lifecycle:
7 * 1. Push lands → `src/hooks/post-receive.ts` calls
8 * `indexChangedFiles(repoId, commitSha, paths)` fire-and-forget.
9 * 2. For each path we resolve the blob sha at HEAD, pull a 1024-dim
10 * embedding from Voyage (preferred — `voyage-code-3` matches our
11 * column dim) or fall back to a deterministic TF-IDF-ish hash,
12 * then UPSERT into `code_embeddings`.
13 * 3. `searchSemantic(repoId, q)` embeds the query in the same space
14 * and ORDER BY `embedding <=> $1` (cosine distance) on the server.
15 *
16 * Anthropic itself doesn't ship an embeddings API (their docs send you
17 * to Voyage), so "Anthropic embeddings" here means "use Voyage when
18 * ANTHROPIC_API_KEY is set, since the Voyage account is part of the
19 * AI bundle" — VOYAGE_API_KEY is the actual auth header. We probe both.
20 *
21 * Hard rules:
22 * - Never throw. Every external call is wrapped; on any error we log
23 * and return safely (empty array for search, void for index).
24 * - Graceful when pgvector is missing: the table won't exist, so
25 * SELECTs/INSERTs fail and we catch + return empty / void.
26 * - On-disk cache keyed by blob-sha so reindexing identical content
27 * (rebases, force-pushes, branch merges) is free.
28 */
29
30import { mkdir, readFile, writeFile } from "fs/promises";
31import { join } from "path";
32import { tmpdir } from "os";
33import { eq, sql } from "drizzle-orm";
34import { db } from "../db";
35import { codeEmbeddings } from "../db/schema";
36import { getBlob } from "../git/repository";
37import { hashEmbed, tokenize, isCodeFile } from "./semantic-search";
38
39// ---------------------------------------------------------------------------
40// Constants
41// ---------------------------------------------------------------------------
42
43/** Target embedding dimension — matches `vector(1024)` in the schema. */
44export const EMBEDDING_DIM = 1024;
45
46/** First N chars of file content surfaced as preview snippet. */
47const SNIPPET_BYTES = 500;
48
49/** Voyage's per-request batch limit. */
50const VOYAGE_BATCH = 128;
51
52/** Max bytes of file content we send to the embedder. */
53const MAX_EMBED_BYTES = 32 * 1024;
54
55const VOYAGE_MODEL = "voyage-code-3";
56const FALLBACK_MODEL = "gluecron-tfidf-1024";
57
58// ---------------------------------------------------------------------------
59// Disk cache — keyed by sha256(model + ":" + blobSha).
60// ---------------------------------------------------------------------------
61
62let _cacheDirPromise: Promise<string> | null = null;
63
64async function getCacheDir(): Promise<string> {
65 if (_cacheDirPromise) return _cacheDirPromise;
66 _cacheDirPromise = (async () => {
67 const dir =
68 process.env.GLUECRON_SEMANTIC_CACHE_DIR ||
69 join(tmpdir(), "gluecron-semantic-cache");
70 try {
71 await mkdir(dir, { recursive: true });
72 } catch {
73 /* tolerate — falls back to fetch every time */
74 }
75 return dir;
76 })();
77 return _cacheDirPromise;
78}
79
80async function cacheKey(model: string, blobSha: string): Promise<string> {
81 const data = new TextEncoder().encode(`${model}:${blobSha}`);
82 const hash = await crypto.subtle.digest("SHA-256", data);
83 return Array.from(new Uint8Array(hash))
84 .map((b) => b.toString(16).padStart(2, "0"))
85 .join("");
86}
87
88async function readCached(
89 model: string,
90 blobSha: string
91): Promise<number[] | null> {
92 try {
93 const dir = await getCacheDir();
94 const key = await cacheKey(model, blobSha);
95 const path = join(dir, `${key}.json`);
96 const text = await readFile(path, "utf8");
97 const v = JSON.parse(text);
98 if (Array.isArray(v) && v.length === EMBEDDING_DIM) return v as number[];
99 return null;
100 } catch {
101 return null;
102 }
103}
104
105async function writeCached(
106 model: string,
107 blobSha: string,
108 vec: number[]
109): Promise<void> {
110 try {
111 const dir = await getCacheDir();
112 const key = await cacheKey(model, blobSha);
113 const path = join(dir, `${key}.json`);
114 await writeFile(path, JSON.stringify(vec), "utf8");
115 } catch {
116 /* cache miss next time, harmless */
117 }
118}
119
120// ---------------------------------------------------------------------------
121// Provider detection
122// ---------------------------------------------------------------------------
123
124function voyageKey(): string | null {
125 return process.env.VOYAGE_API_KEY || null;
126}
127
128/** What backend will `embed()` use right now? Used by the API endpoint. */
129export function semanticIndexProvider(): "voyage" | "fallback" {
130 return voyageKey() ? "voyage" : "fallback";
131}
132
133// ---------------------------------------------------------------------------
134// Fallback embedder — deterministic, no network.
135//
136// Re-uses the FNV-1a sign-trick hasher from `semantic-search.ts` but at
137// 1024 dimensions so the vectors are directly compatible with the
138// `vector(1024)` column. This lets the index degrade smoothly when no
139// API key is set: search still ranks, just less semantically.
140// ---------------------------------------------------------------------------
141
142function fallbackEmbed(text: string): number[] {
143 return hashEmbed(tokenize(text), EMBEDDING_DIM);
144}
145
146// ---------------------------------------------------------------------------
147// Voyage embedder
148// ---------------------------------------------------------------------------
149
150interface EmbedResult {
151 vectors: number[][];
152 model: string;
153}
154
155async function voyageEmbed(
156 apiKey: string,
157 texts: string[],
158 inputType: "document" | "query"
159): Promise<EmbedResult | null> {
160 const all: number[][] = [];
161 for (let i = 0; i < texts.length; i += VOYAGE_BATCH) {
162 const slice = texts.slice(i, i + VOYAGE_BATCH);
163 try {
164 const resp = await fetch("https://api.voyageai.com/v1/embeddings", {
165 method: "POST",
166 headers: {
167 "content-type": "application/json",
168 authorization: `Bearer ${apiKey}`,
169 },
170 body: JSON.stringify({
171 input: slice,
172 model: VOYAGE_MODEL,
173 input_type: inputType,
174 }),
175 });
176 if (!resp.ok) return null;
177 const json: any = await resp.json();
178 const data = Array.isArray(json?.data) ? json.data : null;
179 if (!data || data.length !== slice.length) return null;
180 for (const row of data) {
181 const emb = row?.embedding;
182 if (!Array.isArray(emb) || emb.length !== EMBEDDING_DIM) return null;
183 all.push(emb as number[]);
184 }
185 } catch {
186 return null;
187 }
188 }
189 return { vectors: all, model: VOYAGE_MODEL };
190}
191
192/**
193 * Embed a single text. Returns `{ vector, model }`. Never throws.
194 *
195 * @internal Exported so tests can stub the network round-trip via
196 * `__setEmbedderForTests`.
197 */
198export async function embedOne(
199 text: string,
200 inputType: "document" | "query"
201): Promise<{ vector: number[]; model: string }> {
202 if (_embedderOverride) {
203 return _embedderOverride(text, inputType);
204 }
205 const key = voyageKey();
206 if (key) {
207 const out = await voyageEmbed(key, [text], inputType);
208 if (out && out.vectors[0]) {
209 return { vector: out.vectors[0], model: out.model };
210 }
211 // fall through
212 }
213 return { vector: fallbackEmbed(text), model: FALLBACK_MODEL };
214}
215
216// Test-only seam — bypass the real network/fallback path so unit tests
217// can assert behaviour deterministically without a Voyage key or DB.
218type Embedder = (
219 text: string,
220 inputType: "document" | "query"
221) => Promise<{ vector: number[]; model: string }>;
222
223let _embedderOverride: Embedder | null = null;
224
225/** Test-only: replace `embedOne`'s implementation. Pass `null` to reset. */
226export function __setEmbedderForTests(fn: Embedder | null): void {
227 _embedderOverride = fn;
228}
229
230// ---------------------------------------------------------------------------
231// Index — called from the post-receive hook.
232// ---------------------------------------------------------------------------
233
234/** Cap on the number of files indexed per push. */
235const MAX_FILES_PER_PUSH = 50;
236
237/**
238 * Embed every changed file at the given commit and upsert one row per
239 * file into `code_embeddings`. Best-effort: returns the count of rows
240 * written. Never throws.
241 *
242 * @param repositoryId - DB id of the repo (NOT owner/name)
243 * @param ownerName - For `getBlob` git lookups
244 * @param repoName - For `getBlob` git lookups
245 * @param commitSha - The new commit sha after the push
246 * @param changedPaths - Paths touched by the push (cap applies)
247 */
248export async function indexChangedFiles(args: {
249 repositoryId: string;
250 ownerName: string;
251 repoName: string;
252 commitSha: string;
253 changedPaths: string[];
254}): Promise<{ indexed: number; skipped: number; model: string }> {
255 const { repositoryId, ownerName, repoName, commitSha, changedPaths } = args;
256
257 if (!repositoryId || !commitSha || !changedPaths.length) {
258 return { indexed: 0, skipped: 0, model: FALLBACK_MODEL };
259 }
260
261 // Dedupe, filter to code files, cap.
262 const seen = new Set<string>();
263 const candidates: string[] = [];
264 for (const p of changedPaths) {
265 if (!p || seen.has(p)) continue;
266 seen.add(p);
267 if (!isCodeFile(p)) continue;
268 candidates.push(p);
269 if (candidates.length >= MAX_FILES_PER_PUSH) break;
270 }
271
272 if (!candidates.length) {
273 return { indexed: 0, skipped: changedPaths.length, model: FALLBACK_MODEL };
274 }
275
276 let indexed = 0;
277 let model = FALLBACK_MODEL;
278
279 for (const filePath of candidates) {
280 let blob: Awaited<ReturnType<typeof getBlob>> = null;
281 try {
282 blob = await getBlob(ownerName, repoName, commitSha, filePath);
283 } catch (err) {
284 // File was deleted by this push, or git error — skip cleanly.
285 blob = null;
286 }
287 if (!blob || blob.isBinary || !blob.content) continue;
288
289 // Lock blob sha for cache keying. If we can't resolve one, derive
290 // a content hash so the disk cache still works.
291 const blobSha = await deriveBlobSha(blob.content);
292 const snippet = blob.content.slice(0, SNIPPET_BYTES);
293 const textToEmbed = `${filePath}\n${blob.content.slice(0, MAX_EMBED_BYTES)}`;
294
295 // Disk cache by (model-namespace, blob sha). We don't know the
296 // resolved model until embedOne runs, so we probe both candidates.
297 let vec: number[] | null = null;
298 let resolvedModel = FALLBACK_MODEL;
299
300 // Try Voyage cache first if a key is configured — saves a real call.
301 if (voyageKey()) {
302 const hit = await readCached(VOYAGE_MODEL, blobSha);
303 if (hit) {
304 vec = hit;
305 resolvedModel = VOYAGE_MODEL;
306 }
307 }
308 if (!vec) {
309 const hit = await readCached(FALLBACK_MODEL, blobSha);
310 if (hit) {
311 vec = hit;
312 resolvedModel = FALLBACK_MODEL;
313 }
314 }
315
316 if (!vec) {
317 try {
318 const out = await embedOne(textToEmbed, "document");
319 vec = out.vector;
320 resolvedModel = out.model;
321 if (vec && vec.length === EMBEDDING_DIM) {
322 void writeCached(resolvedModel, blobSha, vec);
323 }
324 } catch {
325 vec = null;
326 }
327 }
328
329 if (!vec || vec.length !== EMBEDDING_DIM) continue;
330 model = resolvedModel;
331
332 // UPSERT (repository_id, file_path) → new embedding + blob/commit.
333 try {
334 await db
335 .insert(codeEmbeddings)
336 .values({
337 repositoryId,
338 filePath,
339 blobSha,
340 commitSha,
341 contentSnippet: snippet,
342 embedding: vec,
343 embeddingModel: resolvedModel,
344 updatedAt: new Date(),
345 })
346 .onConflictDoUpdate({
347 target: [codeEmbeddings.repositoryId, codeEmbeddings.filePath],
348 set: {
349 blobSha,
350 commitSha,
351 contentSnippet: snippet,
352 embedding: vec,
353 embeddingModel: resolvedModel,
354 updatedAt: new Date(),
355 },
356 });
357 indexed++;
358 } catch (err) {
359 // pgvector missing → table missing → swallow. Single noisy log
360 // per push is enough; rely on the migration NOTICE for diagnosis.
361 if (process.env.DEBUG_SEMANTIC_INDEX === "1") {
362 console.warn(
363 `[semantic-index] upsert failed for ${ownerName}/${repoName}:${filePath}:`,
364 err instanceof Error ? err.message : err
365 );
366 }
367 }
368 }
369
370 return {
371 indexed,
372 skipped: candidates.length - indexed,
373 model,
374 };
375}
376
377async function deriveBlobSha(content: string): Promise<string> {
378 // SHA-256 of the raw bytes — only used as a cache key, doesn't need
379 // to match git's SHA-1 blob hash. (We could call `git hash-object`
380 // but that's another subprocess per file and this is plenty stable.)
381 const data = new TextEncoder().encode(content);
382 const hash = await crypto.subtle.digest("SHA-256", data);
383 return Array.from(new Uint8Array(hash))
384 .map((b) => b.toString(16).padStart(2, "0"))
385 .join("");
386}
387
388// ---------------------------------------------------------------------------
389// Search
390// ---------------------------------------------------------------------------
391
392export interface SemanticHit {
393 filePath: string;
394 snippet: string;
395 score: number;
396 blobSha: string;
397}
398
399/**
400 * Cosine-rank the query against the repo's `code_embeddings` rows and
401 * return the top `limit`. Empty array on any failure (pgvector missing,
402 * no rows, embed failure, etc.). Never throws.
403 *
404 * Score is `1 - cosine_distance`, so higher = closer (0..1 inclusive).
405 */
406export async function searchSemantic(args: {
407 repositoryId: string;
408 query: string;
409 limit?: number;
410}): Promise<SemanticHit[]> {
411 const { repositoryId, query } = args;
412 const limit = Math.max(1, Math.min(args.limit ?? 20, 100));
413 const q = (query || "").trim();
414 if (!q || !repositoryId) return [];
415
416 let queryVec: number[];
417 try {
418 const out = await embedOne(q, "query");
419 queryVec = out.vector;
420 } catch {
421 return [];
422 }
423 if (!queryVec || queryVec.length !== EMBEDDING_DIM) return [];
424
425 // Postgres `<=>` is cosine distance (lower = closer). Score is the
426 // similarity (1 - distance) so callers get a familiar "higher is
427 // better" semantic. We coerce the vector literal manually because
428 // drizzle's parameter binding for our customType happens in the
429 // SELECT clause via `sql`, not in WHERE/ORDER BY.
430 const vecLit = "[" + queryVec.join(",") + "]";
431
432 try {
433 const rows = await db
434 .select({
435 filePath: codeEmbeddings.filePath,
436 snippet: codeEmbeddings.contentSnippet,
437 blobSha: codeEmbeddings.blobSha,
438 score: sql<number>`1 - (${codeEmbeddings.embedding} <=> ${vecLit}::vector)`,
439 })
440 .from(codeEmbeddings)
441 .where(eq(codeEmbeddings.repositoryId, repositoryId))
442 .orderBy(sql`${codeEmbeddings.embedding} <=> ${vecLit}::vector`)
443 .limit(limit);
444
445 return rows.map((r) => ({
446 filePath: r.filePath,
447 snippet: r.snippet || "",
448 score: typeof r.score === "number" ? r.score : Number(r.score) || 0,
449 blobSha: r.blobSha,
450 }));
451 } catch {
452 // pgvector missing or DB unavailable — degrade to empty result.
453 return [];
454 }
455}
456
457// ---------------------------------------------------------------------------
458// Test-only exports
459// ---------------------------------------------------------------------------
460
461export const __test = {
462 fallbackEmbed,
463 deriveBlobSha,
464 cacheKey,
465 MAX_FILES_PER_PUSH,
466 FALLBACK_MODEL,
467 VOYAGE_MODEL,
468};