Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

claude-semantic-search.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/claude-semantic-search.ts
claude-semantic-search.tsBlame538 lines · 1 contributor
Normal viewRaw
53c9249Claude1/**
2 * Claude-API-based semantic code search.
3 *
4 * Unlike the embedding-based search in semantic-search.ts (which requires
5 * Voyage AI + a pre-built index), this approach uses Claude to understand
6 * the query and rank files in natural language — no vector DB, no prior
7 * indexing. Works immediately on any repo.
8 *
9 * Algorithm:
10 * 1. List all files via `git ls-tree -r --name-only HEAD` (up to 1000 files).
11 * 2. For large repos (>200 files): pre-filter with `git grep -l keyword`.
12 * 3. Build a compact index: for each candidate file, read its first 50 lines.
13 * 4. Send index + query to Claude (haiku — fast + cheap) and ask for a
14 * JSON-ranked result: [{file, reason, confidence}].
15 * 5. For the top 5 results, read the actual file and extract the most
16 * relevant 20-line snippet (heuristic: find the block that contains the
17 * most query-adjacent tokens).
18 * 6. Cache results in-memory for 5 minutes (Map<`${repoId}:${query}`, ...>).
19 *
20 * Rate limit: 20 semantic searches per user (by userId or IP) per hour.
21 * After the limit is hit, falls back to `git grep` keyword search.
22 *
23 * Fallback: if ANTHROPIC_API_KEY is not set, falls back to `git grep`.
24 */
25
26import { getAnthropic, MODEL_HAIKU, parseJsonResponse } from "./ai-client";
27import { config } from "./config";
28import { getRepoPath } from "../git/repository";
29
30// ---------------------------------------------------------------------------
31// Types
32// ---------------------------------------------------------------------------
33
34export interface SemanticSearchResult {
35 file: string;
36 reason: string; // AI's explanation of why this file is relevant
37 confidence: number; // 0–1
38 snippet: string; // up to 20 lines of the most relevant content
39 lineNumber?: number; // best-guess start line for the snippet
40}
41
42// ---------------------------------------------------------------------------
43// In-memory cache (5-minute TTL)
44// ---------------------------------------------------------------------------
45
46interface CacheEntry {
47 results: SemanticSearchResult[];
48 expiresAt: number;
49}
50
51const resultCache = new Map<string, CacheEntry>();
52const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
53
54function cacheGet(key: string): SemanticSearchResult[] | null {
55 const entry = resultCache.get(key);
56 if (!entry) return null;
57 if (Date.now() > entry.expiresAt) {
58 resultCache.delete(key);
59 return null;
60 }
61 return entry.results;
62}
63
64function cacheSet(key: string, results: SemanticSearchResult[]): void {
65 // Evict old entries to keep memory bounded.
66 if (resultCache.size > 500) {
67 const now = Date.now();
68 for (const [k, v] of resultCache) {
69 if (v.expiresAt < now) resultCache.delete(k);
70 }
71 }
72 resultCache.set(key, { results, expiresAt: Date.now() + CACHE_TTL_MS });
73}
74
75// ---------------------------------------------------------------------------
76// Per-user/IP rate limiting (20 semantic searches per hour)
77// ---------------------------------------------------------------------------
78
79interface BucketEntry {
80 count: number;
81 resetAt: number;
82}
83
84const semanticSearchBuckets = new Map<string, BucketEntry>();
85const SEMANTIC_RATE_WINDOW_MS = 60 * 60 * 1000; // 1 hour
86const SEMANTIC_RATE_MAX = 20;
87
88/** Returns true if the caller is within quota, false if exceeded. */
89export function checkSemanticRateLimit(key: string): boolean {
90 const now = Date.now();
91 let bucket = semanticSearchBuckets.get(key);
92 if (!bucket || bucket.resetAt < now) {
93 bucket = { count: 0, resetAt: now + SEMANTIC_RATE_WINDOW_MS };
94 semanticSearchBuckets.set(key, bucket);
95 }
96 bucket.count++;
97 return bucket.count <= SEMANTIC_RATE_MAX;
98}
99
100/** Returns remaining semantic searches in the current window (never negative). */
101export function semanticRateLimitRemaining(key: string): number {
102 const now = Date.now();
103 const bucket = semanticSearchBuckets.get(key);
104 if (!bucket || bucket.resetAt < now) return SEMANTIC_RATE_MAX;
105 return Math.max(0, SEMANTIC_RATE_MAX - bucket.count);
106}
107
108// ---------------------------------------------------------------------------
109// Skip-list: directories we never look inside
110// ---------------------------------------------------------------------------
111
112const SKIP_DIRS = new Set([
113 "node_modules",
114 ".git",
115 "dist",
116 "build",
117 "vendor",
118 ".next",
119 ".turbo",
120 "target",
121 "__pycache__",
122]);
123
124const SKIP_FILES = new Set([
125 "package-lock.json",
126 "yarn.lock",
127 "pnpm-lock.yaml",
128 "bun.lockb",
129 "bun.lock",
130 "poetry.lock",
131 "cargo.lock",
132 "composer.lock",
133 "gemfile.lock",
134]);
135
136function shouldSkipPath(path: string): boolean {
137 const parts = path.split("/");
138 // Skip if any directory segment is in the skip list.
139 for (const part of parts.slice(0, -1)) {
140 if (SKIP_DIRS.has(part.toLowerCase())) return true;
141 }
142 const basename = parts[parts.length - 1].toLowerCase();
143 if (SKIP_FILES.has(basename)) return true;
144 return false;
145}
146
147// ---------------------------------------------------------------------------
148// Git helpers
149// ---------------------------------------------------------------------------
150
151async function gitExec(
152 cmd: string[],
153 cwd: string
154): Promise<{ stdout: string; exitCode: number }> {
155 const proc = Bun.spawn(cmd, {
156 cwd,
157 env: process.env as Record<string, string>,
158 stdout: "pipe",
159 stderr: "pipe",
160 });
161 const stdout = await new Response(proc.stdout).text();
162 const exitCode = await proc.exited;
163 return { stdout, exitCode };
164}
165
166async function lsFiles(owner: string, repo: string, branch: string): Promise<string[]> {
167 const repoDir = getRepoPath(owner, repo);
168 const { stdout, exitCode } = await gitExec(
169 ["git", "ls-tree", "-r", "--name-only", branch],
170 repoDir
171 );
172 if (exitCode !== 0) return [];
173 return stdout
174 .trim()
175 .split("\n")
176 .filter(Boolean)
177 .filter((p) => !shouldSkipPath(p));
178}
179
180async function grepFiles(
181 owner: string,
182 repo: string,
183 branch: string,
184 keyword: string
185): Promise<string[]> {
186 const repoDir = getRepoPath(owner, repo);
187 const { stdout } = await gitExec(
188 ["git", "grep", "-l", "-i", keyword, branch, "--"],
189 repoDir
190 );
191 return stdout
192 .trim()
193 .split("\n")
194 .filter(Boolean)
195 .map((line) => {
196 // git grep -l output: "<ref>:<path>"
197 const idx = line.indexOf(":");
198 return idx >= 0 ? line.slice(idx + 1) : line;
199 })
200 .filter((p) => !shouldSkipPath(p));
201}
202
203async function readFileHead(
204 owner: string,
205 repo: string,
206 branch: string,
207 filePath: string,
208 maxLines = 50
209): Promise<string> {
210 const repoDir = getRepoPath(owner, repo);
211 const { stdout, exitCode } = await gitExec(
212 ["git", "show", `${branch}:${filePath}`],
213 repoDir
214 );
215 if (exitCode !== 0) return "";
216 return stdout.split("\n").slice(0, maxLines).join("\n");
217}
218
219async function readFileFull(
220 owner: string,
221 repo: string,
222 branch: string,
223 filePath: string
224): Promise<string> {
225 const repoDir = getRepoPath(owner, repo);
226 const { stdout, exitCode } = await gitExec(
227 ["git", "show", `${branch}:${filePath}`],
228 repoDir
229 );
230 if (exitCode !== 0) return "";
231 return stdout;
232}
233
234// ---------------------------------------------------------------------------
235// Snippet extraction
236// ---------------------------------------------------------------------------
237
238/**
239 * Extract the most relevant 20-line window from a file for a given query.
240 * Strategy: split query into lowercase tokens, score each line by how many
241 * tokens it contains, then find the 20-line window with the highest total
242 * score. Falls back to the first 20 lines when no tokens match anything.
243 */
244function extractSnippet(
245 content: string,
246 query: string,
247 windowSize = 20
248): { snippet: string; lineNumber: number } {
249 const lines = content.split("\n");
250 if (lines.length <= windowSize) {
251 return { snippet: content, lineNumber: 1 };
252 }
253
254 const tokens = query
255 .toLowerCase()
256 .split(/[^a-z0-9]+/)
257 .filter((t) => t.length >= 3);
258
259 if (tokens.length === 0) {
260 return { snippet: lines.slice(0, windowSize).join("\n"), lineNumber: 1 };
261 }
262
263 const lineScores = lines.map((line) => {
264 const lower = line.toLowerCase();
265 return tokens.reduce((acc, tok) => acc + (lower.includes(tok) ? 1 : 0), 0);
266 });
267
268 // Sliding window: find the windowSize-line slice with the highest score sum.
269 let windowScore = lineScores.slice(0, windowSize).reduce((a, b) => a + b, 0);
270 let bestStart = 0;
271 let bestScore = windowScore;
272
273 for (let i = 1; i + windowSize <= lines.length; i++) {
274 windowScore = windowScore - lineScores[i - 1] + lineScores[i + windowSize - 1];
275 if (windowScore > bestScore) {
276 bestScore = windowScore;
277 bestStart = i;
278 }
279 }
280
281 const snippet = lines.slice(bestStart, bestStart + windowSize).join("\n");
282 return { snippet, lineNumber: bestStart + 1 };
283}
284
285// ---------------------------------------------------------------------------
286// Claude API call
287// ---------------------------------------------------------------------------
288
289interface ClaudeRankedFile {
290 file: string;
291 reason: string;
292 confidence: number;
293}
294
295const MAX_INDEX_CHARS = 80_000; // cap total index size sent to Claude
296
297async function rankFilesWithClaude(
298 query: string,
299 fileIndex: Array<{ path: string; head: string }>
300): Promise<ClaudeRankedFile[]> {
301 const client = getAnthropic();
302
303 // Build a compact text index, respecting the char cap.
304 let indexText = "";
305 for (const { path, head } of fileIndex) {
306 const entry = `\n--- ${path} ---\n${head.slice(0, 600)}\n`;
307 if (indexText.length + entry.length > MAX_INDEX_CHARS) break;
308 indexText += entry;
309 }
310
311 const prompt = `You are a code navigation assistant. Given a codebase file index and a search query, identify the most relevant files.
312
313QUERY: "${query}"
314
315CODEBASE INDEX (filename + first 50 lines per file):
316${indexText}
317
318Return ONLY a JSON array (no prose, no markdown) of up to 8 objects, ranked by relevance, with this shape:
319[{"file": "<path>", "reason": "<1-sentence explanation>", "confidence": <0.0–1.0>}]
320
321Only include files with confidence > 0.2. Return [] if nothing is relevant.`;
322
323 try {
324 const message = await client.messages.create({
a5705cfClaude325 model: MODEL_SONNET,
53c9249Claude326 max_tokens: 1000,
327 messages: [{ role: "user", content: prompt }],
328 });
329
330 const text =
331 message.content.find((b) => b.type === "text")?.text ?? "";
332 const parsed = parseJsonResponse<ClaudeRankedFile[]>(text);
333 if (!Array.isArray(parsed)) return [];
334 return parsed.filter(
335 (r): r is ClaudeRankedFile =>
336 typeof r.file === "string" &&
337 typeof r.reason === "string" &&
338 typeof r.confidence === "number"
339 );
340 } catch (err) {
341 console.error("[claude-semantic-search] Claude API error:", err);
342 return [];
343 }
344}
345
346// ---------------------------------------------------------------------------
347// Keyword fallback (git grep)
348// ---------------------------------------------------------------------------
349
350async function keywordFallback(
351 owner: string,
352 repo: string,
353 branch: string,
354 query: string
355): Promise<SemanticSearchResult[]> {
356 const repoDir = getRepoPath(owner, repo);
357 const keyword = query.split(/\s+/)[0] || query;
358 const { stdout, exitCode } = await gitExec(
359 ["git", "grep", "-n", "-i", "-m", "5", keyword, branch, "--"],
360 repoDir
361 );
362 if (exitCode !== 0) return [];
363
364 // Group by file, take top 5 files.
365 const byFile = new Map<string, { lineNum: number; line: string }[]>();
366 for (const raw of stdout.trim().split("\n").filter(Boolean)) {
367 // Format: <ref>:<file>:<lineNum>:<content>
368 const refPrefix = branch + ":";
369 const stripped = raw.startsWith(refPrefix) ? raw.slice(refPrefix.length) : raw;
370 const firstColon = stripped.indexOf(":");
371 if (firstColon < 0) continue;
372 const file = stripped.slice(0, firstColon);
373 const rest = stripped.slice(firstColon + 1);
374 const secondColon = rest.indexOf(":");
375 if (secondColon < 0) continue;
376 const lineNum = parseInt(rest.slice(0, secondColon), 10);
377 const line = rest.slice(secondColon + 1);
378 if (!byFile.has(file)) byFile.set(file, []);
379 byFile.get(file)!.push({ lineNum, line });
380 }
381
382 const results: SemanticSearchResult[] = [];
383 let count = 0;
384 for (const [file, matches] of byFile) {
385 if (count++ >= 5) break;
386 const snippet = matches
387 .slice(0, 5)
388 .map((m) => `${m.lineNum}: ${m.line}`)
389 .join("\n");
390 results.push({
391 file,
392 reason: `Keyword match for "${keyword}"`,
393 confidence: 0.5,
394 snippet,
395 lineNumber: matches[0]?.lineNum,
396 });
397 }
398 return results;
399}
400
401// ---------------------------------------------------------------------------
402// Main export
403// ---------------------------------------------------------------------------
404
405export interface SemanticSearchOptions {
406 maxFiles?: number; // max candidate files to index (default 200)
407 branch?: string; // branch/ref to search (default "HEAD")
408 rateLimitKey?: string; // userId or IP for rate limiting
409}
410
411/**
412 * Semantic code search powered by Claude.
413 *
414 * @param owner - repo owner username
415 * @param repo - repo name
416 * @param repoId - DB repository id (used as cache key)
417 * @param query - natural-language search query
418 * @param opts - optional config
419 * @returns ranked list of matching files with AI reasoning
420 */
421export async function claudeSemanticSearch(
422 owner: string,
423 repo: string,
424 repoId: string,
425 query: string,
426 opts: SemanticSearchOptions = {}
427): Promise<{ results: SemanticSearchResult[]; mode: "semantic" | "keyword"; quotaExceeded: boolean }> {
428 const q = query.trim();
429 if (!q) return { results: [], mode: "keyword", quotaExceeded: false };
430
431 const branch = opts.branch || "HEAD";
432 const maxFiles = opts.maxFiles ?? 200;
433 const cacheKey = `${repoId}:${branch}:${q}`;
434
435 // Cache hit
436 const cached = cacheGet(cacheKey);
437 if (cached) {
438 return { results: cached, mode: "semantic", quotaExceeded: false };
439 }
440
441 // Rate limit check (per-user or per-IP)
442 let quotaExceeded = false;
443 if (opts.rateLimitKey) {
444 const within = checkSemanticRateLimit(opts.rateLimitKey);
445 if (!within) {
446 quotaExceeded = true;
447 }
448 }
449
450 // No Claude API key or quota exceeded → keyword fallback
451 if (!config.anthropicApiKey || quotaExceeded) {
452 const results = await keywordFallback(owner, repo, branch, q);
453 return { results, mode: "keyword", quotaExceeded };
454 }
455
456 // Step 1: list all files
457 let allFiles: string[];
458 try {
459 allFiles = await lsFiles(owner, repo, branch);
460 } catch {
461 return { results: await keywordFallback(owner, repo, branch, q), mode: "keyword", quotaExceeded: false };
462 }
463
464 // Step 2: pre-filter if repo is large
465 let candidates: string[];
466 if (allFiles.length > maxFiles) {
467 const keyword = q.split(/\s+/)[0] || q;
468 try {
469 const grepHits = new Set(await grepFiles(owner, repo, branch, keyword));
470 candidates = allFiles.filter((f) => grepHits.has(f));
471 // If grep found nothing, fall back to first maxFiles files.
472 if (candidates.length === 0) {
473 candidates = allFiles.slice(0, maxFiles);
474 } else if (candidates.length > maxFiles) {
475 candidates = candidates.slice(0, maxFiles);
476 }
477 } catch {
478 candidates = allFiles.slice(0, maxFiles);
479 }
480 } else {
481 candidates = allFiles;
482 }
483
484 // Step 3: build file index (filename + first 50 lines)
485 const fileIndex: Array<{ path: string; head: string }> = [];
486 const HEAD_CONCURRENCY = 20;
487 for (let i = 0; i < candidates.length; i += HEAD_CONCURRENCY) {
488 const batch = candidates.slice(i, i + HEAD_CONCURRENCY);
489 const heads = await Promise.all(
490 batch.map((p) => readFileHead(owner, repo, branch, p, 50))
491 );
492 for (let j = 0; j < batch.length; j++) {
493 if (heads[j]) fileIndex.push({ path: batch[j], head: heads[j] });
494 }
495 }
496
497 if (fileIndex.length === 0) {
498 return { results: await keywordFallback(owner, repo, branch, q), mode: "keyword", quotaExceeded: false };
499 }
500
501 // Step 4: ask Claude to rank files
502 const ranked = await rankFilesWithClaude(q, fileIndex);
503
504 if (ranked.length === 0) {
505 // Claude found nothing — try keyword fallback
506 const kwResults = await keywordFallback(owner, repo, branch, q);
507 return { results: kwResults, mode: "keyword", quotaExceeded: false };
508 }
509
510 // Step 5: for each top result, read full file and extract relevant snippet
511 const TOP_N = 5;
512 const top = ranked.slice(0, TOP_N);
513 const results: SemanticSearchResult[] = [];
514
515 await Promise.all(
516 top.map(async (r) => {
517 const content = await readFileFull(owner, repo, branch, r.file).catch(() => "");
518 if (!content) {
519 results.push({ ...r, snippet: "", lineNumber: 1 });
520 return;
521 }
522 const { snippet, lineNumber } = extractSnippet(content, q, 20);
523 results.push({
524 file: r.file,
525 reason: r.reason,
526 confidence: Math.min(1, Math.max(0, r.confidence)),
527 snippet,
528 lineNumber,
529 });
530 })
531 );
532
533 // Sort by confidence desc (Promise.all doesn't preserve order after push).
534 results.sort((a, b) => b.confidence - a.confidence);
535
536 cacheSet(cacheKey, results);
537 return { results, mode: "semantic", quotaExceeded: false };
538}