Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

nl-search.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/nl-search.ts
nl-search.tsBlame516 lines · 1 contributor
Normal viewRaw
77cf834Claude1/**
2 * Natural Language Code Search — Claude-powered intent search.
3 *
4 * Unlike embedding-based semantic search (which requires a pre-built vector
5 * index), NL search uses Claude as the reasoner over actual file content.
6 * A developer types a natural language question like:
7 * "find all places where we validate user email but don't check MX records"
8 * "where do we write to the database without a transaction?"
9 * and Claude finds the matching code.
10 *
11 * Algorithm:
12 * 1. Quick Claude call to extract grep-friendly keywords from the query.
13 * 2. Run `git grep -l <keyword>` for each keyword; union matching files.
14 * Cap at 40 files. Fall back to code_chunks table if grep returns 0.
15 * 3. Read each candidate file via git show (HEAD:<path>), capped at 6KB each.
16 * Build combined context, capped at 80KB total.
17 * 4. Single Claude reasoning pass: find all places matching the query,
18 * return JSON with filePath, lineStart, lineEnd, snippet, explanation,
19 * confidence.
20 * 5. Cache results in-memory per `${repoId}:${query}` for 15 minutes.
21 */
22
23import { getAnthropic, MODEL_SONNET, isAiAvailable, parseJsonResponse } from "./ai-client";
24import { getRepoPath } from "../git/repository";
25import { db } from "../db";
26import { codeChunks } from "../db/schema";
27import { eq } from "drizzle-orm";
28
29// ---------------------------------------------------------------------------
30// Public types
31// ---------------------------------------------------------------------------
32
33export interface NlSearchResult {
34 filePath: string;
35 lineStart: number;
36 lineEnd: number;
37 snippet: string; // the relevant lines
38 explanation: string; // why this matches the query
39 confidence: "high" | "medium" | "low";
40}
41
42export interface NlSearchResponse {
43 query: string;
44 results: NlSearchResult[];
45 totalFilesScanned: number;
46 searchedAt: Date;
47 cached: boolean;
48}
49
50// ---------------------------------------------------------------------------
51// In-memory cache (15-minute TTL)
52// ---------------------------------------------------------------------------
53
54interface CacheEntry {
55 response: NlSearchResponse;
56 expiresAt: number;
57}
58
59const nlCache = new Map<string, CacheEntry>();
60const NL_CACHE_TTL_MS = 15 * 60 * 1000; // 15 minutes
61
62function cacheGet(key: string): NlSearchResponse | null {
63 const entry = nlCache.get(key);
64 if (!entry) return null;
65 if (Date.now() > entry.expiresAt) {
66 nlCache.delete(key);
67 return null;
68 }
69 return entry.response;
70}
71
72function cacheSet(key: string, response: NlSearchResponse): void {
73 // Evict expired entries to keep memory bounded.
74 if (nlCache.size > 200) {
75 const now = Date.now();
76 for (const [k, v] of nlCache) {
77 if (v.expiresAt < now) nlCache.delete(k);
78 }
79 }
80 nlCache.set(key, { response, expiresAt: Date.now() + NL_CACHE_TTL_MS });
81}
82
83// ---------------------------------------------------------------------------
84// Skip-list: skip binary/build/lock files
85// ---------------------------------------------------------------------------
86
87const SKIP_DIRS = new Set([
88 "node_modules",
89 ".git",
90 "dist",
91 "build",
92 "vendor",
93 ".next",
94 ".turbo",
95 "target",
96 "__pycache__",
97]);
98
99const SKIP_EXTS = new Set([
100 ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp",
101 ".woff", ".woff2", ".ttf", ".eot",
102 ".pdf", ".zip", ".gz", ".tar",
103 ".lockb", ".lock",
104 ".min.js", ".min.css",
105]);
106
107function shouldSkipPath(filePath: string): boolean {
108 const parts = filePath.split("/");
109 for (const part of parts.slice(0, -1)) {
110 if (SKIP_DIRS.has(part.toLowerCase())) return true;
111 }
112 const basename = parts[parts.length - 1].toLowerCase();
113 for (const ext of SKIP_EXTS) {
114 if (basename.endsWith(ext)) return true;
115 }
116 // Lock files by exact name
117 const lockFiles = new Set([
118 "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
119 "bun.lockb", "bun.lock", "poetry.lock", "cargo.lock",
120 "composer.lock", "gemfile.lock",
121 ]);
122 if (lockFiles.has(basename)) return true;
123 return false;
124}
125
126// ---------------------------------------------------------------------------
127// Git helpers
128// ---------------------------------------------------------------------------
129
130async function gitExec(
131 cmd: string[],
132 cwd: string
133): Promise<{ stdout: string; exitCode: number }> {
134 const proc = Bun.spawn(cmd, {
135 cwd,
136 env: process.env as Record<string, string>,
137 stdout: "pipe",
138 stderr: "pipe",
139 });
140 const stdout = await new Response(proc.stdout).text();
141 const exitCode = await proc.exited;
142 return { stdout, exitCode };
143}
144
145/**
146 * Run `git grep -l <keyword> HEAD --` and return matching file paths.
147 */
148async function grepForKeyword(
149 ownerName: string,
150 repoName: string,
151 keyword: string
152): Promise<string[]> {
153 const repoDir = getRepoPath(ownerName, repoName);
154 try {
155 const { stdout, exitCode } = await gitExec(
156 ["git", "grep", "-l", "-i", keyword, "HEAD", "--"],
157 repoDir
158 );
159 // exit code 1 = no matches (not an error)
160 if (exitCode !== 0 && exitCode !== 1) return [];
161 return stdout
162 .trim()
163 .split("\n")
164 .filter(Boolean)
165 .map((line) => {
166 // git grep -l with a ref outputs "<ref>:<path>" or just "<path>"
167 const idx = line.indexOf(":");
168 return idx >= 0 ? line.slice(idx + 1) : line;
169 })
170 .filter((p) => !shouldSkipPath(p));
171 } catch {
172 return [];
173 }
174}
175
176/**
177 * Read a file from the repo at HEAD, capped to maxBytes.
178 * Returns empty string on error.
179 */
180async function readFileFromGit(
181 ownerName: string,
182 repoName: string,
183 filePath: string,
184 maxBytes = 6 * 1024
185): Promise<string> {
186 const repoDir = getRepoPath(ownerName, repoName);
187 try {
188 const { stdout, exitCode } = await gitExec(
189 ["git", "show", `HEAD:${filePath}`],
190 repoDir
191 );
192 if (exitCode !== 0) return "";
193 return stdout.slice(0, maxBytes);
194 } catch {
195 return "";
196 }
197}
198
199// ---------------------------------------------------------------------------
200// Step 1 — Extract keywords from the query via Claude
201// ---------------------------------------------------------------------------
202
203interface KeywordExtraction {
204 keywords: string[];
205 fileTypes: string[];
206}
207
208async function extractKeywords(query: string): Promise<KeywordExtraction> {
209 const client = getAnthropic();
210 try {
211 const msg = await client.messages.create({
212 model: MODEL_SONNET,
213 max_tokens: 256,
214 messages: [
215 {
216 role: "user",
217 content:
218 `Extract 3-5 grep-friendly keywords from this natural language search query. ` +
219 `Return JSON only, no prose: {"keywords": string[], "fileTypes": string[]}\n` +
220 `Keywords should be short, concrete identifiers/patterns likely to appear in code. ` +
221 `fileTypes is an optional list of file extensions (e.g. [".ts", ".tsx"]) to narrow the search. ` +
222 `Return [] for fileTypes if the query is language-agnostic.\n\n` +
223 `Query: ${query}`,
224 },
225 ],
226 });
227 const text = msg.content.find((b) => b.type === "text")?.text ?? "";
228 const parsed = parseJsonResponse<KeywordExtraction>(text);
229 if (parsed && Array.isArray(parsed.keywords)) {
230 return {
231 keywords: parsed.keywords.slice(0, 5).filter((k) => typeof k === "string" && k.length > 0),
232 fileTypes: Array.isArray(parsed.fileTypes) ? parsed.fileTypes : [],
233 };
234 }
235 } catch (err) {
236 console.error("[nl-search] keyword extraction error:", err);
237 }
238 // Fallback: split query into words
239 const words = query
240 .split(/[^a-zA-Z0-9_]+/)
241 .filter((w) => w.length >= 3)
242 .slice(0, 5);
243 return { keywords: words, fileTypes: [] };
244}
245
246// ---------------------------------------------------------------------------
247// Step 2 — Gather candidate files via git grep
248// ---------------------------------------------------------------------------
249
250async function gatherCandidates(
251 ownerName: string,
252 repoName: string,
253 repoId: string,
254 extraction: KeywordExtraction,
255 maxFiles = 40
256): Promise<string[]> {
257 const seen = new Set<string>();
258
259 // Run git grep for each keyword (in parallel)
260 const results = await Promise.allSettled(
261 extraction.keywords.map((kw) => grepForKeyword(ownerName, repoName, kw))
262 );
263
264 for (const r of results) {
265 if (r.status === "fulfilled") {
266 for (const p of r.value) {
267 seen.add(p);
268 if (seen.size >= maxFiles * 2) break;
269 }
270 }
271 }
272
273 // Apply file-type filter if provided
274 let candidates = Array.from(seen);
275 if (extraction.fileTypes.length > 0) {
276 const filtered = candidates.filter((p) =>
277 extraction.fileTypes.some((ext) => p.endsWith(ext))
278 );
279 // Only narrow if we still have results
280 if (filtered.length > 0) candidates = filtered;
281 }
282
283 candidates = candidates.slice(0, maxFiles);
284
285 // Fallback: if git grep returned nothing, read from code_chunks table
286 if (candidates.length === 0) {
287 try {
288 const rows = await db
289 .select({ path: codeChunks.path })
290 .from(codeChunks)
291 .where(eq(codeChunks.repositoryId, repoId))
292 .groupBy(codeChunks.path)
293 .limit(maxFiles);
294 candidates = rows.map((r) => r.path).filter((p) => !shouldSkipPath(p));
295 } catch {
296 // DB unavailable — return empty
297 }
298 }
299
300 return candidates;
301}
302
303// ---------------------------------------------------------------------------
304// Step 3 — Read file contents and build context
305// ---------------------------------------------------------------------------
306
307/**
308 * For each candidate file, read content (capped at 6KB).
309 * Build a combined context string where each file is prefixed with a header
310 * showing the filename and line numbers, capped at 80KB total.
311 */
312async function buildContext(
313 ownerName: string,
314 repoName: string,
315 candidates: string[],
316 maxTotalBytes = 80 * 1024
317): Promise<{ contextStr: string; filesRead: string[] }> {
318 const BATCH = 10;
319 const fileContents: Array<{ path: string; content: string }> = [];
320
321 for (let i = 0; i < candidates.length; i += BATCH) {
322 const batch = candidates.slice(i, i + BATCH);
323 const contents = await Promise.all(
324 batch.map((p) => readFileFromGit(ownerName, repoName, p, 6144))
325 );
326 for (let j = 0; j < batch.length; j++) {
327 if (contents[j]) {
328 fileContents.push({ path: batch[j], content: contents[j] });
329 }
330 }
331 }
332
333 // Build numbered context string
334 let contextStr = "";
335 const filesRead: string[] = [];
336
337 for (const { path, content } of fileContents) {
338 if (contextStr.length >= maxTotalBytes) break;
339 const lines = content.split("\n");
340 // Number lines starting at 1
341 const numbered = lines
342 .map((line, idx) => `${idx + 1}: ${line}`)
343 .join("\n");
344 const header = `\n\n=== FILE: ${path} ===\n`;
345 const block = header + numbered;
346 if (contextStr.length + block.length > maxTotalBytes) {
347 // Trim block to fit
348 const remaining = maxTotalBytes - contextStr.length;
349 if (remaining > header.length + 100) {
350 contextStr += block.slice(0, remaining);
351 filesRead.push(path);
352 }
353 } else {
354 contextStr += block;
355 filesRead.push(path);
356 }
357 }
358
359 return { contextStr, filesRead };
360}
361
362// ---------------------------------------------------------------------------
363// Step 4 — Claude reasoning pass
364// ---------------------------------------------------------------------------
365
366interface RawResult {
367 filePath: string;
368 lineStart: number;
369 lineEnd: number;
370 snippet: string;
371 explanation: string;
372 confidence: "high" | "medium" | "low";
373}
374
375async function reasonWithClaude(
376 query: string,
377 contextStr: string
378): Promise<NlSearchResult[]> {
379 const client = getAnthropic();
380
381 const systemPrompt =
382 `You are a code analysis expert. Find all places in the provided code that match the user's query. ` +
383 `Be precise about file paths and line numbers. Only return matches that genuinely satisfy the query — ` +
384 `do not include tangentially related code.`;
385
386 const userPrompt =
387 `Query: ${query}\n\n` +
388 `Code files:\n${contextStr}\n\n` +
389 `Return JSON only, no prose:\n` +
390 `{\n` +
391 ` "results": Array<{\n` +
392 ` "filePath": string,\n` +
393 ` "lineStart": number,\n` +
394 ` "lineEnd": number,\n` +
395 ` "snippet": string,\n` +
396 ` "explanation": string,\n` +
397 ` "confidence": "high" | "medium" | "low"\n` +
398 ` }>\n` +
399 `}\n\n` +
400 `Return {"results": []} if nothing matches. Sort by confidence descending. Max 10 results.`;
401
402 try {
403 const msg = await client.messages.create({
404 model: MODEL_SONNET,
405 max_tokens: 3000,
406 system: systemPrompt,
407 messages: [{ role: "user", content: userPrompt }],
408 });
409
410 const text = msg.content.find((b) => b.type === "text")?.text ?? "";
411 const parsed = parseJsonResponse<{ results: RawResult[] }>(text);
412
413 if (!parsed || !Array.isArray(parsed.results)) return [];
414
415 return parsed.results
416 .filter(
417 (r): r is RawResult =>
418 typeof r.filePath === "string" &&
419 typeof r.lineStart === "number" &&
420 typeof r.lineEnd === "number" &&
421 typeof r.snippet === "string" &&
422 typeof r.explanation === "string" &&
423 (r.confidence === "high" || r.confidence === "medium" || r.confidence === "low")
424 )
425 .slice(0, 10);
426 } catch (err) {
427 console.error("[nl-search] Claude reasoning error:", err);
428 return [];
429 }
430}
431
432// ---------------------------------------------------------------------------
433// Main export
434// ---------------------------------------------------------------------------
435
436/**
437 * Natural language code search powered by Claude.
438 *
439 * @param ownerName - repo owner username
440 * @param repoName - repo name
441 * @param repoId - DB repository id (used as cache key + fallback)
442 * @param query - natural-language search query
443 * @returns NlSearchResponse (never throws)
444 */
445export async function nlSearch(
446 ownerName: string,
447 repoName: string,
448 repoId: string,
449 query: string
450): Promise<NlSearchResponse> {
451 const q = query.trim();
452 const empty: NlSearchResponse = {
453 query: q,
454 results: [],
455 totalFilesScanned: 0,
456 searchedAt: new Date(),
457 cached: false,
458 };
459
460 if (!q) return empty;
461
462 // Guard: AI must be available
463 if (!isAiAvailable()) {
464 return { ...empty, results: [] };
465 }
466
467 // Cache check
468 const cacheKey = `${repoId}:${q}`;
469 const cached = cacheGet(cacheKey);
470 if (cached) {
471 return { ...cached, cached: true };
472 }
473
474 try {
475 // Step 1 — Extract keywords
476 const extraction = await extractKeywords(q);
477
478 // Step 2 — Gather candidate files
479 const candidates = await gatherCandidates(
480 ownerName, repoName, repoId, extraction, 40
481 );
482
483 if (candidates.length === 0) {
484 cacheSet(cacheKey, empty);
485 return empty;
486 }
487
488 // Step 3 — Read file contents
489 const { contextStr, filesRead } = await buildContext(
490 ownerName, repoName, candidates, 80 * 1024
491 );
492
493 if (!contextStr || filesRead.length === 0) {
494 cacheSet(cacheKey, empty);
495 return empty;
496 }
497
498 // Step 4 — Claude reasoning pass
499 const results = await reasonWithClaude(q, contextStr);
500
501 const response: NlSearchResponse = {
502 query: q,
503 results,
504 totalFilesScanned: filesRead.length,
505 searchedAt: new Date(),
506 cached: false,
507 };
508
509 // Step 5 — Cache and return
510 cacheSet(cacheKey, response);
511 return response;
512 } catch (err) {
513 console.error("[nl-search] unexpected error:", err);
514 return empty;
515 }
516}