CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history
symbols.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 4c8f666 | 1 | /** |
| 2 | * Block I8 — Symbol / xref navigation. | |
| 3 | * | |
| 4 | * A pragmatic regex-based top-level symbol extractor. Runs per-language, | |
| 5 | * catches the common definition shapes (function / class / interface / | |
| 6 | * type / const). References are computed at lookup-time by grepping the | |
| 7 | * repository's tree for the symbol name, so this module persists only | |
| 8 | * definitions. Never throws into request path. | |
| 9 | */ | |
| 10 | ||
| 11 | import { and, eq } from "drizzle-orm"; | |
| 12 | import { db } from "../db"; | |
| 13 | import { codeSymbols, repositories, users } from "../db/schema"; | |
| 14 | import { getBlob, getTree, getDefaultBranch, resolveRef } from "../git/repository"; | |
| 15 | import type { CodeSymbol } from "../db/schema"; | |
| 16 | import type { GitTreeEntry } from "../git/repository"; | |
| 17 | ||
| 18 | export type SymbolKind = | |
| 19 | | "function" | |
| 20 | | "class" | |
| 21 | | "interface" | |
| 22 | | "type" | |
| 23 | | "const" | |
| 24 | | "variable"; | |
| 25 | ||
| 26 | export interface ExtractedSymbol { | |
| 27 | name: string; | |
| 28 | kind: SymbolKind; | |
| 29 | line: number; | |
| 30 | signature: string; | |
| 31 | } | |
| 32 | ||
| 33 | type Rule = { kind: SymbolKind; re: RegExp }; | |
| 34 | ||
| 35 | // ---------- Language detection ---------- | |
| 36 | ||
| 37 | const EXT_LANG: Record<string, string> = { | |
| 38 | ts: "ts", | |
| 39 | tsx: "ts", | |
| 40 | js: "ts", | |
| 41 | jsx: "ts", | |
| 42 | mjs: "ts", | |
| 43 | cjs: "ts", | |
| 44 | py: "py", | |
| 45 | rs: "rs", | |
| 46 | go: "go", | |
| 47 | rb: "rb", | |
| 48 | java: "java", | |
| 49 | kt: "kt", | |
| 50 | swift: "swift", | |
| 51 | }; | |
| 52 | ||
| 53 | export function detectLanguage(path: string): string | null { | |
| 54 | const ext = path.split(".").pop()?.toLowerCase() || ""; | |
| 55 | return EXT_LANG[ext] ?? null; | |
| 56 | } | |
| 57 | ||
| 58 | // ---------- Per-language rules ---------- | |
| 59 | ||
| 60 | const RULES: Record<string, Rule[]> = { | |
| 61 | ts: [ | |
| 62 | { | |
| 63 | kind: "function", | |
| 64 | re: /^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)/, | |
| 65 | }, | |
| 66 | { | |
| 67 | kind: "function", | |
| 68 | re: /^\s*(?:export\s+)?const\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\(/, | |
| 69 | }, | |
| 70 | { | |
| 71 | kind: "function", | |
| 72 | re: /^\s*(?:export\s+)?const\s+([A-Za-z_$][\w$]*)\s*:\s*[^=]+=\s*(?:async\s*)?\(/, | |
| 73 | }, | |
| 74 | { | |
| 75 | kind: "class", | |
| 76 | re: /^\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)/, | |
| 77 | }, | |
| 78 | { | |
| 79 | kind: "interface", | |
| 80 | re: /^\s*(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)/, | |
| 81 | }, | |
| 82 | { | |
| 83 | kind: "type", | |
| 84 | re: /^\s*(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\s*=/, | |
| 85 | }, | |
| 86 | { | |
| 87 | kind: "const", | |
| 88 | re: /^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*[:=]/, | |
| 89 | }, | |
| 90 | ], | |
| 91 | py: [ | |
| 92 | { kind: "function", re: /^\s*(?:async\s+)?def\s+([A-Za-z_][\w]*)\s*\(/ }, | |
| 93 | { kind: "class", re: /^\s*class\s+([A-Za-z_][\w]*)\s*[:(]/ }, | |
| 94 | { kind: "const", re: /^([A-Z_][A-Z0-9_]*)\s*=/ }, | |
| 95 | ], | |
| 96 | rs: [ | |
| 97 | { kind: "function", re: /^\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+([A-Za-z_][\w]*)/ }, | |
| 98 | { kind: "class", re: /^\s*(?:pub(?:\([^)]*\))?\s+)?struct\s+([A-Za-z_][\w]*)/ }, | |
| 99 | { kind: "interface", re: /^\s*(?:pub(?:\([^)]*\))?\s+)?trait\s+([A-Za-z_][\w]*)/ }, | |
| 100 | { kind: "type", re: /^\s*(?:pub(?:\([^)]*\))?\s+)?type\s+([A-Za-z_][\w]*)\s*=/ }, | |
| 101 | { kind: "const", re: /^\s*(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z_][A-Z0-9_]*)\s*:/ }, | |
| 102 | ], | |
| 103 | go: [ | |
| 104 | { kind: "function", re: /^\s*func(?:\s+\([^)]*\))?\s+([A-Za-z_][\w]*)\s*\(/ }, | |
| 105 | { kind: "class", re: /^\s*type\s+([A-Za-z_][\w]*)\s+struct\b/ }, | |
| 106 | { kind: "interface", re: /^\s*type\s+([A-Za-z_][\w]*)\s+interface\b/ }, | |
| 107 | { kind: "type", re: /^\s*type\s+([A-Za-z_][\w]*)\s+\w/ }, | |
| 108 | { kind: "const", re: /^\s*const\s+([A-Za-z_][\w]*)\s*=/ }, | |
| 109 | ], | |
| 110 | rb: [ | |
| 111 | { kind: "function", re: /^\s*def\s+(?:self\.)?([A-Za-z_][\w?!=]*)/ }, | |
| 112 | { kind: "class", re: /^\s*class\s+([A-Z][\w]*)/ }, | |
| 113 | ], | |
| 114 | java: [ | |
| 115 | { | |
| 116 | kind: "class", | |
| 117 | re: /^\s*(?:public|private|protected)?\s*(?:abstract\s+|final\s+)?class\s+([A-Z][\w]*)/, | |
| 118 | }, | |
| 119 | { | |
| 120 | kind: "interface", | |
| 121 | re: /^\s*(?:public|private|protected)?\s*interface\s+([A-Z][\w]*)/, | |
| 122 | }, | |
| 123 | ], | |
| 124 | kt: [ | |
| 125 | { kind: "function", re: /^\s*(?:public|private|internal)?\s*fun\s+([A-Za-z_][\w]*)/ }, | |
| 126 | { kind: "class", re: /^\s*(?:public|private|internal)?\s*class\s+([A-Z][\w]*)/ }, | |
| 127 | ], | |
| 128 | swift: [ | |
| 129 | { kind: "function", re: /^\s*(?:public|private|fileprivate|internal)?\s*func\s+([A-Za-z_][\w]*)/ }, | |
| 130 | { kind: "class", re: /^\s*(?:public|private|fileprivate|internal)?\s*class\s+([A-Z][\w]*)/ }, | |
| 131 | { kind: "interface", re: /^\s*(?:public|private|fileprivate|internal)?\s*protocol\s+([A-Z][\w]*)/ }, | |
| 132 | ], | |
| 133 | }; | |
| 134 | ||
| 135 | // ---------- Extractor ---------- | |
| 136 | ||
| 137 | /** Pure — extract top-level symbol definitions from a single file. */ | |
| 138 | export function extractSymbols( | |
| 139 | content: string, | |
| 140 | lang: string | |
| 141 | ): ExtractedSymbol[] { | |
| 142 | const rules = RULES[lang]; | |
| 143 | if (!rules) return []; | |
| 144 | const out: ExtractedSymbol[] = []; | |
| 145 | const seen = new Set<string>(); | |
| 146 | const lines = content.split("\n"); | |
| 147 | for (let i = 0; i < lines.length; i++) { | |
| 148 | const line = lines[i]; | |
| 149 | if (line.length > 500) continue; // skip minified lines | |
| 150 | for (const rule of rules) { | |
| 151 | const m = line.match(rule.re); | |
| 152 | if (m && m[1]) { | |
| 153 | const key = `${rule.kind}:${m[1]}:${i}`; | |
| 154 | if (seen.has(key)) continue; | |
| 155 | seen.add(key); | |
| 156 | out.push({ | |
| 157 | name: m[1], | |
| 158 | kind: rule.kind, | |
| 159 | line: i + 1, | |
| 160 | signature: line.trim().slice(0, 240), | |
| 161 | }); | |
| 162 | break; // one match per line | |
| 163 | } | |
| 164 | } | |
| 165 | } | |
| 166 | return out; | |
| 167 | } | |
| 168 | ||
| 169 | // ---------- Indexer ---------- | |
| 170 | ||
| 171 | const INDEXABLE_MAX_BYTES = 1_000_000; // skip files over 1MB | |
| 172 | const MAX_FILES = 2_000; // cap per reindex | |
| 173 | ||
| 174 | async function walkCodePaths( | |
| 175 | owner: string, | |
| 176 | repo: string, | |
| 177 | ref: string, | |
| 178 | maxFiles = MAX_FILES | |
| 179 | ): Promise<Array<{ path: string; size?: number }>> { | |
| 180 | const out: Array<{ path: string; size?: number }> = []; | |
| 181 | const queue: string[] = [""]; | |
| 182 | while (queue.length && out.length < maxFiles) { | |
| 183 | const dir = queue.shift()!; | |
| 184 | let entries: GitTreeEntry[] = []; | |
| 185 | try { | |
| 186 | entries = await getTree(owner, repo, ref, dir); | |
| 187 | } catch { | |
| 188 | continue; | |
| 189 | } | |
| 190 | for (const e of entries) { | |
| 191 | const p = dir ? `${dir}/${e.name}` : e.name; | |
| 192 | if (e.type === "tree") { | |
| 193 | const base = e.name.toLowerCase(); | |
| 194 | if ( | |
| 195 | base === "node_modules" || | |
| 196 | base === ".git" || | |
| 197 | base === "dist" || | |
| 198 | base === "build" || | |
| 199 | base === "vendor" || | |
| 200 | base === ".next" || | |
| 201 | base === ".turbo" || | |
| 202 | base === "target" || | |
| 203 | base === "__pycache__" | |
| 204 | ) { | |
| 205 | continue; | |
| 206 | } | |
| 207 | queue.push(p); | |
| 208 | } else if (e.type === "blob") { | |
| 209 | if (!detectLanguage(p)) continue; | |
| 210 | if (e.size !== undefined && e.size > INDEXABLE_MAX_BYTES) continue; | |
| 211 | out.push({ path: p, size: e.size }); | |
| 212 | if (out.length >= maxFiles) break; | |
| 213 | } | |
| 214 | } | |
| 215 | } | |
| 216 | return out; | |
| 217 | } | |
| 218 | ||
| 219 | /** Walks the repo tree at HEAD, extracts symbols, replaces the prior set. */ | |
| 220 | export async function indexRepositorySymbols( | |
| 221 | repositoryId: string | |
| 222 | ): Promise<{ indexed: number; files: number; commitSha: string } | null> { | |
| 223 | try { | |
| 224 | const [repo] = await db | |
| 225 | .select() | |
| 226 | .from(repositories) | |
| 227 | .where(eq(repositories.id, repositoryId)) | |
| 228 | .limit(1); | |
| 229 | if (!repo) return null; | |
| 230 | ||
| 231 | const [owner] = await db | |
| 232 | .select({ username: users.username }) | |
| 233 | .from(users) | |
| 234 | .where(eq(users.id, repo.ownerId)) | |
| 235 | .limit(1); | |
| 236 | if (!owner) return null; | |
| 237 | ||
| 238 | const defaultBranch = | |
| 239 | (await getDefaultBranch(owner.username, repo.name)) || "main"; | |
| 240 | const head = await resolveRef(owner.username, repo.name, defaultBranch); | |
| 241 | if (!head) return null; | |
| 242 | ||
| 243 | const files = await walkCodePaths(owner.username, repo.name, head); | |
| 244 | ||
| 245 | const rows: Array<Omit<CodeSymbol, "id" | "createdAt">> = []; | |
| 246 | let processed = 0; | |
| 247 | ||
| 248 | for (const f of files) { | |
| 249 | const lang = detectLanguage(f.path); | |
| 250 | if (!lang) continue; | |
| 251 | try { | |
| 252 | const blob = await getBlob(owner.username, repo.name, head, f.path); | |
| 253 | if (!blob || blob.isBinary) continue; | |
| 254 | const syms = extractSymbols(blob.content, lang); | |
| 255 | for (const s of syms) { | |
| 256 | rows.push({ | |
| 257 | repositoryId: repo.id, | |
| 258 | commitSha: head, | |
| 259 | name: s.name, | |
| 260 | kind: s.kind, | |
| 261 | path: f.path, | |
| 262 | line: s.line, | |
| 263 | signature: s.signature, | |
| 264 | }); | |
| 265 | } | |
| 266 | processed++; | |
| 267 | } catch { | |
| 268 | // skip unreadable files | |
| 269 | } | |
| 270 | } | |
| 271 | ||
| 272 | // Replace the prior index (DELETE + batched INSERTs). | |
| 273 | await db.delete(codeSymbols).where(eq(codeSymbols.repositoryId, repo.id)); | |
| 274 | const BATCH = 500; | |
| 275 | for (let i = 0; i < rows.length; i += BATCH) { | |
| 276 | await db.insert(codeSymbols).values(rows.slice(i, i + BATCH)); | |
| 277 | } | |
| 278 | ||
| 279 | return { indexed: rows.length, files: processed, commitSha: head }; | |
| 280 | } catch (err) { | |
| 281 | console.error("[symbols] indexRepositorySymbols error:", err); | |
| 282 | return null; | |
| 283 | } | |
| 284 | } | |
| 285 | ||
| 286 | /** Find definitions of a symbol name within a repo. */ | |
| 287 | export async function findDefinitions( | |
| 288 | repositoryId: string, | |
| 289 | name: string | |
| 290 | ): Promise<CodeSymbol[]> { | |
| 291 | try { | |
| 292 | return await db | |
| 293 | .select() | |
| 294 | .from(codeSymbols) | |
| 295 | .where( | |
| 296 | and(eq(codeSymbols.repositoryId, repositoryId), eq(codeSymbols.name, name)) | |
| 297 | ); | |
| 298 | } catch { | |
| 299 | return []; | |
| 300 | } | |
| 301 | } | |
| 302 | ||
| 303 | /** Count total indexed symbols for a repo (pagination helper). */ | |
| 304 | export async function countSymbolsForRepo( | |
| 305 | repositoryId: string | |
| 306 | ): Promise<number> { | |
| 307 | try { | |
| 308 | const rows = await db | |
| 309 | .select({ id: codeSymbols.id }) | |
| 310 | .from(codeSymbols) | |
| 311 | .where(eq(codeSymbols.repositoryId, repositoryId)); | |
| 312 | return rows.length; | |
| 313 | } catch { | |
| 314 | return 0; | |
| 315 | } | |
| 316 | } | |
| 317 | ||
| 318 | // Test-only hook | |
| 319 | export const __internal = { RULES, EXT_LANG }; |