Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

repo-explainer.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

repo-explainer.tsBlame672 lines · 1 contributor
0a69faaClaude1/**
2 * repo-explainer.ts
3 *
4 * Core AI analysis engine for the "Explain This Repo" feature.
5 * Reads the repo's file tree + key files, then calls the Anthropic API
6 * (or falls back to heuristic analysis) to produce a structured JSON result.
7 *
8 * Never throws — all errors produce a degraded-but-valid result.
9 */
10
11import { eq, and } from "drizzle-orm";
12import { db } from "../db";
13import { repoExplainCache, repositories, users } from "../db/schema";
14import { getBlob, getTree, getDefaultBranch, resolveRef } from "../git/repository";
15import type { GitTreeEntry } from "../git/repository";
16import { config } from "./config";
17
18// ---------------------------------------------------------------------------
19// Types
20// ---------------------------------------------------------------------------
21
22export interface EntryPoint {
23 file: string;
24 role: string;
25}
26
27export interface SuggestedIssue {
28 title: string;
29 description: string;
30}
31
32export interface ExplainJobResult {
33 summary: string;
34 techStack: string[];
35 architecture: string;
36 entryPoints: EntryPoint[];
37 gettingStarted: string;
38 healthScore: "Elite" | "Strong" | "Improving" | "Needs Attention";
39 suggestedIssues: SuggestedIssue[];
40}
41
42export interface ExplainJob {
43 id: string;
44 repoId: string;
45 owner: string;
46 repo: string;
47 status: "running" | "done" | "failed";
48 result?: ExplainJobResult;
49 error?: string;
50 createdAt: Date;
51 completedAt?: Date;
52}
53
54// ---------------------------------------------------------------------------
55// In-memory job store (per-process; good enough for a single-process Bun app)
56// ---------------------------------------------------------------------------
57
58export const explainJobs = new Map<string, ExplainJob>();
59
60// ---------------------------------------------------------------------------
61// File sampling config
62// ---------------------------------------------------------------------------
63
64const MAX_TOTAL_CHARS = 80_000;
65const MAX_FILES = 50;
66const MAX_FILE_SIZE = 8_000; // chars per file (2000 lines * ~40 chars/line avg)
67
68const PRIORITY_ROOT_FILES = [
69 "README.md",
70 "README",
71 "readme.md",
72 "Readme.md",
73 "README.rst",
74 "README.txt",
75 "CLAUDE.md",
76 "package.json",
77 "pyproject.toml",
78 "Cargo.toml",
79 "go.mod",
80 "Gemfile",
81 "pom.xml",
82 "build.gradle",
83 "Dockerfile",
84 "tsconfig.json",
85 "bun.lockb",
86 ".env.example",
87];
88
89const CANDIDATE_SRC_DIRS = ["src", "lib", "app", "server", "backend", "pkg", "cmd"];
90
91// ---------------------------------------------------------------------------
92// DB cache helpers
93// ---------------------------------------------------------------------------
94
95export async function getCachedExplainResult(
96 repoId: string
97): Promise<ExplainJobResult | null> {
98 try {
99 const [row] = await db
100 .select()
101 .from(repoExplainCache)
102 .where(eq(repoExplainCache.repoId, repoId))
103 .limit(1);
104 if (!row) return null;
105 return row.result as ExplainJobResult;
106 } catch {
107 return null;
108 }
109}
110
111async function upsertExplainCache(
112 repoId: string,
113 result: ExplainJobResult
114): Promise<void> {
115 try {
116 // Try update first
117 const existing = await db
118 .select({ id: repoExplainCache.id })
119 .from(repoExplainCache)
120 .where(eq(repoExplainCache.repoId, repoId))
121 .limit(1);
122 if (existing.length > 0) {
123 await db
124 .update(repoExplainCache)
125 .set({ result, createdAt: new Date() })
126 .where(eq(repoExplainCache.id, existing[0].id));
127 } else {
128 await db.insert(repoExplainCache).values({ repoId, result });
129 }
130 } catch {
131 // Swallow — cache miss is not a fatal error
132 }
133}
134
135// ---------------------------------------------------------------------------
136// Repo resolution helpers
137// ---------------------------------------------------------------------------
138
139export async function resolveRepoForExplain(
140 owner: string,
141 repoName: string
142): Promise<{ repoId: string; ownerId: string } | null> {
143 try {
144 const [ownerRow] = await db
145 .select({ id: users.id })
146 .from(users)
147 .where(eq(users.username, owner))
148 .limit(1);
149 if (!ownerRow) return null;
150
151 const [repoRow] = await db
152 .select({ id: repositories.id, ownerId: repositories.ownerId })
153 .from(repositories)
154 .where(and(eq(repositories.ownerId, ownerRow.id), eq(repositories.name, repoName)))
155 .limit(1);
156 if (!repoRow) return null;
157
158 return { repoId: repoRow.id, ownerId: repoRow.ownerId };
159 } catch {
160 return null;
161 }
162}
163
164// ---------------------------------------------------------------------------
165// Main entry point: kick off an explain job asynchronously
166// ---------------------------------------------------------------------------
167
168export function startExplainJob(
169 jobId: string,
170 owner: string,
171 repoName: string,
172 repoId: string
173): void {
174 const job: ExplainJob = {
175 id: jobId,
176 repoId,
177 owner,
178 repo: repoName,
179 status: "running",
180 createdAt: new Date(),
181 };
182 explainJobs.set(jobId, job);
183
184 // Fire-and-forget
185 runExplainJob(job, owner, repoName, repoId).catch(() => {
186 const j = explainJobs.get(jobId);
187 if (j) {
188 j.status = "failed";
189 j.error = "Unexpected error during analysis";
190 j.completedAt = new Date();
191 }
192 });
193}
194
195async function runExplainJob(
196 job: ExplainJob,
197 owner: string,
198 repoName: string,
199 repoId: string
200): Promise<void> {
201 try {
202 const result = await explainRepo(owner, repoName);
203 await upsertExplainCache(repoId, result);
204 job.result = result;
205 job.status = "done";
206 job.completedAt = new Date();
207 } catch (err) {
208 job.status = "failed";
209 job.error = err instanceof Error ? err.message : "Analysis failed";
210 job.completedAt = new Date();
211 }
212}
213
214// ---------------------------------------------------------------------------
215// Core analysis function
216// ---------------------------------------------------------------------------
217
218export async function explainRepo(
219 owner: string,
220 repoName: string
221): Promise<ExplainJobResult> {
222 const branch = await getDefaultBranch(owner, repoName);
223 if (!branch) {
224 return buildHeuristicResult(owner, repoName, null, []);
225 }
226
227 const sha = await resolveRef(owner, repoName, branch);
228 if (!sha) {
229 return buildHeuristicResult(owner, repoName, null, []);
230 }
231
232 const samples = await gatherRepresentativeFiles(owner, repoName, sha);
233
234 const apiKey = config.anthropicApiKey;
235 if (apiKey && samples.files.length > 0) {
236 try {
237 return await callAnthropicForStructuredResult(
238 owner,
239 repoName,
240 samples,
241 apiKey
242 );
243 } catch {
244 // Fall through to heuristic
245 }
246 }
247
248 return buildHeuristicResult(owner, repoName, samples.packageJson, samples.topLevelTree);
249}
250
251// ---------------------------------------------------------------------------
252// File gathering
253// ---------------------------------------------------------------------------
254
255interface SampledFile {
256 path: string;
257 content: string;
258}
259
260interface Samples {
261 files: SampledFile[];
262 topLevelTree: GitTreeEntry[];
263 packageJson: Record<string, unknown> | null;
264}
265
266async function gatherRepresentativeFiles(
267 owner: string,
268 repo: string,
269 sha: string
270): Promise<Samples> {
271 const out: SampledFile[] = [];
272 let totalChars = 0;
273 const seen = new Set<string>();
274
275 let root: GitTreeEntry[] = [];
276 try {
277 root = await getTree(owner, repo, sha, "");
278 } catch {
279 root = [];
280 }
281
282 async function tryAdd(path: string): Promise<void> {
283 if (out.length >= MAX_FILES) return;
284 if (totalChars >= MAX_TOTAL_CHARS) return;
285 if (seen.has(path)) return;
286 seen.add(path);
287 try {
288 const blob = await getBlob(owner, repo, sha, path);
289 if (!blob || blob.isBinary) return;
290 if (!blob.content) return;
291 const snippet = blob.content.slice(0, Math.min(MAX_FILE_SIZE, MAX_TOTAL_CHARS - totalChars));
292 if (!snippet) return;
293 out.push({ path, content: snippet });
294 totalChars += snippet.length;
295 } catch {
296 /* skip */
297 }
298 }
299
300 // 1. Priority root-level files
301 for (const name of PRIORITY_ROOT_FILES) {
302 if (root.find((e) => e.type === "blob" && e.name === name)) {
303 await tryAdd(name);
304 }
305 }
306
307 // 2. Other manifest/config files at root
308 for (const entry of root) {
309 if (out.length >= MAX_FILES) break;
310 if (entry.type !== "blob") continue;
311 if (seen.has(entry.name)) continue;
312 if (!looksLikeManifest(entry.name)) continue;
313 await tryAdd(entry.name);
314 }
315
316 // 3. Entry-point files in common src dirs
317 for (const dir of CANDIDATE_SRC_DIRS) {
318 if (out.length >= MAX_FILES) break;
319 if (totalChars >= MAX_TOTAL_CHARS) break;
320 const dirEntry = root.find((e) => e.type === "tree" && e.name === dir);
321 if (!dirEntry) continue;
322 let children: GitTreeEntry[] = [];
323 try {
324 children = await getTree(owner, repo, sha, dir);
325 } catch {
326 children = [];
327 }
328 const entryNames = [
329 "index.ts", "index.tsx", "index.js", "main.ts", "main.tsx", "main.js",
330 "app.ts", "app.tsx", "mod.rs", "lib.rs", "__init__.py", "main.py",
331 "server.ts", "server.js",
332 ];
333 for (const name of entryNames) {
334 const hit = children.find((e) => e.type === "blob" && e.name === name);
335 if (hit) await tryAdd(`${dir}/${name}`);
336 }
337 for (const child of children) {
338 if (out.length >= MAX_FILES) break;
339 if (totalChars >= MAX_TOTAL_CHARS) break;
340 if (child.type !== "blob") continue;
341 if (!isLikelySource(child.name)) continue;
342 await tryAdd(`${dir}/${child.name}`);
343 }
344 }
345
346 // 4. Remaining top-level source files
347 for (const entry of root) {
348 if (out.length >= MAX_FILES) break;
349 if (totalChars >= MAX_TOTAL_CHARS) break;
350 if (entry.type !== "blob") continue;
351 if (!isLikelySource(entry.name)) continue;
352 await tryAdd(entry.name);
353 }
354
355 let pkg: Record<string, unknown> | null = null;
356 const packageFile = out.find((f) => f.path === "package.json");
357 if (packageFile) {
358 try {
359 pkg = JSON.parse(packageFile.content) as Record<string, unknown>;
360 } catch {
361 pkg = null;
362 }
363 }
364
365 return { files: out, topLevelTree: root, packageJson: pkg };
366}
367
368function looksLikeManifest(name: string): boolean {
369 const lower = name.toLowerCase();
370 return (
371 lower.endsWith(".toml") ||
372 lower.endsWith(".yaml") ||
373 lower.endsWith(".yml") ||
374 lower.endsWith(".json") ||
375 lower === "makefile" ||
376 lower === "dockerfile" ||
377 lower === "procfile"
378 );
379}
380
381function isLikelySource(name: string): boolean {
382 const lower = name.toLowerCase();
383 return (
384 lower.endsWith(".ts") ||
385 lower.endsWith(".tsx") ||
386 lower.endsWith(".js") ||
387 lower.endsWith(".jsx") ||
388 lower.endsWith(".mjs") ||
389 lower.endsWith(".cjs") ||
390 lower.endsWith(".py") ||
391 lower.endsWith(".rs") ||
392 lower.endsWith(".go") ||
393 lower.endsWith(".rb") ||
394 lower.endsWith(".java") ||
395 lower.endsWith(".kt") ||
396 lower.endsWith(".swift") ||
397 lower.endsWith(".c") ||
398 lower.endsWith(".cc") ||
399 lower.endsWith(".cpp") ||
400 lower.endsWith(".h") ||
401 lower.endsWith(".hpp")
402 );
403}
404
405// ---------------------------------------------------------------------------
406// Anthropic API call — structured JSON result
407// ---------------------------------------------------------------------------
408
409async function callAnthropicForStructuredResult(
410 owner: string,
411 repoName: string,
412 samples: Samples,
413 apiKey: string
414): Promise<ExplainJobResult> {
415 const treeListing = samples.topLevelTree
416 .slice(0, 80)
417 .map((e) => (e.type === "tree" ? `${e.name}/` : e.name))
418 .join("\n");
419
420 const fileContents = samples.files
421 .map((f) => `----- FILE: ${f.path} -----\n${f.content}`)
422 .join("\n\n");
423
424 const systemPrompt = `You are a senior software architect. Analyze this codebase and return a JSON object with the following keys (no markdown wrapper, just raw JSON):
425
426{
427 "summary": "2-3 sentence plain-English overview of what this project is and does",
428 "techStack": ["array", "of", "technology", "names", "detected"],
429 "architecture": "Markdown description of the architecture — folder layout, key patterns, data flows. Use bullet points or a short diagram.",
430 "entryPoints": [
431 {"file": "src/index.ts", "role": "Server entry point — starts the HTTP server"},
432 {"file": "src/app.tsx", "role": "Hono app — middleware + route composition"}
433 ],
434 "gettingStarted": "Step-by-step Markdown guide for a new developer to get the project running locally",
435 "healthScore": "one of: Elite | Strong | Improving | Needs Attention",
436 "suggestedIssues": [
437 {"title": "Short issue title", "description": "1-2 sentence description of the improvement"},
438 {"title": "Another issue", "description": "Why this matters for the project"},
439 {"title": "Third suggestion", "description": "Concrete actionable task"}
440 ]
441}
442
443Rules:
444- techStack: include runtime, language, framework, DB, and notable libraries
445- healthScore: base on code quality signals visible in the files (tests, docs, types, CI)
446- suggestedIssues: exactly 3 items, actionable and specific to THIS codebase
447- Be specific to what you see in the code — do not invent features
448- Return ONLY the JSON object, nothing else`;
449
450 const userContent = `Repository: ${owner}/${repoName}
451
452File tree:
453\`\`\`
454${treeListing || "(empty)"}
455\`\`\`
456
457Key files:
458${fileContents}`;
459
460 const response = await fetch("https://api.anthropic.com/v1/messages", {
461 method: "POST",
462 headers: {
463 "x-api-key": apiKey,
464 "anthropic-version": "2023-06-01",
465 "content-type": "application/json",
466 },
467 body: JSON.stringify({
468 model: "claude-opus-4-8",
469 max_tokens: 4000,
470 system: systemPrompt,
471 messages: [{ role: "user", content: userContent }],
472 }),
473 });
474
475 if (!response.ok) {
476 throw new Error(`Anthropic API error: ${response.status}`);
477 }
478
479 const data = (await response.json()) as {
480 content: Array<{ type: string; text: string }>;
481 };
482
483 const text = data.content?.find((b) => b.type === "text")?.text ?? "";
484
485 // Parse JSON — handle possible ```json fences
486 let parsed: ExplainJobResult | null = null;
487 const fenced = text.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
488 if (fenced) {
489 try {
490 parsed = JSON.parse(fenced[1]) as ExplainJobResult;
491 } catch {
492 // fall through
493 }
494 }
495 if (!parsed) {
496 const braceMatch = text.match(/\{[\s\S]*\}/);
497 if (braceMatch) {
498 try {
499 parsed = JSON.parse(braceMatch[0]) as ExplainJobResult;
500 } catch {
501 // fall through
502 }
503 }
504 }
505
506 if (!parsed) {
507 throw new Error("Failed to parse structured JSON from AI response");
508 }
509
510 return normalizeResult(parsed, owner, repoName, samples);
511}
512
513// ---------------------------------------------------------------------------
514// Heuristic fallback (no API key or API failure)
515// ---------------------------------------------------------------------------
516
517function buildHeuristicResult(
518 owner: string,
519 repoName: string,
520 pkg: Record<string, unknown> | null,
521 topLevelTree: GitTreeEntry[]
522): ExplainJobResult {
523 const description =
524 pkg && typeof pkg.description === "string" && pkg.description.trim()
525 ? pkg.description.trim()
526 : `The ${owner}/${repoName} repository.`;
527
528 const techStack = detectTechStackHeuristic(pkg, topLevelTree);
529 const scripts =
530 pkg && pkg.scripts && typeof pkg.scripts === "object"
531 ? Object.keys(pkg.scripts as Record<string, unknown>)
532 : [];
533
534 const gettingStartedLines: string[] = [
535 "```bash",
536 "# Clone the repo",
537 `git clone <repo-url> && cd ${repoName}`,
538 "",
539 ];
540 if (scripts.includes("install") || pkg?.dependencies || pkg?.devDependencies) {
541 gettingStartedLines.push("# Install dependencies");
542 gettingStartedLines.push(pkg ? "bun install # or npm install" : "# see README");
543 gettingStartedLines.push("");
544 }
545 if (scripts.includes("dev")) {
546 gettingStartedLines.push("# Start dev server");
547 gettingStartedLines.push("bun dev");
548 } else if (scripts.includes("start")) {
549 gettingStartedLines.push("bun start");
550 } else {
551 gettingStartedLines.push("# See README for run instructions");
552 }
553 gettingStartedLines.push("```");
554
555 const entryPoints: EntryPoint[] = topLevelTree
556 .filter((e) => e.type === "blob" && isLikelySource(e.name))
557 .slice(0, 5)
558 .map((e) => ({ file: e.name, role: "Top-level source file" }));
559
560 return {
561 summary: description,
562 techStack,
563 architecture:
564 "Architecture analysis requires an `ANTHROPIC_API_KEY` to be configured. " +
565 "The heuristic scan detected the technologies listed in the Tech Stack section.\n\n" +
566 "Top-level layout:\n\n" +
567 topLevelTree
568 .slice(0, 20)
569 .map((e) => `- \`${e.name}${e.type === "tree" ? "/" : ""}\``)
570 .join("\n"),
571 entryPoints: entryPoints.length > 0 ? entryPoints : [
572 { file: "See README", role: "Entry point not auto-detected" },
573 ],
574 gettingStarted: gettingStartedLines.join("\n"),
575 healthScore: "Improving",
576 suggestedIssues: [
577 {
578 title: "Add ANTHROPIC_API_KEY for AI-powered analysis",
579 description:
580 "Set the ANTHROPIC_API_KEY environment variable to enable full AI-powered codebase analysis, architecture diagrams, and smart onboarding suggestions.",
581 },
582 {
583 title: "Add a comprehensive README",
584 description:
585 "A good README with setup steps, architecture overview, and contributing guidelines helps new developers onboard faster.",
586 },
587 {
588 title: "Add automated tests",
589 description:
590 "A test suite with good coverage makes it safe to refactor and ship faster with confidence.",
591 },
592 ],
593 };
594}
595
596function detectTechStackHeuristic(
597 pkg: Record<string, unknown> | null,
598 topLevelTree: GitTreeEntry[]
599): string[] {
600 const stack: string[] = [];
601 const names = topLevelTree.map((e) => e.name.toLowerCase());
602
603 if (names.includes("package.json") || names.includes("bun.lockb")) {
604 stack.push("TypeScript", "Node.js");
605 if (names.includes("bun.lockb")) stack.push("Bun");
606 }
607 if (names.includes("cargo.toml")) stack.push("Rust");
608 if (names.includes("go.mod")) stack.push("Go");
609 if (names.includes("pyproject.toml") || names.includes("requirements.txt")) stack.push("Python");
610 if (names.includes("gemfile")) stack.push("Ruby");
611 if (names.includes("pom.xml") || names.includes("build.gradle")) stack.push("Java");
612 if (names.includes("dockerfile")) stack.push("Docker");
613
614 if (pkg) {
615 const deps = {
616 ...(pkg.dependencies as Record<string, string> | undefined),
617 ...(pkg.devDependencies as Record<string, string> | undefined),
618 };
619 const dkeys = Object.keys(deps);
620 if (dkeys.includes("hono")) stack.push("Hono");
621 if (dkeys.includes("react") || dkeys.includes("react-dom")) stack.push("React");
622 if (dkeys.includes("next")) stack.push("Next.js");
623 if (dkeys.includes("drizzle-orm")) stack.push("Drizzle ORM");
624 if (dkeys.some((k) => k.includes("postgres") || k.includes("pg") || k.includes("neon")))
625 stack.push("PostgreSQL");
626 if (dkeys.includes("sqlite") || dkeys.includes("better-sqlite3"))
627 stack.push("SQLite");
628 if (dkeys.includes("express")) stack.push("Express");
629 if (dkeys.includes("fastify")) stack.push("Fastify");
630 if (dkeys.includes("vite")) stack.push("Vite");
631 if (dkeys.includes("tailwindcss")) stack.push("Tailwind CSS");
632 if (dkeys.some((k) => k.startsWith("@anthropic"))) stack.push("Anthropic Claude");
633 }
634
635 return [...new Set(stack)].slice(0, 12);
636}
637
638// ---------------------------------------------------------------------------
639// Normalize AI result — fill in any missing fields
640// ---------------------------------------------------------------------------
641
642function normalizeResult(
643 raw: Partial<ExplainJobResult>,
644 owner: string,
645 repoName: string,
646 samples: Samples
647): ExplainJobResult {
648 const validScores = ["Elite", "Strong", "Improving", "Needs Attention"] as const;
649
650 return {
651 summary: typeof raw.summary === "string" && raw.summary
652 ? raw.summary
653 : `The ${owner}/${repoName} repository.`,
654 techStack: Array.isArray(raw.techStack) ? raw.techStack.slice(0, 15) : [],
655 architecture: typeof raw.architecture === "string" && raw.architecture
656 ? raw.architecture
657 : "_Architecture not detected._",
658 entryPoints: Array.isArray(raw.entryPoints)
659 ? (raw.entryPoints as EntryPoint[]).slice(0, 10)
660 : [],
661 gettingStarted: typeof raw.gettingStarted === "string" && raw.gettingStarted
662 ? raw.gettingStarted
663 : "_See README for setup instructions._",
664 healthScore:
665 validScores.includes(raw.healthScore as typeof validScores[number])
666 ? (raw.healthScore as ExplainJobResult["healthScore"])
667 : "Improving",
668 suggestedIssues: Array.isArray(raw.suggestedIssues)
669 ? (raw.suggestedIssues as SuggestedIssue[]).slice(0, 3)
670 : [],
671 };
672}