Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

deps.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/deps.ts
deps.tsBlame602 lines · 1 contributor
Normal viewRaw
8098672Claude1/**
2 * Block J1 — Dependency graph.
3 *
4 * Parses common manifest files and records each dependency as a row in
5 * `repo_dependencies`. Per-reindex we REPLACE the whole set for the repo,
6 * so querying the table is always a snapshot.
7 *
8 * Supported ecosystems:
9 * - npm → package.json (dependencies + devDependencies)
10 * - pypi → requirements.txt, pyproject.toml (project.dependencies)
11 * - go → go.mod (require blocks)
12 * - cargo → Cargo.toml ([dependencies] + [dev-dependencies])
13 * - rubygems → Gemfile (gem "name", "version")
14 * - composer → composer.json (require + require-dev)
15 *
16 * We deliberately do not resolve transitive dependencies — we only surface
17 * what's declared in the manifest files. That's the GitHub "dependency
18 * graph" contract: authoritative wrt the repo, not wrt the runtime.
19 */
20
21import { and, desc, eq } from "drizzle-orm";
22import { db } from "../db";
23import {
24 repoDependencies,
25 repositories,
26 users,
27 type RepoDependency,
28} from "../db/schema";
29import {
30 getBlob,
31 getDefaultBranch,
32 getTree,
33 resolveRef,
34 type GitTreeEntry,
35} from "../git/repository";
36
37// ----------------------------------------------------------------------------
38// Types
39// ----------------------------------------------------------------------------
40
41export type Ecosystem =
42 | "npm"
43 | "pypi"
44 | "go"
45 | "rubygems"
46 | "cargo"
47 | "composer";
48
49export interface ParsedDep {
50 ecosystem: Ecosystem;
51 name: string;
52 versionSpec: string | null;
53 isDev: boolean;
54}
55
56/**
57 * Filename → (content) → ParsedDep[]. Keys are the basename; we match
58 * case-insensitively. `manifestPath` (the full path) is attached at a
59 * higher level.
60 */
61const PARSERS: Record<string, (content: string) => ParsedDep[]> = {
62 "package.json": parsePackageJson,
63 "requirements.txt": parseRequirementsTxt,
64 "pyproject.toml": parsePyprojectToml,
65 "go.mod": parseGoMod,
66 "cargo.toml": parseCargoToml,
67 gemfile: parseGemfile,
68 "composer.json": parseComposerJson,
69};
70
71// ----------------------------------------------------------------------------
72// Individual parsers — each is defensive; one bad file does not abort indexing
73// ----------------------------------------------------------------------------
74
75export function parsePackageJson(content: string): ParsedDep[] {
76 const out: ParsedDep[] = [];
77 let obj: any;
78 try {
79 obj = JSON.parse(content);
80 } catch {
81 return out;
82 }
83 if (!obj || typeof obj !== "object") return out;
84 for (const [key, isDev] of [
85 ["dependencies", false],
86 ["devDependencies", true],
87 ["peerDependencies", false],
88 ["optionalDependencies", false],
89 ] as const) {
90 const deps = obj[key];
91 if (!deps || typeof deps !== "object") continue;
92 for (const [name, spec] of Object.entries(deps)) {
93 if (typeof name !== "string" || !name) continue;
94 out.push({
95 ecosystem: "npm",
96 name,
97 versionSpec: typeof spec === "string" ? spec : null,
98 isDev,
99 });
100 }
101 }
102 return out;
103}
104
105export function parseRequirementsTxt(content: string): ParsedDep[] {
106 const out: ParsedDep[] = [];
107 for (const rawLine of content.split(/\r?\n/)) {
108 const line = rawLine.split("#")[0].trim();
109 if (!line) continue;
110 // Skip editable / url installs
111 if (line.startsWith("-e ") || line.startsWith("--") || line.startsWith("git+")) continue;
112 // Match "name", "name==1.2.3", "name>=1.0,<2.0", "name[extra]==1.0"
113 const m = line.match(
114 /^([A-Za-z0-9_][A-Za-z0-9_.\-]*)(?:\[[^\]]+\])?\s*([<>=!~].+)?$/
115 );
116 if (!m) continue;
117 out.push({
118 ecosystem: "pypi",
119 name: m[1],
120 versionSpec: m[2] ? m[2].trim() : null,
121 isDev: false,
122 });
123 }
124 return out;
125}
126
127export function parsePyprojectToml(content: string): ParsedDep[] {
128 const out: ParsedDep[] = [];
129 // We only look for `[project]` → `dependencies = [...]` and
130 // `[project.optional-dependencies]` → `dev = [...]`. Full TOML parsing
131 // is overkill — a regex-over-sections pass is sufficient here.
132 const sections = splitTomlSections(content);
133 const projectDeps = extractTomlArray(sections.project, "dependencies");
134 for (const dep of projectDeps) {
135 const parsed = pythonRequirementToDep(dep, false);
136 if (parsed) out.push(parsed);
137 }
138 const optDeps = sections["project.optional-dependencies"] || "";
139 for (const line of optDeps.split(/\r?\n/)) {
140 const keyMatch = line.match(/^\s*([A-Za-z0-9_-]+)\s*=\s*\[/);
141 if (keyMatch) {
142 // Consume until closing ]
143 const start = line.indexOf("[");
144 const tail = line.slice(start);
145 const closeIdx = tail.indexOf("]");
146 if (closeIdx > -1) {
147 const inner = tail.slice(1, closeIdx);
148 for (const item of splitTomlArrayItems(inner)) {
149 const parsed = pythonRequirementToDep(item, true);
150 if (parsed) out.push(parsed);
151 }
152 }
153 }
154 }
155 return out;
156}
157
158function pythonRequirementToDep(
159 raw: string,
160 isDev: boolean
161): ParsedDep | null {
162 const s = raw.trim().replace(/^["']|["']$/g, "");
163 if (!s) return null;
164 const m = s.match(
165 /^([A-Za-z0-9_][A-Za-z0-9_.\-]*)(?:\[[^\]]+\])?\s*([<>=!~].+)?$/
166 );
167 if (!m) return null;
168 return {
169 ecosystem: "pypi",
170 name: m[1],
171 versionSpec: m[2] ? m[2].trim() : null,
172 isDev,
173 };
174}
175
176export function parseGoMod(content: string): ParsedDep[] {
177 const out: ParsedDep[] = [];
178 // `require (` block or single-line `require foo v1.0.0`
179 const blockMatch = content.match(/require\s*\(([\s\S]*?)\)/);
180 const lines: string[] = [];
181 if (blockMatch) {
182 lines.push(...blockMatch[1].split(/\r?\n/));
183 }
184 for (const rawLine of content.split(/\r?\n/)) {
185 const m = rawLine.match(/^\s*require\s+(\S+)\s+(\S+)/);
186 if (m) lines.push(`${m[1]} ${m[2]}`);
187 }
188 for (const rawLine of lines) {
189 const line = rawLine.split("//")[0].trim();
190 if (!line) continue;
191 const m = line.match(/^(\S+)\s+(\S+)(\s+\/\/ indirect)?$/);
192 if (!m) continue;
193 out.push({
194 ecosystem: "go",
195 name: m[1],
196 versionSpec: m[2],
197 isDev: false,
198 });
199 }
200 return out;
201}
202
203export function parseCargoToml(content: string): ParsedDep[] {
204 const out: ParsedDep[] = [];
205 const sections = splitTomlSections(content);
206 for (const [header, isDev] of [
207 ["dependencies", false],
208 ["dev-dependencies", true],
209 ["build-dependencies", false],
210 ] as const) {
211 const body = sections[header];
212 if (!body) continue;
213 for (const line of body.split(/\r?\n/)) {
214 const trimmed = line.split("#")[0].trim();
215 if (!trimmed) continue;
216 // foo = "1.2.3" OR foo = { version = "1.2.3" }
217 const m = trimmed.match(/^([A-Za-z0-9_-]+)\s*=\s*(.+)$/);
218 if (!m) continue;
219 const name = m[1];
220 const rhs = m[2].trim();
221 let versionSpec: string | null = null;
222 if (rhs.startsWith('"') || rhs.startsWith("'")) {
223 const q = rhs[0];
224 const end = rhs.indexOf(q, 1);
225 if (end > 0) versionSpec = rhs.slice(1, end);
226 } else if (rhs.startsWith("{")) {
227 const vm = rhs.match(/version\s*=\s*["']([^"']+)["']/);
228 if (vm) versionSpec = vm[1];
229 }
230 out.push({
231 ecosystem: "cargo",
232 name,
233 versionSpec,
234 isDev,
235 });
236 }
237 }
238 return out;
239}
240
241export function parseGemfile(content: string): ParsedDep[] {
242 const out: ParsedDep[] = [];
243 let inDevGroup = false;
244 for (const rawLine of content.split(/\r?\n/)) {
245 const line = rawLine.split("#")[0].trim();
246 if (!line) continue;
247 // group :development, :test do
248 if (/^group\s+.*(:development|:test)/i.test(line)) {
249 inDevGroup = true;
250 continue;
251 }
252 if (/^end\b/.test(line)) {
253 inDevGroup = false;
254 continue;
255 }
256 const m = line.match(
257 /^gem\s+["']([^"']+)["'](?:\s*,\s*["']([^"']+)["'])?/
258 );
259 if (!m) continue;
260 out.push({
261 ecosystem: "rubygems",
262 name: m[1],
263 versionSpec: m[2] || null,
264 isDev: inDevGroup,
265 });
266 }
267 return out;
268}
269
270export function parseComposerJson(content: string): ParsedDep[] {
271 const out: ParsedDep[] = [];
272 let obj: any;
273 try {
274 obj = JSON.parse(content);
275 } catch {
276 return out;
277 }
278 if (!obj || typeof obj !== "object") return out;
279 for (const [key, isDev] of [
280 ["require", false],
281 ["require-dev", true],
282 ] as const) {
283 const deps = obj[key];
284 if (!deps || typeof deps !== "object") continue;
285 for (const [name, spec] of Object.entries(deps)) {
286 if (!name || name === "php") continue;
287 out.push({
288 ecosystem: "composer",
289 name,
290 versionSpec: typeof spec === "string" ? spec : null,
291 isDev,
292 });
293 }
294 }
295 return out;
296}
297
298// ----------------------------------------------------------------------------
299// TOML helpers — intentionally minimal
300// ----------------------------------------------------------------------------
301
302function splitTomlSections(content: string): Record<string, string> {
303 const out: Record<string, string> = { __root__: "" };
304 let current = "__root__";
305 for (const rawLine of content.split(/\r?\n/)) {
306 const headerMatch = rawLine.match(/^\s*\[([^\]]+)\]\s*$/);
307 if (headerMatch) {
308 current = headerMatch[1].trim();
309 if (!out[current]) out[current] = "";
310 continue;
311 }
312 out[current] = (out[current] || "") + rawLine + "\n";
313 }
314 // Also expose `project` when it appears at the root.
315 if (!out["project"] && out["__root__"]) {
316 const m = out["__root__"].match(/\[project\]([\s\S]*)/);
317 if (m) out["project"] = m[1];
318 }
319 return out;
320}
321
322function extractTomlArray(body: string | undefined, key: string): string[] {
323 if (!body) return [];
324 const start = body.search(new RegExp(`^\\s*${key}\\s*=\\s*\\[`, "m"));
325 if (start < 0) return [];
326 const open = body.indexOf("[", start);
327 if (open < 0) return [];
328 const close = findMatchingBracket(body, open);
329 if (close < 0) return [];
330 const inner = body.slice(open + 1, close);
331 return splitTomlArrayItems(inner);
332}
333
334function findMatchingBracket(s: string, open: number): number {
335 let depth = 0;
336 for (let i = open; i < s.length; i++) {
337 if (s[i] === "[") depth++;
338 else if (s[i] === "]") {
339 depth--;
340 if (depth === 0) return i;
341 }
342 }
343 return -1;
344}
345
346function splitTomlArrayItems(inner: string): string[] {
347 // Strip newlines + split by commas; handle simple quoted strings.
348 const items: string[] = [];
349 let cur = "";
350 let inQ: string | null = null;
351 for (const ch of inner) {
352 if (inQ) {
353 cur += ch;
354 if (ch === inQ) inQ = null;
355 continue;
356 }
357 if (ch === '"' || ch === "'") {
358 inQ = ch;
359 cur += ch;
360 continue;
361 }
362 if (ch === ",") {
363 if (cur.trim()) items.push(cur);
364 cur = "";
365 continue;
366 }
367 cur += ch;
368 }
369 if (cur.trim()) items.push(cur);
370 return items.map((s) => s.trim()).filter(Boolean);
371}
372
373// ----------------------------------------------------------------------------
374// File detection + tree walk
375// ----------------------------------------------------------------------------
376
377const MANIFEST_BASENAMES = new Set(Object.keys(PARSERS));
378const MAX_MANIFEST_BYTES = 1_000_000;
379const MAX_MANIFESTS = 200;
380
381export function isManifestPath(path: string): boolean {
382 const base = path.split("/").pop()?.toLowerCase() || "";
383 return MANIFEST_BASENAMES.has(base);
384}
385
386export function parseManifest(
387 path: string,
388 content: string
389): ParsedDep[] {
390 const base = path.split("/").pop()?.toLowerCase() || "";
391 const parser = PARSERS[base];
392 if (!parser) return [];
393 try {
394 return parser(content);
395 } catch {
396 return [];
397 }
398}
399
400async function walkManifestPaths(
401 owner: string,
402 repo: string,
403 ref: string
404): Promise<Array<{ path: string; size?: number }>> {
405 const out: Array<{ path: string; size?: number }> = [];
406 const queue: string[] = [""];
407 while (queue.length && out.length < MAX_MANIFESTS) {
408 const dir = queue.shift()!;
409 let entries: GitTreeEntry[] = [];
410 try {
411 entries = await getTree(owner, repo, ref, dir);
412 } catch {
413 continue;
414 }
415 for (const e of entries) {
416 const p = dir ? `${dir}/${e.name}` : e.name;
417 if (e.type === "tree") {
418 const base = e.name.toLowerCase();
419 if (
420 base === "node_modules" ||
421 base === ".git" ||
422 base === "dist" ||
423 base === "build" ||
424 base === "vendor" ||
425 base === "target" ||
426 base === "__pycache__"
427 ) {
428 continue;
429 }
430 queue.push(p);
431 } else if (e.type === "blob") {
432 if (!isManifestPath(p)) continue;
433 if (e.size !== undefined && e.size > MAX_MANIFEST_BYTES) continue;
434 out.push({ path: p, size: e.size });
435 if (out.length >= MAX_MANIFESTS) break;
436 }
437 }
438 }
439 return out;
440}
441
442// ----------------------------------------------------------------------------
443// Reindex + queries
444// ----------------------------------------------------------------------------
445
446export async function indexRepositoryDependencies(
447 repositoryId: string
448): Promise<
449 | {
450 indexed: number;
451 manifests: number;
452 commitSha: string;
453 }
454 | null
455> {
456 try {
457 const [repo] = await db
458 .select()
459 .from(repositories)
460 .where(eq(repositories.id, repositoryId))
461 .limit(1);
462 if (!repo) return null;
463
464 const [owner] = await db
465 .select({ username: users.username })
466 .from(users)
467 .where(eq(users.id, repo.ownerId))
468 .limit(1);
469 if (!owner) return null;
470
471 const defaultBranch =
472 (await getDefaultBranch(owner.username, repo.name)) || "main";
473 const head = await resolveRef(owner.username, repo.name, defaultBranch);
474 if (!head) return null;
475
476 const manifests = await walkManifestPaths(
477 owner.username,
478 repo.name,
479 head
480 );
481
482 const rows: Array<Omit<RepoDependency, "id" | "indexedAt">> = [];
483 for (const f of manifests) {
484 const blob = await getBlob(owner.username, repo.name, head, f.path).catch(
485 () => null
486 );
487 if (!blob) continue;
488 const content =
489 typeof blob === "string"
490 ? blob
491 : new TextDecoder().decode(blob as any);
492 const deps = parseManifest(f.path, content);
493 for (const dep of deps) {
494 rows.push({
495 repositoryId,
496 ecosystem: dep.ecosystem,
497 name: dep.name,
498 versionSpec: dep.versionSpec,
499 manifestPath: f.path,
500 isDev: dep.isDev,
501 commitSha: head,
502 });
503 }
504 }
505
506 await db
507 .delete(repoDependencies)
508 .where(eq(repoDependencies.repositoryId, repositoryId));
509
510 // Insert in chunks to stay under parameter limits.
511 const CHUNK = 500;
512 for (let i = 0; i < rows.length; i += CHUNK) {
513 const slice = rows.slice(i, i + CHUNK);
514 if (slice.length) await db.insert(repoDependencies).values(slice);
515 }
516
517 return {
518 indexed: rows.length,
519 manifests: manifests.length,
520 commitSha: head,
521 };
522 } catch (err) {
523 console.error("[deps] indexRepositoryDependencies:", err);
524 return null;
525 }
526}
527
528export async function listDependenciesForRepo(
529 repositoryId: string
530): Promise<RepoDependency[]> {
531 try {
532 return await db
533 .select()
534 .from(repoDependencies)
535 .where(eq(repoDependencies.repositoryId, repositoryId))
536 .orderBy(desc(repoDependencies.ecosystem), desc(repoDependencies.name));
537 } catch {
538 return [];
539 }
540}
541
542export interface EcosystemSummary {
543 ecosystem: string;
544 count: number;
545}
546
547export async function summarizeDependencies(
548 repositoryId: string
549): Promise<EcosystemSummary[]> {
550 const rows = await listDependenciesForRepo(repositoryId);
551 const counts = new Map<string, number>();
552 for (const r of rows) {
553 counts.set(r.ecosystem, (counts.get(r.ecosystem) || 0) + 1);
554 }
555 return Array.from(counts.entries())
556 .map(([ecosystem, count]) => ({ ecosystem, count }))
557 .sort((a, b) => b.count - a.count);
558}
559
560/** Look up reverse deps — which repos list this package? (Network graph.) */
561export async function repositoriesDependingOn(
562 ecosystem: string,
563 name: string,
564 limit = 50
565): Promise<
566 Array<{
567 repositoryId: string;
568 versionSpec: string | null;
569 manifestPath: string;
570 }>
571> {
572 try {
573 const rows = await db
574 .select({
575 repositoryId: repoDependencies.repositoryId,
576 versionSpec: repoDependencies.versionSpec,
577 manifestPath: repoDependencies.manifestPath,
578 })
579 .from(repoDependencies)
580 .where(
581 and(
582 eq(repoDependencies.ecosystem, ecosystem),
583 eq(repoDependencies.name, name)
584 )
585 )
586 .limit(limit);
587 return rows;
588 } catch {
589 return [];
590 }
591}
592
593// ----------------------------------------------------------------------------
594// Test-only exports
595// ----------------------------------------------------------------------------
596
597export const __internal = {
598 splitTomlSections,
599 extractTomlArray,
600 splitTomlArrayItems,
601 pythonRequirementToDep,
602};