Blame · Line-by-line history

`index-existing-repo.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

index-existing-repo.tsBlame149 lines · 1 contributor

Normal view Raw

5d13cf5ccantynz-alt	1	/**
	2	* Semantically index a repository that arrived without a push.
	3	*
	4	* `indexChangedFiles()` is called from exactly one place: the git post-receive
	5	* hook. That is correct for the push path and wrong for every other way a
	6	* repository comes into existence:
	7	*
	8	* - Bulk import clones with `git clone`, which never fires post-receive.
	9	* - POST /new + first-repo-scaffold writes through the git plumbing API,
	10	* which also never fires post-receive.
	11	* - Forks copy the bare repo directly.
	12	*
	13	* So the two onboarding paths built in this session produced repositories
	14	* where `gluecron_semantic_search`, the repo chat, and "ask this repo" all
	15	* returned nothing — silently, because an empty index is indistinguishable
	16	* from "no matches". The agent-first experience depends entirely on that
	17	* index being populated, which is how this was found.
	18	*
	19	* post-receive indexes the paths that CHANGED in a push. There is no such
	20	* delta for a repo that just arrived, so this walks the tree at HEAD and
	21	* indexes what is there, then delegates to the same indexChangedFiles() the
	22	* hook uses — same filtering, same embedding model, same cap.
	23	*/
	24
	25	import { mapWithConcurrency, DB_FANOUT_LIMIT } from "./concurrency";
	26
	27	export interface IndexRepoResult {
	28	indexed: number;
	29	skipped: number;
	30	model: string;
	31	error?: string;
	32	}
	33
	34	export interface IndexRepoDeps {
	35	getTree: (
	36	owner: string,
	37	repo: string,
	38	ref: string,
	39	path?: string
	40	) => Promise<Array<{ name: string; type: string; path?: string }>>;
	41	indexChangedFiles: (args: {
	42	repositoryId: string;
	43	ownerName: string;
	44	repoName: string;
	45	commitSha: string;
	46	changedPaths: string[];
	47	}) => Promise<{ indexed: number; skipped: number; model: string }>;
	48	}
	49
	50	async function realDeps(): Promise<IndexRepoDeps> {
	51	const { getTree } = await import("../git/repository");
	52	const { indexChangedFiles } = await import("./semantic-index");
	53	return { getTree: getTree as IndexRepoDeps["getTree"], indexChangedFiles };
	54	}
	55
	56	/** Directories never worth embedding. Keeps the walk cheap on a big import. */
	57	const SKIP_DIRS = new Set([
	58	".git", "node_modules", "dist", "build", "out", "target", "vendor",
	59	".next", ".cache", "coverage", "__pycache__", ".venv", "venv",
	60	]);
	61
	62	/** Depth cap. A pathological monorepo should not turn one import into a
	63	* thousand tree reads; indexChangedFiles caps files independently. */
	64	const MAX_DEPTH = 6;
	65
	66	/**
	67	* Collect blob paths at `ref`, breadth-first, bounded.
	68	*
	69	* Concurrency is bounded for the same reason as everywhere else in the
	70	* onboarding path: an import of 100 repos must not open an unbounded number
	71	* of concurrent tree reads against one disk.
	72	*/
	73	async function collectPaths(
	74	deps: IndexRepoDeps,
	75	owner: string,
	76	repo: string,
	77	ref: string
	78	): Promise<string[]> {
	79	const files: string[] = [];
	80	let level: string[] = [""];
	81
	82	for (let depth = 0; depth < MAX_DEPTH && level.length > 0; depth++) {
	83	const results = await mapWithConcurrency(level, DB_FANOUT_LIMIT, async (dir) => {
	84	try {
	85	return await deps.getTree(owner, repo, ref, dir \|\| undefined);
	86	} catch {
	87	// A single unreadable directory must not abandon the whole index.
	88	return [];
	89	}
	90	});
	91
	92	const next: string[] = [];
	93	for (let i = 0; i < results.length; i++) {
	94	const dir = level[i];
	95	for (const entry of results[i] ?? []) {
	96	const full = entry.path ?? (dir ? `${dir}/${entry.name}` : entry.name);
	97	if (entry.type === "tree") {
	98	if (SKIP_DIRS.has(entry.name)) continue;
	99	next.push(full);
	100	} else if (entry.type === "blob") {
	101	files.push(full);
	102	}
	103	}
	104	}
	105	level = next;
	106	}
	107
	108	return files;
	109	}
	110
	111	/**
	112	* Index a repository at `commitSha`. Never throws — an unindexed repo is a
	113	* degraded search experience, not a failed import or a failed repo creation.
	114	*/
	115	export async function indexExistingRepo(
	116	args: {
	117	repositoryId: string;
	118	owner: string;
	119	repoName: string;
	120	ref: string;
	121	commitSha: string;
	122	},
	123	injected?: IndexRepoDeps
	124	): Promise<IndexRepoResult> {
	125	try {
	126	const deps = injected ?? (await realDeps());
	127	const paths = await collectPaths(deps, args.owner, args.repoName, args.ref);
	128	if (paths.length === 0) {
	129	return { indexed: 0, skipped: 0, model: "none" };
	130	}
	131	// indexChangedFiles does the filtering (isCodeFile), the cap, and the
	132	// embedding — reusing it keeps scaffolded, imported and pushed repos on
	133	// exactly the same indexing behaviour.
	134	return await deps.indexChangedFiles({
	135	repositoryId: args.repositoryId,
	136	ownerName: args.owner,
	137	repoName: args.repoName,
	138	commitSha: args.commitSha,
	139	changedPaths: paths,
	140	});
	141	} catch (err) {
	142	return {
	143	indexed: 0,
	144	skipped: 0,
	145	model: "none",
	146	error: err instanceof Error ? err.message : String(err),
	147	};
	148	}
	149	}