Blame · Line-by-line history

`mirrors.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

mirrors.tsBlame389 lines · 1 contributor

Normal view Raw

4a0dea1Claude	1	/**
	2	* Block I9 — Repository mirroring.
	3	*
	4	* Pull-style mirroring: a mirrored repo has an upstream URL that we
	5	* periodically `git fetch` from. We run it as `git remote update` into
	6	* the bare repo so refs/heads/* are kept in sync with upstream's.
	7	*
	8	* SECURITY: only http(s) and git:// URLs are accepted. We refuse any URL
	9	* with shell metacharacters, `file://`, `ssh://`, or paths that could
	10	* escape the bare repo. Credentials embedded in URLs are allowed (the
	11	* caller decides whether to persist them) but stripped from logs.
	12	*/
	13
	14	import { and, desc, eq } from "drizzle-orm";
	15	import { db } from "../db";
	16	import {
	17	repoMirrors,
	18	repoMirrorRuns,
	19	repositories,
	20	users,
	21	} from "../db/schema";
	22	import { getRepoPath } from "../git/repository";
	23
	24	const MIRROR_REMOTE_NAME = "gluecron-mirror";
	25	const FETCH_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
	26
	27	export interface ValidationResult {
	28	ok: boolean;
	29	error?: string;
	30	}
	31
	32	/** Pure — validates an upstream URL for use as a mirror. */
	33	export function validateUpstreamUrl(url: string): ValidationResult {
	34	if (!url \|\| typeof url !== "string") {
	35	return { ok: false, error: "URL is required" };
	36	}
	37	const trimmed = url.trim();
	38	if (trimmed.length === 0) {
	39	return { ok: false, error: "URL is required" };
	40	}
	41	if (trimmed.length > 2048) {
	42	return { ok: false, error: "URL too long" };
	43	}
	44	// Reject shell metacharacters that Bun.spawn would pass through safely,
	45	// but which should never appear in a legitimate git URL.
	46	if (/[\s;&\|`$\\<>]/.test(trimmed)) {
	47	return { ok: false, error: "URL contains invalid characters" };
	48	}
	49	// Accept only https/http/git schemes. Reject ssh/file/local paths —
	50	// ssh needs key management we don't have yet, file:// lets the user
	51	// escape into the server's filesystem.
	52	const allowed = /^(https?:\/\/\|git:\/\/)/i;
	53	if (!allowed.test(trimmed)) {
	54	return { ok: false, error: "URL must start with https://, http://, or git://" };
	55	}
	56	return { ok: true };
	57	}
	58
	59	/** Strip credentials from a URL for safe logging. */
	60	export function safeUrlForLog(url: string): string {
	61	try {
	62	const u = new URL(url);
	63	if (u.username \|\| u.password) {
	64	u.username = "***";
	65	u.password = "";
	66	return u.toString();
	67	}
	68	return url;
	69	} catch {
	70	return url;
	71	}
	72	}
	73
	74	export interface UpsertMirrorInput {
	75	repositoryId: string;
	76	upstreamUrl: string;
	77	intervalMinutes?: number;
	78	isEnabled?: boolean;
	79	}
	80
	81	/** Create or update the mirror config for a repository. */
	82	export async function upsertMirror(
	83	input: UpsertMirrorInput
	84	): Promise<{ ok: true; id: string } \| { ok: false; error: string }> {
	85	const v = validateUpstreamUrl(input.upstreamUrl);
	86	if (!v.ok) return { ok: false, error: v.error! };
	87
	88	try {
	89	const [existing] = await db
	90	.select()
	91	.from(repoMirrors)
	92	.where(eq(repoMirrors.repositoryId, input.repositoryId))
	93	.limit(1);
	94
	95	if (existing) {
	96	await db
	97	.update(repoMirrors)
	98	.set({
	99	upstreamUrl: input.upstreamUrl.trim(),
	100	intervalMinutes: input.intervalMinutes ?? existing.intervalMinutes,
	101	isEnabled: input.isEnabled ?? existing.isEnabled,
	102	updatedAt: new Date(),
	103	})
	104	.where(eq(repoMirrors.id, existing.id));
	105	return { ok: true, id: existing.id };
	106	}
	107
	108	const [row] = await db
	109	.insert(repoMirrors)
	110	.values({
	111	repositoryId: input.repositoryId,
	112	upstreamUrl: input.upstreamUrl.trim(),
	113	intervalMinutes: input.intervalMinutes ?? 1440,
	114	isEnabled: input.isEnabled ?? true,
	115	})
	116	.returning({ id: repoMirrors.id });
	117	return { ok: true, id: row.id };
	118	} catch (err) {
	119	console.error("[mirrors] upsertMirror error:", err);
	120	return { ok: false, error: "Failed to save mirror configuration" };
	121	}
	122	}
	123
	124	export async function deleteMirror(repositoryId: string): Promise<void> {
	125	try {
	126	await db
	127	.delete(repoMirrors)
	128	.where(eq(repoMirrors.repositoryId, repositoryId));
	129	} catch (err) {
	130	console.error("[mirrors] deleteMirror error:", err);
	131	}
	132	}
	133
	134	export async function getMirrorForRepo(repositoryId: string) {
	135	try {
	136	const [row] = await db
	137	.select()
	138	.from(repoMirrors)
	139	.where(eq(repoMirrors.repositoryId, repositoryId))
	140	.limit(1);
	141	return row \|\| null;
	142	} catch {
	143	return null;
	144	}
	145	}
	146
	147	export async function listRecentRuns(
	148	mirrorId: string,
	149	limit = 20
	150	): Promise<Array<typeof repoMirrorRuns.$inferSelect>> {
	151	try {
	152	return await db
	153	.select()
	154	.from(repoMirrorRuns)
	155	.where(eq(repoMirrorRuns.mirrorId, mirrorId))
	156	.orderBy(desc(repoMirrorRuns.startedAt))
	157	.limit(limit);
	158	} catch {
	159	return [];
	160	}
	161	}
	162
	163	/**
	164	* Execute one sync run for a mirror. Returns the run row after completion.
	165	* Safe to call concurrently per-repo (we'd still end up fetching serially
	166	* because git locks `packed-refs` during fetch).
	167	*/
	168	export async function runMirrorSync(
	169	mirrorId: string
	170	): Promise<{ ok: boolean; message: string; exitCode: number }> {
	171	// Load mirror + owning repo in one shot.
	172	const [row] = await db
	173	.select({
	174	mirror: repoMirrors,
	175	repoName: repositories.name,
	176	ownerName: users.username,
	177	})
	178	.from(repoMirrors)
	179	.innerJoin(repositories, eq(repoMirrors.repositoryId, repositories.id))
	180	.innerJoin(users, eq(repositories.ownerId, users.id))
	181	.where(eq(repoMirrors.id, mirrorId))
	182	.limit(1);
	183
	184	if (!row) {
	185	return { ok: false, message: "mirror not found", exitCode: -1 };
	186	}
	187	if (!row.mirror.isEnabled) {
	188	return { ok: false, message: "mirror disabled", exitCode: -1 };
	189	}
	190
	191	const repoPath = getRepoPath(row.ownerName, row.repoName);
	192
	193	const [runRow] = await db
	194	.insert(repoMirrorRuns)
	195	.values({ mirrorId, status: "running" })
	196	.returning();
	197
	198	const url = row.mirror.upstreamUrl;
	199	let exitCode = -1;
	200	let stdout = "";
	201	let stderr = "";
	202
	203	try {
	204	// Ensure the remote exists and points at the current URL.
	205	await runGit(["git", "remote", "remove", MIRROR_REMOTE_NAME], repoPath);
	206	const addRes = await runGit(
	207	["git", "remote", "add", MIRROR_REMOTE_NAME, url],
	208	repoPath
	209	);
	210	if (addRes.exitCode !== 0) {
	211	throw new Error(`remote add failed: ${addRes.stderr}`);
	212	}
	213
	214	const fetchRes = await runGit(
	215	[
	216	"git",
	217	"fetch",
	218	"--prune",
	219	"--tags",
	220	"--no-write-fetch-head",
	221	MIRROR_REMOTE_NAME,
	222	"+refs/heads/:refs/heads/",
	223	],
	224	repoPath,
	225	FETCH_TIMEOUT_MS
	226	);
	227	exitCode = fetchRes.exitCode;
	228	stdout = fetchRes.stdout;
	229	stderr = fetchRes.stderr;
	230	if (exitCode !== 0) {
	231	throw new Error(stderr.slice(0, 4000) \|\| "git fetch failed");
	232	}
	233
	234	const messageLines: string[] = [];
	235	if (stdout.trim()) messageLines.push(`stdout:\n${stdout.trim()}`);
	236	if (stderr.trim()) messageLines.push(`stderr:\n${stderr.trim()}`);
	237	const message =
	238	messageLines.join("\n\n").slice(0, 4000) \|\| "Mirror synced (no changes)";
	239
	240	await db
	241	.update(repoMirrorRuns)
	242	.set({
	243	finishedAt: new Date(),
	244	status: "ok",
	245	message,
	246	exitCode,
	247	})
	248	.where(eq(repoMirrorRuns.id, runRow.id));
	249
	250	await db
	251	.update(repoMirrors)
	252	.set({
	253	lastSyncedAt: new Date(),
	254	lastStatus: "ok",
	255	lastError: null,
	256	updatedAt: new Date(),
	257	})
	258	.where(eq(repoMirrors.id, mirrorId));
	259
	260	return { ok: true, message, exitCode };
	261	} catch (err: any) {
	262	const errMsg = String(err?.message \|\| err \|\| "unknown error").slice(0, 4000);
	263	await db
	264	.update(repoMirrorRuns)
	265	.set({
	266	finishedAt: new Date(),
	267	status: "error",
	268	message: errMsg,
	269	exitCode,
	270	})
	271	.where(eq(repoMirrorRuns.id, runRow.id));
	272
	273	await db
	274	.update(repoMirrors)
	275	.set({
	276	lastSyncedAt: new Date(),
	277	lastStatus: "error",
	278	lastError: errMsg,
	279	updatedAt: new Date(),
	280	})
	281	.where(eq(repoMirrors.id, mirrorId));
	282
	283	return { ok: false, message: errMsg, exitCode };
	284	}
	285	}
	286
	287	// ---------- Internal ----------
	288
	289	async function runGit(
	290	cmd: string[],
	291	cwd: string,
	292	timeoutMs = 60_000
	293	): Promise<{ exitCode: number; stdout: string; stderr: string }> {
	294	try {
	295	const proc = Bun.spawn(cmd, {
	296	cwd,
	297	stdout: "pipe",
	298	stderr: "pipe",
	299	env: {
	300	...process.env,
	301	GIT_TERMINAL_PROMPT: "0", // never prompt for creds
	302	},
	303	});
	304	const timer = setTimeout(() => {
	305	try {
	306	proc.kill("SIGKILL");
	307	} catch {
	308	// ignore
	309	}
	310	}, timeoutMs);
	311	const [stdout, stderr] = await Promise.all([
	312	new Response(proc.stdout).text(),
	313	new Response(proc.stderr).text(),
	314	]);
	315	const exitCode = await proc.exited;
	316	clearTimeout(timer);
	317	return { exitCode, stdout, stderr };
	318	} catch (err: any) {
	319	return {
	320	exitCode: -1,
	321	stdout: "",
	322	stderr: String(err?.message \|\| err \|\| "spawn failed"),
	323	};
	324	}
	325	}
	326
	327	/** Returns mirrors that are due for a sync (used by admin cron trigger). */
	328	export async function listDueMirrors(
	329	now: Date = new Date()
	330	): Promise<Array<{ id: string; repositoryId: string; upstreamUrl: string }>> {
	331	try {
	332	const rows = await db
	333	.select()
	334	.from(repoMirrors)
	335	.where(eq(repoMirrors.isEnabled, true));
	336	const due: Array<{
	337	id: string;
	338	repositoryId: string;
	339	upstreamUrl: string;
	340	}> = [];
	341	for (const r of rows) {
	342	if (!r.lastSyncedAt) {
	343	due.push({
	344	id: r.id,
	345	repositoryId: r.repositoryId,
	346	upstreamUrl: r.upstreamUrl,
	347	});
	348	continue;
	349	}
	350	const last = new Date(r.lastSyncedAt as any).getTime();
	351	const elapsedMin = (now.getTime() - last) / 60000;
	352	if (elapsedMin >= r.intervalMinutes) {
	353	due.push({
	354	id: r.id,
	355	repositoryId: r.repositoryId,
	356	upstreamUrl: r.upstreamUrl,
	357	});
	358	}
	359	}
	360	return due;
	361	} catch {
	362	return [];
	363	}
	364	}
	365
	366	/** Run sync for every due mirror. Returns summary counts. */
	367	export async function syncAllDue(): Promise<{
	368	total: number;
	369	ok: number;
	370	failed: number;
	371	}> {
	372	const due = await listDueMirrors();
	373	let ok = 0;
	374	let failed = 0;
	375	for (const m of due) {
	376	const r = await runMirrorSync(m.id);
	377	if (r.ok) ok++;
	378	else failed++;
	379	}
	380	return { total: due.length, ok, failed };
	381	}
	382
	383	// Suppress unused import warning for `and`.
	384	void and;
	385
	386	export const __internal = {
	387	MIRROR_REMOTE_NAME,
	388	FETCH_TIMEOUT_MS,
	389	};