Blame · Line-by-line history

`mirrors.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

mirrors.tsBlame410 lines · 1 contributor

Normal view Raw

4a0dea1Claude	1	/**
	2	* Block I9 — Repository mirroring.
	3	*
	4	* Pull-style mirroring: a mirrored repo has an upstream URL that we
	5	* periodically `git fetch` from. We run it as `git remote update` into
	6	* the bare repo so refs/heads/* are kept in sync with upstream's.
	7	*
	8	* SECURITY: only http(s) and git:// URLs are accepted. We refuse any URL
	9	* with shell metacharacters, `file://`, `ssh://`, or paths that could
	10	* escape the bare repo. Credentials embedded in URLs are allowed (the
	11	* caller decides whether to persist them) but stripped from logs.
	12	*/
	13
	14	import { and, desc, eq } from "drizzle-orm";
	15	import { db } from "../db";
	16	import {
	17	repoMirrors,
	18	repoMirrorRuns,
	19	repositories,
	20	users,
	21	} from "../db/schema";
	22	import { getRepoPath } from "../git/repository";
bc519aeClaude	23	import { assertPublicUrl } from "./ssrf-guard";
4a0dea1Claude	24
	25	const MIRROR_REMOTE_NAME = "gluecron-mirror";
bc519aeClaude	26
	27	/** Mirror upstreams may use git:// in addition to http(s). */
	28	const MIRROR_SCHEMES = ["http:", "https:", "git:"];
4a0dea1Claude	29	const FETCH_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
	30
	31	export interface ValidationResult {
	32	ok: boolean;
	33	error?: string;
	34	}
	35
	36	/** Pure — validates an upstream URL for use as a mirror. */
	37	export function validateUpstreamUrl(url: string): ValidationResult {
	38	if (!url \|\| typeof url !== "string") {
	39	return { ok: false, error: "URL is required" };
	40	}
	41	const trimmed = url.trim();
	42	if (trimmed.length === 0) {
	43	return { ok: false, error: "URL is required" };
	44	}
	45	if (trimmed.length > 2048) {
	46	return { ok: false, error: "URL too long" };
	47	}
	48	// Reject shell metacharacters that Bun.spawn would pass through safely,
	49	// but which should never appear in a legitimate git URL.
	50	if (/[\s;&\|`$\\<>]/.test(trimmed)) {
	51	return { ok: false, error: "URL contains invalid characters" };
	52	}
	53	// Accept only https/http/git schemes. Reject ssh/file/local paths —
	54	// ssh needs key management we don't have yet, file:// lets the user
	55	// escape into the server's filesystem.
	56	const allowed = /^(https?:\/\/\|git:\/\/)/i;
	57	if (!allowed.test(trimmed)) {
	58	return { ok: false, error: "URL must start with https://, http://, or git://" };
	59	}
bc519aeClaude	60	// SSRF guard (BUILD_BIBLE §7): refuse upstreams that point at private/
	61	// internal address space (localhost, RFC1918, link-local metadata, etc).
	62	// SSRF_ALLOW_PRIVATE=1 opts out for self-hosted setups mirroring
	63	// internal git servers.
	64	const guard = assertPublicUrl(trimmed, { schemes: MIRROR_SCHEMES });
	65	if (!guard.ok) {
	66	return { ok: false, error: `blocked: ${guard.reason} (SSRF protection)` };
	67	}
4a0dea1Claude	68	return { ok: true };
	69	}
	70
	71	/** Strip credentials from a URL for safe logging. */
	72	export function safeUrlForLog(url: string): string {
	73	try {
	74	const u = new URL(url);
	75	if (u.username \|\| u.password) {
	76	u.username = "***";
	77	u.password = "";
	78	return u.toString();
	79	}
	80	return url;
	81	} catch {
	82	return url;
	83	}
	84	}
	85
	86	export interface UpsertMirrorInput {
	87	repositoryId: string;
	88	upstreamUrl: string;
	89	intervalMinutes?: number;
	90	isEnabled?: boolean;
	91	}
	92
	93	/** Create or update the mirror config for a repository. */
	94	export async function upsertMirror(
	95	input: UpsertMirrorInput
	96	): Promise<{ ok: true; id: string } \| { ok: false; error: string }> {
	97	const v = validateUpstreamUrl(input.upstreamUrl);
	98	if (!v.ok) return { ok: false, error: v.error! };
	99
	100	try {
	101	const [existing] = await db
	102	.select()
	103	.from(repoMirrors)
	104	.where(eq(repoMirrors.repositoryId, input.repositoryId))
	105	.limit(1);
	106
	107	if (existing) {
	108	await db
	109	.update(repoMirrors)
	110	.set({
	111	upstreamUrl: input.upstreamUrl.trim(),
	112	intervalMinutes: input.intervalMinutes ?? existing.intervalMinutes,
	113	isEnabled: input.isEnabled ?? existing.isEnabled,
	114	updatedAt: new Date(),
	115	})
	116	.where(eq(repoMirrors.id, existing.id));
	117	return { ok: true, id: existing.id };
	118	}
	119
	120	const [row] = await db
	121	.insert(repoMirrors)
	122	.values({
	123	repositoryId: input.repositoryId,
	124	upstreamUrl: input.upstreamUrl.trim(),
	125	intervalMinutes: input.intervalMinutes ?? 1440,
	126	isEnabled: input.isEnabled ?? true,
	127	})
	128	.returning({ id: repoMirrors.id });
	129	return { ok: true, id: row.id };
	130	} catch (err) {
	131	console.error("[mirrors] upsertMirror error:", err);
	132	return { ok: false, error: "Failed to save mirror configuration" };
	133	}
	134	}
	135
	136	export async function deleteMirror(repositoryId: string): Promise<void> {
	137	try {
	138	await db
	139	.delete(repoMirrors)
	140	.where(eq(repoMirrors.repositoryId, repositoryId));
	141	} catch (err) {
	142	console.error("[mirrors] deleteMirror error:", err);
	143	}
	144	}
	145
	146	export async function getMirrorForRepo(repositoryId: string) {
	147	try {
	148	const [row] = await db
	149	.select()
	150	.from(repoMirrors)
	151	.where(eq(repoMirrors.repositoryId, repositoryId))
	152	.limit(1);
	153	return row \|\| null;
	154	} catch {
	155	return null;
	156	}
	157	}
	158
	159	export async function listRecentRuns(
	160	mirrorId: string,
	161	limit = 20
	162	): Promise<Array<typeof repoMirrorRuns.$inferSelect>> {
	163	try {
	164	return await db
	165	.select()
	166	.from(repoMirrorRuns)
	167	.where(eq(repoMirrorRuns.mirrorId, mirrorId))
	168	.orderBy(desc(repoMirrorRuns.startedAt))
	169	.limit(limit);
	170	} catch {
	171	return [];
	172	}
	173	}
	174
	175	/**
	176	* Execute one sync run for a mirror. Returns the run row after completion.
	177	* Safe to call concurrently per-repo (we'd still end up fetching serially
	178	* because git locks `packed-refs` during fetch).
	179	*/
	180	export async function runMirrorSync(
	181	mirrorId: string
	182	): Promise<{ ok: boolean; message: string; exitCode: number }> {
	183	// Load mirror + owning repo in one shot.
	184	const [row] = await db
	185	.select({
	186	mirror: repoMirrors,
	187	repoName: repositories.name,
	188	ownerName: users.username,
	189	})
	190	.from(repoMirrors)
	191	.innerJoin(repositories, eq(repoMirrors.repositoryId, repositories.id))
	192	.innerJoin(users, eq(repositories.ownerId, users.id))
	193	.where(eq(repoMirrors.id, mirrorId))
	194	.limit(1);
	195
	196	if (!row) {
	197	return { ok: false, message: "mirror not found", exitCode: -1 };
	198	}
	199	if (!row.mirror.isEnabled) {
	200	return { ok: false, message: "mirror disabled", exitCode: -1 };
	201	}
	202
	203	const repoPath = getRepoPath(row.ownerName, row.repoName);
	204
	205	const [runRow] = await db
	206	.insert(repoMirrorRuns)
	207	.values({ mirrorId, status: "running" })
	208	.returning();
	209
	210	const url = row.mirror.upstreamUrl;
	211	let exitCode = -1;
	212	let stdout = "";
	213	let stderr = "";
	214
	215	try {
bc519aeClaude	216	// SSRF guard — re-check at use time: the stored URL may predate the
	217	// guard or have been saved while SSRF_ALLOW_PRIVATE was set. Throwing
	218	// here lands in the catch below, which records the run as failed and
	219	// never throws out.
	220	const guard = assertPublicUrl(url, { schemes: MIRROR_SCHEMES });
	221	if (!guard.ok) {
	222	throw new Error(`blocked: ${guard.reason} (SSRF protection)`);
	223	}
	224
4a0dea1Claude	225	// Ensure the remote exists and points at the current URL.
	226	await runGit(["git", "remote", "remove", MIRROR_REMOTE_NAME], repoPath);
	227	const addRes = await runGit(
	228	["git", "remote", "add", MIRROR_REMOTE_NAME, url],
	229	repoPath
	230	);
	231	if (addRes.exitCode !== 0) {
	232	throw new Error(`remote add failed: ${addRes.stderr}`);
	233	}
	234
	235	const fetchRes = await runGit(
	236	[
	237	"git",
	238	"fetch",
	239	"--prune",
	240	"--tags",
	241	"--no-write-fetch-head",
	242	MIRROR_REMOTE_NAME,
	243	"+refs/heads/:refs/heads/",
	244	],
	245	repoPath,
	246	FETCH_TIMEOUT_MS
	247	);
	248	exitCode = fetchRes.exitCode;
	249	stdout = fetchRes.stdout;
	250	stderr = fetchRes.stderr;
	251	if (exitCode !== 0) {
	252	throw new Error(stderr.slice(0, 4000) \|\| "git fetch failed");
	253	}
	254
	255	const messageLines: string[] = [];
	256	if (stdout.trim()) messageLines.push(`stdout:\n${stdout.trim()}`);
	257	if (stderr.trim()) messageLines.push(`stderr:\n${stderr.trim()}`);
	258	const message =
	259	messageLines.join("\n\n").slice(0, 4000) \|\| "Mirror synced (no changes)";
	260
	261	await db
	262	.update(repoMirrorRuns)
	263	.set({
	264	finishedAt: new Date(),
	265	status: "ok",
	266	message,
	267	exitCode,
	268	})
	269	.where(eq(repoMirrorRuns.id, runRow.id));
	270
	271	await db
	272	.update(repoMirrors)
	273	.set({
	274	lastSyncedAt: new Date(),
	275	lastStatus: "ok",
	276	lastError: null,
	277	updatedAt: new Date(),
	278	})
	279	.where(eq(repoMirrors.id, mirrorId));
	280
	281	return { ok: true, message, exitCode };
	282	} catch (err: any) {
	283	const errMsg = String(err?.message \|\| err \|\| "unknown error").slice(0, 4000);
	284	await db
	285	.update(repoMirrorRuns)
	286	.set({
	287	finishedAt: new Date(),
	288	status: "error",
	289	message: errMsg,
	290	exitCode,
	291	})
	292	.where(eq(repoMirrorRuns.id, runRow.id));
	293
	294	await db
	295	.update(repoMirrors)
	296	.set({
	297	lastSyncedAt: new Date(),
	298	lastStatus: "error",
	299	lastError: errMsg,
	300	updatedAt: new Date(),
	301	})
	302	.where(eq(repoMirrors.id, mirrorId));
	303
	304	return { ok: false, message: errMsg, exitCode };
	305	}
	306	}
	307
	308	// ---------- Internal ----------
	309
	310	async function runGit(
	311	cmd: string[],
	312	cwd: string,
	313	timeoutMs = 60_000
	314	): Promise<{ exitCode: number; stdout: string; stderr: string }> {
	315	try {
	316	const proc = Bun.spawn(cmd, {
	317	cwd,
	318	stdout: "pipe",
	319	stderr: "pipe",
	320	env: {
	321	...process.env,
	322	GIT_TERMINAL_PROMPT: "0", // never prompt for creds
	323	},
	324	});
	325	const timer = setTimeout(() => {
	326	try {
	327	proc.kill("SIGKILL");
	328	} catch {
	329	// ignore
	330	}
	331	}, timeoutMs);
	332	const [stdout, stderr] = await Promise.all([
	333	new Response(proc.stdout).text(),
	334	new Response(proc.stderr).text(),
	335	]);
	336	const exitCode = await proc.exited;
	337	clearTimeout(timer);
	338	return { exitCode, stdout, stderr };
	339	} catch (err: any) {
	340	return {
	341	exitCode: -1,
	342	stdout: "",
	343	stderr: String(err?.message \|\| err \|\| "spawn failed"),
	344	};
	345	}
	346	}
	347
	348	/** Returns mirrors that are due for a sync (used by admin cron trigger). */
	349	export async function listDueMirrors(
	350	now: Date = new Date()
	351	): Promise<Array<{ id: string; repositoryId: string; upstreamUrl: string }>> {
	352	try {
	353	const rows = await db
	354	.select()
	355	.from(repoMirrors)
	356	.where(eq(repoMirrors.isEnabled, true));
	357	const due: Array<{
	358	id: string;
	359	repositoryId: string;
	360	upstreamUrl: string;
	361	}> = [];
	362	for (const r of rows) {
	363	if (!r.lastSyncedAt) {
	364	due.push({
	365	id: r.id,
	366	repositoryId: r.repositoryId,
	367	upstreamUrl: r.upstreamUrl,
	368	});
	369	continue;
	370	}
	371	const last = new Date(r.lastSyncedAt as any).getTime();
	372	const elapsedMin = (now.getTime() - last) / 60000;
	373	if (elapsedMin >= r.intervalMinutes) {
	374	due.push({
	375	id: r.id,
	376	repositoryId: r.repositoryId,
	377	upstreamUrl: r.upstreamUrl,
	378	});
	379	}
	380	}
	381	return due;
	382	} catch {
	383	return [];
	384	}
	385	}
	386
	387	/** Run sync for every due mirror. Returns summary counts. */
	388	export async function syncAllDue(): Promise<{
	389	total: number;
	390	ok: number;
	391	failed: number;
	392	}> {
	393	const due = await listDueMirrors();
	394	let ok = 0;
	395	let failed = 0;
	396	for (const m of due) {
	397	const r = await runMirrorSync(m.id);
	398	if (r.ok) ok++;
	399	else failed++;
	400	}
	401	return { total: due.length, ok, failed };
	402	}
	403
	404	// Suppress unused import warning for `and`.
	405	void and;
	406
	407	export const __internal = {
	408	MIRROR_REMOTE_NAME,
	409	FETCH_TIMEOUT_MS,
	410	};