Blame · Line-by-line history

`import-bulk.tsx`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

import-bulk.tsxBlame464 lines · 1 contributor

Normal view Raw

14c3cc8Claude	1	/**
	2	* Bulk GitHub import — "paste my org + token → import everything".
	3	*
	4	* Owner flow for migrating many products at once. Reuses the single-repo
	5	* import logic from `src/lib/import-helper.ts` so the clone + DB insert
	6	* code path is identical to `/import`.
	7	*
	8	* Token never leaves this process: it's read from the form body, passed
	9	* to GitHub's API via `Authorization` header, and embedded in the git
	10	* clone URL only at the moment of spawning `git`. Results never contain
	11	* the token — `scrubSecrets()` in the helper redacts it before display.
	12	*/
	13
	14	import { Hono } from "hono";
	15	import { Layout } from "../views/layout";
	16	import { softAuth, requireAuth } from "../middleware/auth";
	17	import type { AuthEnv } from "../middleware/auth";
	18	import {
	19	sanitizeRepoName,
	20	importOneRepo,
	21	type ImportOneRepoResult,
	22	} from "../lib/import-helper";
	23
	24	const importBulkRoutes = new Hono<AuthEnv>();
	25
	26	importBulkRoutes.use("*", softAuth);
	27
	28	// Hard limits to keep a single request bounded.
	29	const MAX_REPOS = 200;
	30	const MAX_REPO_SIZE_KB = 500 * 1024; // 500 MB in KB (GitHub reports size in KB)
	31	const GITHUB_PER_PAGE = 100;
	32
	33	interface GitHubRepo {
	34	name: string;
	35	full_name: string;
	36	description: string \| null;
	37	private: boolean;
	38	clone_url: string;
	39	default_branch: string;
	40	fork: boolean;
	41	size: number; // KB
	42	}
	43
	44	type Visibility = "public" \| "private" \| "both";
	45
	46	/**
	47	* Paginate the GitHub "list org repos" endpoint. Caps at MAX_REPOS so a
	48	* single request can't fan out indefinitely. Throws on non-2xx so the
	49	* caller can surface a friendly error.
	50	*/
	51	async function fetchOrgRepos(
	52	org: string,
	53	token: string
	54	): Promise<GitHubRepo[]> {
	55	const headers: Record<string, string> = {
	56	Accept: "application/vnd.github.v3+json",
	57	"User-Agent": "gluecron/1.0",
	58	Authorization: `Bearer ${token}`,
	59	};
	60
	61	const repos: GitHubRepo[] = [];
	62	let page = 1;
	63	while (repos.length < MAX_REPOS) {
	64	const url = `https://api.github.com/orgs/${encodeURIComponent(
	65	org
	66	)}/repos?per_page=${GITHUB_PER_PAGE}&page=${page}&type=all`;
	67	const res = await fetch(url, { headers });
	68	if (!res.ok) {
	69	// Never echo the token. Include only the status + first slice of body.
	70	const errBody = (await res.text()).slice(0, 200);
	71	throw new Error(`GitHub API error (${res.status}): ${errBody}`);
	72	}
	73	const batch = (await res.json()) as GitHubRepo[];
	74	if (!Array.isArray(batch) \|\| batch.length === 0) break;
	75	repos.push(...batch);
	76	if (batch.length < GITHUB_PER_PAGE) break;
	77	page++;
	78	if (page > 10) break; // hard page ceiling: 1000 entries, we cap earlier anyway
	79	}
	80	return repos.slice(0, MAX_REPOS);
	81	}
	82
	83	function matchesVisibility(repo: GitHubRepo, v: Visibility): boolean {
	84	if (v === "both") return true;
	85	if (v === "public") return repo.private === false;
	86	if (v === "private") return repo.private === true;
	87	return true;
	88	}
	89
	90	// ─── FORM PAGE ───────────────────────────────────────────────
	91
	92	importBulkRoutes.get("/import/bulk", requireAuth, async (c) => {
	93	const user = c.get("user")!;
	94	const error = c.req.query("error");
	95
	96	return c.html(
	97	<Layout title="Bulk import from GitHub" user={user}>
	98	<div style="max-width: 720px">
	99	<h2 style="margin-bottom: 4px">Bulk import from GitHub</h2>
	100	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 20px">
	101	Paste a GitHub org + personal access token. Gluecron will clone
	102	every repo into your namespace as a mirror.
	103	</p>
	104
	105	{error && <div class="auth-error">{decodeURIComponent(error)}</div>}
	106
	107	<div style="background: var(--bg-secondary); border: 1px solid var(--border); border-radius: var(--radius); padding: 16px 18px; margin-bottom: 20px; font-size: 13px; color: var(--text-muted)">
	108	<strong style="color: var(--text)">What this does</strong>
	109	<ul style="margin: 8px 0 0 18px; line-height: 1.6">
	110	<li>
	111	Lists every repo in the org via the GitHub API
	112	(<code>/orgs/{"{org}"}/repos</code>, paginated).
	113	</li>
	114	<li>
	115	Clones each one as a bare mirror into your gluecron account
	116	(<code>{user.username}/{"{repo}"}</code>).
	117	</li>
	118	<li>
	119	Reports per-repo success / failure / skipped-if-exists at
	120	the end. One failure does not abort the batch.
	121	</li>
	122	<li>
	123	Hard caps: {MAX_REPOS} repos per run, 500MB per repo.
	124	</li>
	125	</ul>
	126	</div>
	127
	128	<form method="POST" action="/import/bulk">
	129	<div class="form-group">
	130	<label style="display:block; margin-bottom:4px; font-size:13px">
	131	GitHub org
	132	</label>
	133	<input
	134	type="text"
	135	name="githubOrg"
	136	required
	137	placeholder="my-company"
	138	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; width: 100%"
	139	/>
	140	</div>
	141
	142	<div class="form-group">
	143	<label style="display:block; margin-bottom:4px; font-size:13px">
	144	GitHub personal access token (<code>repo:read</code> scope)
	145	</label>
	146	<input
	147	type="password"
	148	name="githubToken"
	149	required
	150	placeholder="ghp_xxxxxxxxxxxx"
	151	autocomplete="off"
	152	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; font-family: var(--font-mono); width: 100%"
	153	/>
	154	</div>
	155
	156	<div class="form-group">
	157	<label style="display:block; margin-bottom:4px; font-size:13px">
	158	Visibility filter
	159	</label>
	160	<select
	161	name="visibility"
	162	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; width: 100%"
	163	>
	164	<option value="both" selected>Both (public + private)</option>
	165	<option value="public">Public only</option>
	166	<option value="private">Private only</option>
	167	</select>
	168	</div>
	169
	170	<div class="form-group" style="margin: 12px 0">
	171	<label style="display:flex; align-items:center; gap:8px; font-size:13px">
	172	<input type="checkbox" name="dryRun" value="1" checked />
	173	Dry run — preview the list without cloning
	174	</label>
	175	</div>
	176
	177	<button type="submit" class="btn btn-primary">
	178	Run bulk import
	179	</button>
	180	<a href="/import" class="btn" style="margin-left: 8px">
	181	Back to /import
	182	</a>
	183	</form>
	184	</div>
	185	</Layout>
	186	);
	187	});
	188
	189	// ─── POST HANDLER ────────────────────────────────────────────
	190
	191	importBulkRoutes.post("/import/bulk", requireAuth, async (c) => {
	192	const user = c.get("user")!;
	193	const body = await c.req.parseBody();
	194
	195	const githubOrg = String(body.githubOrg \|\| "").trim();
	196	const githubToken = String(body.githubToken \|\| "").trim();
	197	const visibilityRaw = String(body.visibility \|\| "both").trim();
	198	const visibility: Visibility =
	199	visibilityRaw === "public" \|\| visibilityRaw === "private"
	200	? (visibilityRaw as Visibility)
	201	: "both";
	202	const dryRun = Boolean(body.dryRun); // unchecked box = undefined = false
	203
	204	if (!githubOrg) {
	205	return c.redirect("/import/bulk?error=GitHub+org+is+required");
	206	}
	207	if (!githubToken) {
	208	return c.redirect(
	209	"/import/bulk?error=GitHub+token+is+required+%28repo%3Aread+scope%29"
	210	);
	211	}
	212
	213	// Validate the token has at least read access before we start cloning.
	214	// `GET /user` is the cheapest call that requires a valid token. We also
	215	// inspect the `X-OAuth-Scopes` header so we can warn early if the token
	216	// is missing `repo`/`repo:read`.
	217	try {
	218	const userRes = await fetch("https://api.github.com/user", {
	219	headers: {
	220	Accept: "application/vnd.github.v3+json",
	221	"User-Agent": "gluecron/1.0",
	222	Authorization: `Bearer ${githubToken}`,
	223	},
	224	});
	225	if (!userRes.ok) {
	226	return c.redirect(
	227	`/import/bulk?error=${encodeURIComponent(
	228	`Invalid GitHub token (${userRes.status}). Check scope repo:read.`
	229	)}`
	230	);
	231	}
	232	const scopes = (userRes.headers.get("x-oauth-scopes") \|\| "").toLowerCase();
	233	if (
	234	scopes &&
	235	!scopes.includes("repo") &&
	236	!scopes.includes("public_repo")
	237	) {
	238	return c.redirect(
	239	`/import/bulk?error=${encodeURIComponent(
	240	"Token is missing repo:read scope. Regenerate with repo (or public_repo) checked."
	241	)}`
	242	);
	243	}
	244	} catch (err) {
	245	// Network-level failure talking to GitHub. Don't leak err details.
	246	return c.redirect(
	247	"/import/bulk?error=Could+not+reach+GitHub+to+validate+the+token"
	248	);
	249	}
	250
	251	// Pull the repo list.
	252	let allRepos: GitHubRepo[];
	253	try {
	254	allRepos = await fetchOrgRepos(githubOrg, githubToken);
	255	} catch (err) {
	256	const msg = (err as Error).message \|\| "Unknown error";
	257	return c.redirect(
	258	`/import/bulk?error=${encodeURIComponent(msg).slice(0, 400)}`
	259	);
	260	}
	261
	262	if (allRepos.length === 0) {
	263	return c.redirect(
	264	`/import/bulk?error=${encodeURIComponent(
	265	`No repos visible for org "${githubOrg}" with this token.`
	266	)}`
	267	);
	268	}
	269
	270	// Apply visibility filter + size cap; track why things were skipped.
	271	const candidates: GitHubRepo[] = [];
	272	const oversized: { name: string; sizeKB: number }[] = [];
	273	for (const r of allRepos) {
	274	if (!matchesVisibility(r, visibility)) continue;
	275	if (typeof r.size === "number" && r.size > MAX_REPO_SIZE_KB) {
	276	oversized.push({ name: r.name, sizeKB: r.size });
	277	continue;
	278	}
	279	candidates.push(r);
	280	}
	281
	282	// Dry run: render a preview + counts, never touch disk or DB.
	283	if (dryRun) {
	284	return c.html(
	285	<Layout title="Bulk import preview" user={user}>
	286	<div style="max-width: 820px">
	287	<h2 style="margin-bottom: 4px">Bulk import — dry-run preview</h2>
	288	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 16px">
	289	Org <code>{githubOrg}</code> · visibility <code>{visibility}</code>
	290	{" · "}
	291	{candidates.length} repo(s) would be imported
	292	{oversized.length > 0 ? `, ${oversized.length} skipped (>500MB)` : ""}
	293	.
	294	</p>
	295
	296	<ResultsTable
	297	rows={candidates.map((r) => ({
	298	name: sanitizeRepoName(r.name),
	299	status: "dry-run",
	300	notes: `${r.private ? "private" : "public"} · ${(
	301	r.size / 1024
	302	).toFixed(1)} MB`,
	303	}))}
	304	/>
	305
	306	{oversized.length > 0 && (
	307	<>
	308	<h3 style="margin-top:24px">Skipped — over 500MB</h3>
	309	<ResultsTable
	310	rows={oversized.map((r) => ({
	311	name: sanitizeRepoName(r.name),
	312	status: "too-large",
	313	notes: `${(r.sizeKB / 1024).toFixed(1)} MB`,
	314	}))}
	315	/>
	316	</>
	317	)}
	318
	319	<div style="margin-top:20px; padding:12px 14px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); font-size:13px">
	320	Looks good? Go back and uncheck <em>Dry run</em> to actually
	321	import.
	322	</div>
	323
	324	<div style="margin-top:16px">
	325	<a href="/import/bulk" class="btn btn-primary">
	326	Back to form
	327	</a>
	328	</div>
	329	</div>
	330	</Layout>
	331	);
	332	}
	333
	334	// Real run: clone each candidate sequentially. Collect results.
	335	const results: ImportOneRepoResult[] = [];
	336	for (const r of candidates) {
	337	// eslint-disable-next-line no-await-in-loop
	338	const res = await importOneRepo({
	339	cloneUrl: r.clone_url,
	340	targetName: r.name,
	341	ownerId: user.id,
	342	ownerUsername: user.username,
	343	token: githubToken,
	344	description: r.description,
	345	isPrivate: r.private,
	346	defaultBranch: r.default_branch,
	347	});
	348	results.push(res);
	349	}
	350
	351	for (const o of oversized) {
	352	results.push({
	353	status: "failed",
	354	name: sanitizeRepoName(o.name),
	355	notes: `Skipped — over 500MB (${(o.sizeKB / 1024).toFixed(1)} MB)`,
	356	});
	357	}
	358
	359	const counts = results.reduce(
	360	(acc, r) => {
	361	acc[r.status] = (acc[r.status] \|\| 0) + 1;
	362	return acc;
	363	},
	364	{} as Record<string, number>
	365	);
	366
	367	return c.html(
	368	<Layout title="Bulk import results" user={user}>
	369	<div style="max-width: 820px">
	370	<h2 style="margin-bottom: 4px">Bulk import — results</h2>
	371	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 16px">
	372	Org <code>{githubOrg}</code>: {counts["success"] \|\| 0} imported,
	373	{" "}
	374	{counts["skipped-exists"] \|\| 0} skipped (exists),
	375	{" "}
	376	{counts["failed"] \|\| 0} failed.
	377	</p>
	378
	379	<ResultsTable rows={results} />
	380
	381	<div style="margin-top:20px; display:flex; gap:8px">
	382	<a href={`/${user.username}`} class="btn btn-primary">
	383	View my repositories
	384	</a>
	385	<a href="/import/bulk" class="btn">
	386	Run another import
	387	</a>
	388	</div>
	389	</div>
	390	</Layout>
	391	);
	392	});
	393
	394	// ─── COMPONENTS ──────────────────────────────────────────────
	395
	396	function ResultsTable({
	397	rows,
	398	}: {
	399	rows: { name: string; status: string; notes: string }[];
	400	}) {
	401	if (rows.length === 0) {
	402	return (
	403	<div style="padding:14px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); font-size:13px; color:var(--text-muted)">
	404	No rows.
	405	</div>
	406	);
	407	}
	408	return (
	409	<table
	410	style="width:100%; border-collapse:collapse; font-size:13px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); overflow:hidden"
	411	>
	412	<thead>
	413	<tr style="background:var(--bg); text-align:left">
	414	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	415	Name
	416	</th>
	417	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	418	Status
	419	</th>
	420	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	421	Notes
	422	</th>
	423	</tr>
	424	</thead>
	425	<tbody>
	426	{rows.map((r) => (
	427	<tr>
	428	<td style="padding:6px 12px; border-bottom:1px solid var(--border); font-family:var(--font-mono)">
	429	{r.name}
	430	</td>
	431	<td style="padding:6px 12px; border-bottom:1px solid var(--border)">
	432	<StatusBadge status={r.status} />
	433	</td>
	434	<td style="padding:6px 12px; border-bottom:1px solid var(--border); color:var(--text-muted)">
	435	{r.notes}
	436	</td>
	437	</tr>
	438	))}
	439	</tbody>
	440	</table>
	441	);
	442	}
	443
	444	function StatusBadge({ status }: { status: string }) {
	445	const color =
	446	status === "success"
	447	? "#3fb950"
	448	: status === "skipped-exists"
	449	? "#f0b429"
	450	: status === "dry-run"
	451	? "#58a6ff"
	452	: status === "too-large"
	453	? "#f0b429"
	454	: "#f85149";
	455	return (
	456	<span
	457	style={`display:inline-block; padding:2px 8px; border-radius:10px; background:${color}22; color:${color}; font-size:12px; font-weight:600`}
	458	>
	459	{status}
	460	</span>
	461	);
	462	}
	463
	464	export default importBulkRoutes;