Blame · Line-by-line history

`import-bulk.tsx`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

import-bulk.tsxBlame467 lines · 2 contributors

Normal view Raw

14c3cc8Claude	1	/**
	2	* Bulk GitHub import — "paste my org + token → import everything".
	3	*
	4	* Owner flow for migrating many products at once. Reuses the single-repo
	5	* import logic from `src/lib/import-helper.ts` so the clone + DB insert
	6	* code path is identical to `/import`.
	7	*
	8	* Token never leaves this process: it's read from the form body, passed
	9	* to GitHub's API via `Authorization` header, and embedded in the git
	10	* clone URL only at the moment of spawning `git`. Results never contain
	11	* the token — `scrubSecrets()` in the helper redacts it before display.
	12	*/
	13
	14	import { Hono } from "hono";
	15	import { Layout } from "../views/layout";
	16	import { softAuth, requireAuth } from "../middleware/auth";
	17	import type { AuthEnv } from "../middleware/auth";
	18	import {
	19	sanitizeRepoName,
	20	importOneRepo,
	21	type ImportOneRepoResult,
	22	} from "../lib/import-helper";
	23
	24	const importBulkRoutes = new Hono<AuthEnv>();
	25
	26	importBulkRoutes.use("*", softAuth);
	27
	28	// Hard limits to keep a single request bounded.
	29	const MAX_REPOS = 200;
	30	const MAX_REPO_SIZE_KB = 500 * 1024; // 500 MB in KB (GitHub reports size in KB)
	31	const GITHUB_PER_PAGE = 100;
	32
	33	interface GitHubRepo {
	34	name: string;
	35	full_name: string;
	36	description: string \| null;
	37	private: boolean;
	38	clone_url: string;
	39	default_branch: string;
	40	fork: boolean;
	41	size: number; // KB
	42	}
	43
	44	type Visibility = "public" \| "private" \| "both";
	45
	46	/**
	47	* Paginate the GitHub "list org repos" endpoint. Caps at MAX_REPOS so a
	48	* single request can't fan out indefinitely. Throws on non-2xx so the
	49	* caller can surface a friendly error.
	50	*/
	51	async function fetchOrgRepos(
	52	org: string,
	53	token: string
	54	): Promise<GitHubRepo[]> {
	55	const headers: Record<string, string> = {
	56	Accept: "application/vnd.github.v3+json",
	57	"User-Agent": "gluecron/1.0",
	58	Authorization: `Bearer ${token}`,
	59	};
	60
	61	const repos: GitHubRepo[] = [];
	62	let page = 1;
	63	while (repos.length < MAX_REPOS) {
	64	const url = `https://api.github.com/orgs/${encodeURIComponent(
	65	org
	66	)}/repos?per_page=${GITHUB_PER_PAGE}&page=${page}&type=all`;
	67	const res = await fetch(url, { headers });
	68	if (!res.ok) {
	69	// Never echo the token. Include only the status + first slice of body.
	70	const errBody = (await res.text()).slice(0, 200);
	71	throw new Error(`GitHub API error (${res.status}): ${errBody}`);
	72	}
	73	const batch = (await res.json()) as GitHubRepo[];
	74	if (!Array.isArray(batch) \|\| batch.length === 0) break;
	75	repos.push(...batch);
	76	if (batch.length < GITHUB_PER_PAGE) break;
	77	page++;
	78	if (page > 10) break; // hard page ceiling: 1000 entries, we cap earlier anyway
	79	}
	80	return repos.slice(0, MAX_REPOS);
	81	}
	82
	83	function matchesVisibility(repo: GitHubRepo, v: Visibility): boolean {
	84	if (v === "both") return true;
	85	if (v === "public") return repo.private === false;
	86	if (v === "private") return repo.private === true;
	87	return true;
	88	}
	89
	90	// ─── FORM PAGE ───────────────────────────────────────────────
	91
	92	importBulkRoutes.get("/import/bulk", requireAuth, async (c) => {
	93	const user = c.get("user")!;
	94	const error = c.req.query("error");
	95
	96	return c.html(
	97	<Layout title="Bulk import from GitHub" user={user}>
	98	<div style="max-width: 720px">
	99	<h2 style="margin-bottom: 4px">Bulk import from GitHub</h2>
	100	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 20px">
	101	Paste a GitHub org + personal access token. Gluecron will clone
	102	every repo into your namespace as a mirror.
	103	</p>
	104
	105	{error && <div class="auth-error">{decodeURIComponent(error)}</div>}
	106
	107	<div style="background: var(--bg-secondary); border: 1px solid var(--border); border-radius: var(--radius); padding: 16px 18px; margin-bottom: 20px; font-size: 13px; color: var(--text-muted)">
	108	<strong style="color: var(--text)">What this does</strong>
	109	<ul style="margin: 8px 0 0 18px; line-height: 1.6">
	110	<li>
	111	Lists every repo in the org via the GitHub API
	112	(<code>/orgs/{"{org}"}/repos</code>, paginated).
	113	</li>
	114	<li>
	115	Clones each one as a bare mirror into your gluecron account
	116	(<code>{user.username}/{"{repo}"}</code>).
	117	</li>
	118	<li>
	119	Reports per-repo success / failure / skipped-if-exists at
	120	the end. One failure does not abort the batch.
	121	</li>
	122	<li>
	123	Hard caps: {MAX_REPOS} repos per run, 500MB per repo.
	124	</li>
	125	</ul>
	126	</div>
	127
	128	<form method="POST" action="/import/bulk">
	129	<div class="form-group">
	130	<label style="display:block; margin-bottom:4px; font-size:13px">
	131	GitHub org
	132	</label>
	133	<input
	134	type="text"
	135	name="githubOrg"
	136	required
	137	placeholder="my-company"
2c3ba6ecopilot-swe-agent[bot]	138	aria-label="GitHub org"
14c3cc8Claude	139	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; width: 100%"
	140	/>
	141	</div>
	142
	143	<div class="form-group">
	144	<label style="display:block; margin-bottom:4px; font-size:13px">
	145	GitHub personal access token (<code>repo:read</code> scope)
	146	</label>
	147	<input
	148	type="password"
	149	name="githubToken"
	150	required
	151	placeholder="ghp_xxxxxxxxxxxx"
	152	autocomplete="off"
2c3ba6ecopilot-swe-agent[bot]	153	aria-label="GitHub personal access token"
14c3cc8Claude	154	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; font-family: var(--font-mono); width: 100%"
	155	/>
	156	</div>
	157
	158	<div class="form-group">
	159	<label style="display:block; margin-bottom:4px; font-size:13px">
	160	Visibility filter
	161	</label>
	162	<select
	163	name="visibility"
2c3ba6ecopilot-swe-agent[bot]	164	aria-label="Visibility filter"
14c3cc8Claude	165	style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; width: 100%"
	166	>
	167	<option value="both" selected>Both (public + private)</option>
	168	<option value="public">Public only</option>
	169	<option value="private">Private only</option>
	170	</select>
	171	</div>
	172
	173	<div class="form-group" style="margin: 12px 0">
	174	<label style="display:flex; align-items:center; gap:8px; font-size:13px">
	175	<input type="checkbox" name="dryRun" value="1" checked />
	176	Dry run — preview the list without cloning
	177	</label>
	178	</div>
	179
	180	<button type="submit" class="btn btn-primary">
	181	Run bulk import
	182	</button>
	183	<a href="/import" class="btn" style="margin-left: 8px">
	184	Back to /import
	185	</a>
	186	</form>
	187	</div>
	188	</Layout>
	189	);
	190	});
	191
	192	// ─── POST HANDLER ────────────────────────────────────────────
	193
	194	importBulkRoutes.post("/import/bulk", requireAuth, async (c) => {
	195	const user = c.get("user")!;
	196	const body = await c.req.parseBody();
	197
	198	const githubOrg = String(body.githubOrg \|\| "").trim();
	199	const githubToken = String(body.githubToken \|\| "").trim();
	200	const visibilityRaw = String(body.visibility \|\| "both").trim();
	201	const visibility: Visibility =
	202	visibilityRaw === "public" \|\| visibilityRaw === "private"
	203	? (visibilityRaw as Visibility)
	204	: "both";
	205	const dryRun = Boolean(body.dryRun); // unchecked box = undefined = false
	206
	207	if (!githubOrg) {
	208	return c.redirect("/import/bulk?error=GitHub+org+is+required");
	209	}
	210	if (!githubToken) {
	211	return c.redirect(
	212	"/import/bulk?error=GitHub+token+is+required+%28repo%3Aread+scope%29"
	213	);
	214	}
	215
	216	// Validate the token has at least read access before we start cloning.
	217	// `GET /user` is the cheapest call that requires a valid token. We also
	218	// inspect the `X-OAuth-Scopes` header so we can warn early if the token
	219	// is missing `repo`/`repo:read`.
	220	try {
	221	const userRes = await fetch("https://api.github.com/user", {
	222	headers: {
	223	Accept: "application/vnd.github.v3+json",
	224	"User-Agent": "gluecron/1.0",
	225	Authorization: `Bearer ${githubToken}`,
	226	},
	227	});
	228	if (!userRes.ok) {
	229	return c.redirect(
	230	`/import/bulk?error=${encodeURIComponent(
	231	`Invalid GitHub token (${userRes.status}). Check scope repo:read.`
	232	)}`
	233	);
	234	}
	235	const scopes = (userRes.headers.get("x-oauth-scopes") \|\| "").toLowerCase();
	236	if (
	237	scopes &&
	238	!scopes.includes("repo") &&
	239	!scopes.includes("public_repo")
	240	) {
	241	return c.redirect(
	242	`/import/bulk?error=${encodeURIComponent(
	243	"Token is missing repo:read scope. Regenerate with repo (or public_repo) checked."
	244	)}`
	245	);
	246	}
	247	} catch (err) {
	248	// Network-level failure talking to GitHub. Don't leak err details.
	249	return c.redirect(
	250	"/import/bulk?error=Could+not+reach+GitHub+to+validate+the+token"
	251	);
	252	}
	253
	254	// Pull the repo list.
	255	let allRepos: GitHubRepo[];
	256	try {
	257	allRepos = await fetchOrgRepos(githubOrg, githubToken);
	258	} catch (err) {
	259	const msg = (err as Error).message \|\| "Unknown error";
	260	return c.redirect(
	261	`/import/bulk?error=${encodeURIComponent(msg).slice(0, 400)}`
	262	);
	263	}
	264
	265	if (allRepos.length === 0) {
	266	return c.redirect(
	267	`/import/bulk?error=${encodeURIComponent(
	268	`No repos visible for org "${githubOrg}" with this token.`
	269	)}`
	270	);
	271	}
	272
	273	// Apply visibility filter + size cap; track why things were skipped.
	274	const candidates: GitHubRepo[] = [];
	275	const oversized: { name: string; sizeKB: number }[] = [];
	276	for (const r of allRepos) {
	277	if (!matchesVisibility(r, visibility)) continue;
	278	if (typeof r.size === "number" && r.size > MAX_REPO_SIZE_KB) {
	279	oversized.push({ name: r.name, sizeKB: r.size });
	280	continue;
	281	}
	282	candidates.push(r);
	283	}
	284
	285	// Dry run: render a preview + counts, never touch disk or DB.
	286	if (dryRun) {
	287	return c.html(
	288	<Layout title="Bulk import preview" user={user}>
	289	<div style="max-width: 820px">
	290	<h2 style="margin-bottom: 4px">Bulk import — dry-run preview</h2>
	291	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 16px">
	292	Org <code>{githubOrg}</code> · visibility <code>{visibility}</code>
	293	{" · "}
	294	{candidates.length} repo(s) would be imported
	295	{oversized.length > 0 ? `, ${oversized.length} skipped (>500MB)` : ""}
	296	.
	297	</p>
	298
	299	<ResultsTable
	300	rows={candidates.map((r) => ({
	301	name: sanitizeRepoName(r.name),
	302	status: "dry-run",
	303	notes: `${r.private ? "private" : "public"} · ${(
	304	r.size / 1024
	305	).toFixed(1)} MB`,
	306	}))}
	307	/>
	308
	309	{oversized.length > 0 && (
	310	<>
	311	<h3 style="margin-top:24px">Skipped — over 500MB</h3>
	312	<ResultsTable
	313	rows={oversized.map((r) => ({
	314	name: sanitizeRepoName(r.name),
	315	status: "too-large",
	316	notes: `${(r.sizeKB / 1024).toFixed(1)} MB`,
	317	}))}
	318	/>
	319	</>
	320	)}
	321
	322	<div style="margin-top:20px; padding:12px 14px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); font-size:13px">
	323	Looks good? Go back and uncheck <em>Dry run</em> to actually
	324	import.
	325	</div>
	326
	327	<div style="margin-top:16px">
	328	<a href="/import/bulk" class="btn btn-primary">
	329	Back to form
	330	</a>
	331	</div>
	332	</div>
	333	</Layout>
	334	);
	335	}
	336
	337	// Real run: clone each candidate sequentially. Collect results.
	338	const results: ImportOneRepoResult[] = [];
	339	for (const r of candidates) {
	340	// eslint-disable-next-line no-await-in-loop
	341	const res = await importOneRepo({
	342	cloneUrl: r.clone_url,
	343	targetName: r.name,
	344	ownerId: user.id,
	345	ownerUsername: user.username,
	346	token: githubToken,
	347	description: r.description,
	348	isPrivate: r.private,
	349	defaultBranch: r.default_branch,
	350	});
	351	results.push(res);
	352	}
	353
	354	for (const o of oversized) {
	355	results.push({
	356	status: "failed",
	357	name: sanitizeRepoName(o.name),
	358	notes: `Skipped — over 500MB (${(o.sizeKB / 1024).toFixed(1)} MB)`,
	359	});
	360	}
	361
	362	const counts = results.reduce(
	363	(acc, r) => {
	364	acc[r.status] = (acc[r.status] \|\| 0) + 1;
	365	return acc;
	366	},
	367	{} as Record<string, number>
	368	);
	369
	370	return c.html(
	371	<Layout title="Bulk import results" user={user}>
	372	<div style="max-width: 820px">
	373	<h2 style="margin-bottom: 4px">Bulk import — results</h2>
	374	<p style="color: var(--text-muted); font-size: 14px; margin-bottom: 16px">
	375	Org <code>{githubOrg}</code>: {counts["success"] \|\| 0} imported,
	376	{" "}
	377	{counts["skipped-exists"] \|\| 0} skipped (exists),
	378	{" "}
	379	{counts["failed"] \|\| 0} failed.
	380	</p>
	381
	382	<ResultsTable rows={results} />
	383
	384	<div style="margin-top:20px; display:flex; gap:8px">
	385	<a href={`/${user.username}`} class="btn btn-primary">
	386	View my repositories
	387	</a>
	388	<a href="/import/bulk" class="btn">
	389	Run another import
	390	</a>
	391	</div>
	392	</div>
	393	</Layout>
	394	);
	395	});
	396
	397	// ─── COMPONENTS ──────────────────────────────────────────────
	398
	399	function ResultsTable({
	400	rows,
	401	}: {
	402	rows: { name: string; status: string; notes: string }[];
	403	}) {
	404	if (rows.length === 0) {
	405	return (
	406	<div style="padding:14px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); font-size:13px; color:var(--text-muted)">
	407	No rows.
	408	</div>
	409	);
	410	}
	411	return (
	412	<table
	413	style="width:100%; border-collapse:collapse; font-size:13px; background:var(--bg-secondary); border:1px solid var(--border); border-radius:var(--radius); overflow:hidden"
	414	>
	415	<thead>
	416	<tr style="background:var(--bg); text-align:left">
	417	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	418	Name
	419	</th>
	420	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	421	Status
	422	</th>
	423	<th style="padding:8px 12px; border-bottom:1px solid var(--border)">
	424	Notes
	425	</th>
	426	</tr>
	427	</thead>
	428	<tbody>
	429	{rows.map((r) => (
	430	<tr>
	431	<td style="padding:6px 12px; border-bottom:1px solid var(--border); font-family:var(--font-mono)">
	432	{r.name}
	433	</td>
	434	<td style="padding:6px 12px; border-bottom:1px solid var(--border)">
	435	<StatusBadge status={r.status} />
	436	</td>
	437	<td style="padding:6px 12px; border-bottom:1px solid var(--border); color:var(--text-muted)">
	438	{r.notes}
	439	</td>
	440	</tr>
	441	))}
	442	</tbody>
	443	</table>
	444	);
	445	}
	446
	447	function StatusBadge({ status }: { status: string }) {
	448	const color =
	449	status === "success"
	450	? "#3fb950"
	451	: status === "skipped-exists"
	452	? "#f0b429"
	453	: status === "dry-run"
	454	? "#58a6ff"
	455	: status === "too-large"
	456	? "#f0b429"
	457	: "#f85149";
	458	return (
	459	<span
	460	style={`display:inline-block; padding:2px 8px; border-radius:10px; background:${color}22; color:${color}; font-size:12px; font-weight:600`}
	461	>
	462	{status}
	463	</span>
	464	);
	465	}
	466
	467	export default importBulkRoutes;