Blame · Line-by-line history

`codebase-migrator.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

codebase-migrator.tsBlame694 lines · 1 contributor

Normal view Raw

b665cacClaude	1	/**
	2	* Codebase Migration Service — AI-powered one-click language/framework translation.
	3	*
	4	* Accepts a MigrationTarget (language swap, framework swap, or custom instruction),
	5	* analyses the repository, generates a Claude-driven translation plan, translates
	6	* up to 40 files, commits everything to a new branch, and opens a pull request.
	7	*
	8	* Job lifecycle: queued → analyzing → translating → committing → opening-pr → done
	9	* ↘ failed
	10	*
	11	* All jobs live in an in-memory Map, auto-purged 4 hours after completion.
	12	*/
	13
	14	import { join } from "path";
	15	import { mkdir, rm, writeFile, readFile } from "fs/promises";
	16	import { existsSync } from "fs";
	17	import { config } from "./config";
	18	import { getAnthropic, MODEL_SONNET, extractText, parseJsonResponse } from "./ai-client";
	19	import { db } from "../db";
	20	import { users, repositories, pullRequests } from "../db/schema";
	21	import { eq, and } from "drizzle-orm";
	22
	23	// ---------------------------------------------------------------------------
	24	// Types
	25	// ---------------------------------------------------------------------------
	26
	27	export type MigrationTarget =
	28	\| { type: "language"; from: string; to: string }
	29	\| { type: "framework"; from: string; to: string }
	30	\| { type: "custom"; description: string };
	31
	32	export interface MigrationJob {
	33	id: string;
	34	repoId: string;
	35	owner: string;
	36	repo: string;
	37	userId: string;
	38	target: MigrationTarget;
	39	status:
	40	\| "queued"
	41	\| "analyzing"
	42	\| "translating"
	43	\| "committing"
	44	\| "opening-pr"
	45	\| "done"
	46	\| "failed";
	47	progress: number; // 0-100
	48	currentFile?: string;
	49	branchName: string;
	50	prNumber?: number;
	51	error?: string;
	52	filesTotal: number;
	53	filesTranslated: number;
	54	startedAt: string;
	55	completedAt?: string;
	56	}
	57
	58	interface MigrationPlan {
	59	filesToTranslate: Array<{ from: string; to: string; notes: string }>;
	60	filesToSkip: string[];
	61	newFiles: Array<{ path: string; content: string }>;
	62	}
	63
	64	interface ResolvedRepo {
	65	ownerId: string;
	66	repoId: string;
	67	defaultBranch: string;
	68	diskPath: string;
	69	}
	70
	71	// ---------------------------------------------------------------------------
	72	// In-memory job store
	73	// ---------------------------------------------------------------------------
	74
	75	const migrationJobs = new Map<string, MigrationJob>();
	76
	77	// Sweep jobs older than 4 hours after completion.
	78	setInterval(() => {
	79	const cutoff = Date.now() - 4 * 60 * 60 * 1000;
	80	for (const [id, job] of migrationJobs) {
	81	if (
	82	(job.status === "done" \|\| job.status === "failed") &&
	83	job.completedAt &&
	84	new Date(job.completedAt).getTime() < cutoff
	85	) {
	86	migrationJobs.delete(id);
	87	}
	88	}
	89	}, 5 * 60 * 1000);
	90
	91	// ---------------------------------------------------------------------------
	92	// Rate limiting — 1 active migration per repo, 3 per user per day
	93	// ---------------------------------------------------------------------------
	94
	95	/** jobId → repoId for active jobs */
	96	const activeByRepo = new Map<string, string>(); // repoId → jobId
	97
	98	/** userId → [timestamp, ...] rolling daily window */
	99	const dailyCounts = new Map<string, number[]>();
	100
	101	function recordDailyUse(userId: string): boolean {
	102	const now = Date.now();
	103	const dayMs = 24 * 60 * 60 * 1000;
	104	const existing = (dailyCounts.get(userId) ?? []).filter(
	105	(ts) => now - ts < dayMs
	106	);
	107	if (existing.length >= 3) return false;
	108	existing.push(now);
	109	dailyCounts.set(userId, existing);
	110	return true;
	111	}
	112
	113	export function isRepoMigrating(repoId: string): boolean {
	114	const jobId = activeByRepo.get(repoId);
	115	if (!jobId) return false;
	116	const job = migrationJobs.get(jobId);
	117	if (!job) {
	118	activeByRepo.delete(repoId);
	119	return false;
	120	}
	121	if (job.status === "done" \|\| job.status === "failed") {
	122	activeByRepo.delete(repoId);
	123	return false;
	124	}
	125	return true;
	126	}
	127
	128	export function getJob(jobId: string): MigrationJob \| undefined {
	129	return migrationJobs.get(jobId);
	130	}
	131
	132	// ---------------------------------------------------------------------------
	133	// Git helpers (subprocess via Bun.spawn)
	134	// ---------------------------------------------------------------------------
	135
	136	async function git(
	137	args: string[],
	138	opts?: { cwd?: string }
	139	): Promise<{ stdout: string; stderr: string; exitCode: number }> {
	140	const proc = Bun.spawn(["git", ...args], {
	141	cwd: opts?.cwd,
	142	stdout: "pipe",
	143	stderr: "pipe",
	144	env: {
	145	...process.env,
	146	GIT_AUTHOR_NAME: "Gluecron Migration Bot",
	147	GIT_AUTHOR_EMAIL: "migration-bot@gluecron.com",
	148	GIT_COMMITTER_NAME: "Gluecron Migration Bot",
	149	GIT_COMMITTER_EMAIL: "migration-bot@gluecron.com",
	150	},
	151	});
	152	const [stdout, stderr] = await Promise.all([
	153	new Response(proc.stdout).text(),
	154	new Response(proc.stderr).text(),
	155	]);
	156	const exitCode = await proc.exited;
	157	return { stdout, stderr, exitCode };
	158	}
	159
	160	/** Return true if the buffer looks like a binary file (null byte in first 512 B) */
	161	function isBinaryContent(content: string): boolean {
	162	const sample = content.slice(0, 512);
	163	return sample.includes("\0");
	164	}
	165
	166	/** True if the path should always be skipped */
	167	function shouldSkipPath(path: string): boolean {
	168	const lower = path.toLowerCase();
	169	const skip = [
	170	"node_modules/",
	171	"dist/",
	172	".git/",
	173	".next/",
	174	"build/",
	175	"target/",
	176	"__pycache__/",
	177	".venv/",
	178	"vendor/",
	179	"package-lock.json",
	180	"yarn.lock",
	181	"pnpm-lock.yaml",
	182	"bun.lockb",
	183	"poetry.lock",
	184	"cargo.lock",
	185	"go.sum",
	186	"composer.lock",
	187	"gemfile.lock",
	188	];
	189	return skip.some((s) => lower.includes(s));
	190	}
	191
	192	// ---------------------------------------------------------------------------
	193	// Claude helpers
	194	// ---------------------------------------------------------------------------
	195
	196	function targetLabel(target: MigrationTarget): string {
	197	if (target.type === "language") return `${target.from} → ${target.to}`;
	198	if (target.type === "framework") return `${target.from} → ${target.to}`;
	199	return target.description;
	200	}
	201
	202	async function planMigration(
	203	fileList: string[],
	204	target: MigrationTarget
	205	): Promise<MigrationPlan \| null> {
	206	const anthropic = getAnthropic();
	207
	208	let goalDescription: string;
	209	if (target.type === "language") {
	210	goalDescription = `Convert all source code from ${target.from} to ${target.to}`;
	211	} else if (target.type === "framework") {
	212	goalDescription = `Migrate the codebase from ${target.from} framework to ${target.to} framework`;
	213	} else {
	214	goalDescription = target.description;
	215	}
	216
	217	const fileListStr = fileList.slice(0, 500).join("\n");
	218
	219	const prompt = `You are a migration expert. Given this repository's file list, create a migration plan.
	220
	221	Goal: ${goalDescription}
	222
	223	Files in repository:
	224	${fileListStr}
	225
	226	Return a JSON object (no markdown, no code fences, raw JSON only) with this exact shape:
	227	{
	228	"filesToTranslate": [
	229	{ "from": "src/index.ts", "to": "src/index.py", "notes": "convert Express handlers to Flask routes" }
	230	],
	231	"filesToSkip": ["package-lock.json", "node_modules/..."],
	232	"newFiles": [
	233	{ "path": "requirements.txt", "content": "flask==3.0.0\n..." }
	234	]
	235	}
	236
	237	Rules:
	238	- filesToTranslate: only source code files (no binaries, no lock files, no minified JS in dist/)
	239	- Cap filesToTranslate at 40 entries maximum
	240	- filesToSkip: lock files, binary assets, generated files that don't need translation
	241	- newFiles: new config/manifest files needed for the target (e.g. requirements.txt for Python, go.mod for Go)
	242	- Keep new file content concise and correct for the target stack
	243	- The "to" field should use the correct extension for the target language`;
	244
	245	try {
	246	const msg = await anthropic.messages.create({
	247	model: MODEL_SONNET,
	248	max_tokens: 4096,
	249	messages: [{ role: "user", content: prompt }],
	250	});
	251	const text = extractText(msg);
	252	const plan = parseJsonResponse<MigrationPlan>(text);
	253	if (!plan) return null;
	254	// Clamp to 40 files
	255	if (plan.filesToTranslate && plan.filesToTranslate.length > 40) {
	256	plan.filesToTranslate = plan.filesToTranslate.slice(0, 40);
	257	}
	258	return plan;
	259	} catch {
	260	return null;
	261	}
	262	}
	263
	264	async function translateFile(
	265	content: string,
	266	fromPath: string,
	267	target: MigrationTarget,
	268	notes: string
	269	): Promise<string \| null> {
	270	const anthropic = getAnthropic();
	271
	272	let instruction: string;
	273	if (target.type === "language") {
	274	instruction = `Translate this ${target.from} file to ${target.to}.${notes ? ` Notes: ${notes}` : ""}`;
	275	} else if (target.type === "framework") {
	276	instruction = `Migrate this file from ${target.from} to ${target.to}.${notes ? ` Notes: ${notes}` : ""}`;
	277	} else {
	278	instruction = `Apply this transformation: ${target.description}${notes ? `. Notes: ${notes}` : ""}`;
	279	}
	280
	281	const prompt = `${instruction}
	282
	283	Return ONLY the translated file content, no explanation, no code fences, no markdown. Start the output with the actual file content.
	284
	285	Original file (${fromPath}):
	286	${content}`;
	287
	288	try {
	289	const msg = await anthropic.messages.create({
	290	model: MODEL_SONNET,
	291	max_tokens: 8192,
	292	messages: [{ role: "user", content: prompt }],
	293	});
	294	return extractText(msg);
	295	} catch {
	296	return null;
	297	}
	298	}
	299
	300	// ---------------------------------------------------------------------------
	301	// DB helpers
	302	// ---------------------------------------------------------------------------
	303
	304	async function resolveRepo(
	305	ownerName: string,
	306	repoName: string
	307	): Promise<ResolvedRepo \| null> {
	308	try {
	309	const [ownerRow] = await db
	310	.select()
	311	.from(users)
	312	.where(eq(users.username, ownerName))
	313	.limit(1);
	314	if (!ownerRow) return null;
	315	const [repoRow] = await db
	316	.select()
	317	.from(repositories)
	318	.where(
	319	and(
	320	eq(repositories.ownerId, ownerRow.id),
	321	eq(repositories.name, repoName)
	322	)
	323	)
	324	.limit(1);
	325	if (!repoRow) return null;
	326	return {
	327	ownerId: ownerRow.id,
	328	repoId: repoRow.id,
	329	defaultBranch: repoRow.defaultBranch \|\| "main",
	330	diskPath: repoRow.diskPath,
	331	};
	332	} catch {
	333	return null;
	334	}
	335	}
	336
	337	async function insertPullRequest(params: {
	338	repositoryId: string;
	339	authorId: string;
	340	title: string;
	341	body: string;
	342	baseBranch: string;
	343	headBranch: string;
	344	}): Promise<number> {
	345	const [row] = await db
	346	.insert(pullRequests)
	347	.values({
	348	repositoryId: params.repositoryId,
	349	authorId: params.authorId,
	350	title: params.title,
	351	body: params.body,
	352	state: "open",
	353	baseBranch: params.baseBranch,
	354	headBranch: params.headBranch,
	355	isDraft: true,
	356	})
	357	.returning({ number: pullRequests.number });
	358	return row.number;
	359	}
	360
	361	// ---------------------------------------------------------------------------
	362	// Main migration pipeline
	363	// ---------------------------------------------------------------------------
	364
	365	async function runMigration(job: MigrationJob): Promise<void> {
	366	const worktreeBase = join(config.gitReposPath, ".migration-worktrees");
	367	const worktreePath = join(worktreeBase, job.id);
	368
	369	try {
	370	// ── 1. Resolve the repo ──────────────────────────────────────────────────
	371	job.status = "analyzing";
	372	job.progress = 5;
	373
	374	const resolved = await resolveRepo(job.owner, job.repo);
	375	if (!resolved) throw new Error("Repository not found");
	376
	377	const bareRepoPath = resolved.diskPath;
	378
	379	// ── 2. Get file list ─────────────────────────────────────────────────────
	380	const lsResult = await git(["ls-tree", "-r", "--name-only", "HEAD"], {
	381	cwd: bareRepoPath,
	382	});
	383	if (lsResult.exitCode !== 0) {
	384	// Empty repo
	385	throw new Error("Repository has no commits yet — nothing to migrate");
	386	}
	387
	388	const allFiles = lsResult.stdout
	389	.split("\n")
	390	.map((f) => f.trim())
	391	.filter(Boolean)
	392	.filter((f) => !shouldSkipPath(f));
	393
	394	if (allFiles.length === 0) {
	395	throw new Error("No translatable files found in the repository");
	396	}
	397
	398	job.progress = 10;
	399
	400	// ── 3. Plan the migration ────────────────────────────────────────────────
	401	const plan = await planMigration(allFiles, job.target);
	402	if (!plan) throw new Error("Failed to generate migration plan from Claude");
	403
	404	job.filesTotal = plan.filesToTranslate.length + plan.newFiles.length;
	405	job.progress = 20;
	406
	407	// ── 4. Create worktree ──────────────────────────────────────────────────
	408	await mkdir(worktreeBase, { recursive: true });
	409
	410	// git worktree add creates a linked working tree from the bare repo
	411	const wtResult = await git(
	412	["worktree", "add", "--no-checkout", worktreePath, "HEAD"],
	413	{ cwd: bareRepoPath }
	414	);
	415	if (wtResult.exitCode !== 0) {
	416	throw new Error(`Failed to create worktree: ${wtResult.stderr}`);
	417	}
	418
	419	// Checkout the default branch content into the worktree
	420	const checkoutResult = await git(["checkout", "-f", "HEAD", "--", "."], {
	421	cwd: worktreePath,
	422	});
	423	// Silently continue even if partial checkout — some files may not exist
	424
	425	// Create and switch to the migration branch
	426	const branchResult = await git(
	427	["checkout", "-b", job.branchName],
	428	{ cwd: worktreePath }
	429	);
	430	if (branchResult.exitCode !== 0) {
	431	throw new Error(`Failed to create branch: ${branchResult.stderr}`);
	432	}
	433
	434	// ── 5. Translate files ───────────────────────────────────────────────────
	435	job.status = "translating";
	436	job.progress = 25;
	437
	438	const progressPerFile = plan.filesToTranslate.length > 0
	439	? 50 / plan.filesToTranslate.length
	440	: 50;
	441
	442	for (let i = 0; i < plan.filesToTranslate.length; i++) {
	443	const entry = plan.filesToTranslate[i];
	444	job.currentFile = entry.from;
	445	job.filesTranslated = i;
	446
	447	// Read original file content
	448	let originalContent: string;
	449	try {
	450	const showResult = await git(
	451	["show", `HEAD:${entry.from}`],
	452	{ cwd: bareRepoPath }
	453	);
	454	if (showResult.exitCode !== 0) {
	455	// File doesn't exist or can't be read — skip
	456	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	457	continue;
	458	}
	459	originalContent = showResult.stdout;
	460	} catch {
	461	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	462	continue;
	463	}
	464
	465	// Skip binary files
	466	if (isBinaryContent(originalContent)) {
	467	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	468	continue;
	469	}
	470
	471	// Skip very large files (50KB)
	472	if (originalContent.length > 50 * 1024) {
	473	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	474	continue;
	475	}
	476
	477	// Translate via Claude
	478	const translated = await translateFile(
	479	originalContent,
	480	entry.from,
	481	job.target,
	482	entry.notes \|\| ""
	483	);
	484	if (!translated) {
	485	// Translation failed for this file — skip gracefully
	486	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	487	continue;
	488	}
	489
	490	// Write translated file to the destination path
	491	const destPath = join(worktreePath, entry.to);
	492	const destDir = destPath.substring(0, destPath.lastIndexOf("/"));
	493	if (destDir && destDir !== worktreePath) {
	494	await mkdir(destDir, { recursive: true });
	495	}
	496	await writeFile(destPath, translated, "utf-8");
	497
	498	// If the source and destination paths differ, remove the old file
	499	if (entry.from !== entry.to) {
	500	const srcPath = join(worktreePath, entry.from);
	501	if (existsSync(srcPath)) {
	502	try {
	503	await rm(srcPath);
	504	} catch {
	505	// Non-fatal
	506	}
	507	}
	508	}
	509
	510	job.filesTranslated = i + 1;
	511	job.progress = Math.round(25 + (i + 1) * progressPerFile);
	512	}
	513
	514	// Write new files (package.json, requirements.txt, etc.)
	515	for (const newFile of plan.newFiles) {
	516	const destPath = join(worktreePath, newFile.path);
	517	const destDir = destPath.substring(0, destPath.lastIndexOf("/"));
	518	if (destDir && destDir !== worktreePath) {
	519	await mkdir(destDir, { recursive: true });
	520	}
	521	await writeFile(destPath, newFile.content, "utf-8");
	522	job.filesTranslated = Math.min(
	523	job.filesTranslated + 1,
	524	job.filesTotal
	525	);
	526	}
	527
	528	job.currentFile = undefined;
	529	job.progress = 75;
	530
	531	// ── 6. Commit ────────────────────────────────────────────────────────────
	532	job.status = "committing";
	533
	534	const label = targetLabel(job.target);
	535	const addResult = await git(["add", "-A"], { cwd: worktreePath });
	536	if (addResult.exitCode !== 0) {
	537	throw new Error(`git add failed: ${addResult.stderr}`);
	538	}
	539
	540	// Check if there's anything to commit
	541	const statusResult = await git(
	542	["status", "--porcelain"],
	543	{ cwd: worktreePath }
	544	);
	545	if (!statusResult.stdout.trim()) {
	546	throw new Error("No changes were produced by the migration — all files may have been skipped");
	547	}
	548
	549	const commitMsg = `migrate: AI translation — ${label}\n\nAutomatically generated by Gluecron AI Codebase Migrator.\nFiles translated: ${job.filesTranslated}/${job.filesTotal}`;
	550	const commitResult = await git(
	551	["commit", "-m", commitMsg],
	552	{ cwd: worktreePath }
	553	);
	554	if (commitResult.exitCode !== 0) {
	555	throw new Error(`git commit failed: ${commitResult.stderr}`);
	556	}
	557
	558	job.progress = 85;
	559
	560	// ── 7. Push ──────────────────────────────────────────────────────────────
	561	// Push the new branch from the worktree into the bare repo
	562	const pushResult = await git(
	563	["push", bareRepoPath, `HEAD:refs/heads/${job.branchName}`],
	564	{ cwd: worktreePath }
	565	);
	566	if (pushResult.exitCode !== 0) {
	567	throw new Error(`git push failed: ${pushResult.stderr}`);
	568	}
	569
	570	job.progress = 92;
	571
	572	// ── 8. Open PR ───────────────────────────────────────────────────────────
	573	job.status = "opening-pr";
	574
	575	const prBody = [
	576	`## AI Codebase Migration — ${label}`,
	577	"",
	578	"This pull request was automatically generated by the Gluecron AI Codebase Migrator.",
	579	"",
	580	`Migration type: ${job.target.type}`,
	581	`Target: ${label}`,
	582	`Files translated: ${job.filesTranslated}`,
	583	`Total files processed: ${job.filesTotal}`,
	584	"",
	585	"> Review carefully before merging. AI translation is thorough but not perfect.",
	586	"> Test the migrated code in a staging environment before landing to main.",
	587	].join("\n");
	588
	589	const prTitle = `migrate: AI codebase migration — ${label}`;
	590
	591	const prNumber = await insertPullRequest({
	592	repositoryId: resolved.repoId,
	593	authorId: job.userId,
	594	title: prTitle,
	595	body: prBody,
	596	baseBranch: resolved.defaultBranch,
	597	headBranch: job.branchName,
	598	});
	599
	600	job.prNumber = prNumber;
	601	job.progress = 100;
	602	job.status = "done";
	603	job.completedAt = new Date().toISOString();
	604	} catch (err) {
	605	job.status = "failed";
	606	job.error = err instanceof Error ? err.message : "Unknown error";
	607	job.completedAt = new Date().toISOString();
	608	} finally {
	609	// Clean up the worktree regardless of success or failure
	610	try {
	611	if (existsSync(worktreePath)) {
	612	// Prune the worktree registration from the bare repo
	613	const resolved2 = await resolveRepo(job.owner, job.repo);
	614	if (resolved2) {
	615	await git(["worktree", "prune"], { cwd: resolved2.diskPath });
	616	}
	617	await rm(worktreePath, { recursive: true, force: true });
	618	}
	619	} catch {
	620	// Best-effort cleanup
	621	}
	622	// Release the repo lock
	623	activeByRepo.delete(job.repoId);
	624	}
	625	}
	626
	627	// ---------------------------------------------------------------------------
	628	// Public API
	629	// ---------------------------------------------------------------------------
	630
	631	export interface StartMigrationParams {
	632	owner: string;
	633	repo: string;
	634	repoId: string;
	635	userId: string;
	636	target: MigrationTarget;
	637	}
	638
	639	export async function startMigration(
	640	params: StartMigrationParams
	641	): Promise<{ ok: true; job: MigrationJob } \| { ok: false; error: string }> {
	642	// Rate limit: 1 active migration per repo
	643	if (isRepoMigrating(params.repoId)) {
	644	return {
	645	ok: false,
	646	error: "A migration is already in progress for this repository. Wait for it to finish.",
	647	};
	648	}
	649
	650	// Rate limit: 3 per user per day
	651	if (!recordDailyUse(params.userId)) {
	652	return {
	653	ok: false,
	654	error: "You have reached the daily limit of 3 migrations. Try again tomorrow.",
	655	};
	656	}
	657
	658	const jobId = crypto.randomUUID().replace(/-/g, "").slice(0, 16);
	659	const timestamp = Math.floor(Date.now() / 1000);
	660	let branchSuffix: string;
	661	if (params.target.type === "language") {
	662	branchSuffix = `${params.target.from.toLowerCase()}-to-${params.target.to.toLowerCase()}`;
	663	} else if (params.target.type === "framework") {
	664	branchSuffix = `${params.target.from.toLowerCase()}-to-${params.target.to.toLowerCase()}`;
	665	} else {
	666	branchSuffix = "custom";
	667	}
	668	// Sanitize branch name
	669	branchSuffix = branchSuffix.replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").slice(0, 40);
	670	const branchName = `migrate/${branchSuffix}-${timestamp}`;
	671
	672	const job: MigrationJob = {
	673	id: jobId,
	674	repoId: params.repoId,
	675	owner: params.owner,
	676	repo: params.repo,
	677	userId: params.userId,
	678	target: params.target,
	679	status: "queued",
	680	progress: 0,
	681	branchName,
	682	filesTotal: 0,
	683	filesTranslated: 0,
	684	startedAt: new Date().toISOString(),
	685	};
	686
	687	migrationJobs.set(jobId, job);
	688	activeByRepo.set(params.repoId, jobId);
	689
	690	// Fire and forget
	691	void runMigration(job);
	692
	693	return { ok: true, job };
	694	}