Blame · Line-by-line history

`ai-incident.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-incident.tsBlame393 lines · 1 contributor

Normal view Raw

1e162a8Claude	1	/**
	2	* Block D4 — AI Incident Responder.
	3	*
	4	* When a deployment fails, this module automatically opens an issue with an
	5	* AI-generated root-cause analysis. Invoked by the post-receive hook (from
	6	* `triggerCrontechDeploy`) whenever the Crontech deploy call returns a non-2xx
	7	* response or throws. Also retriggerable via the deployments route.
	8	*
	9	* Everything here degrades gracefully:
	10	* - Without `ANTHROPIC_API_KEY` we still open an issue, just with a
	11	* deterministic fallback body saying "AI analysis unavailable".
	12	* - Any DB/git/network failure is caught; the function never throws and
	13	* returns `{ issueNumber: null, reason: <err.message> }` instead.
	14	*/
	15
	16	import { and, desc, eq } from "drizzle-orm";
	17	import { db } from "../db";
	18	import {
	19	deployments,
	20	issueLabels,
	21	issues,
	22	labels,
	23	repositories,
	24	users,
	25	} from "../db/schema";
	26	import { getDefaultBranch, listCommits } from "../git/repository";
	27	import {
	28	MODEL_SONNET,
	29	extractText,
	30	getAnthropic,
	31	isAiAvailable,
	32	parseJsonResponse,
	33	} from "./ai-client";
	34
	35	export interface IncidentAnalysis {
	36	title: string;
	37	likelyCause: string;
	38	suspectedCommit: string \| null;
	39	remediation: string;
	40	}
	41
	42	export interface OnDeployFailureArgs {
	43	repositoryId: string;
	44	deploymentId: string;
	45	ref?: string \| null;
	46	commitSha?: string \| null;
	47	target?: string \| null;
	48	errorMessage?: string \| null;
	49	}
	50
	51	export interface OnDeployFailureResult {
	52	issueNumber: number \| null;
	53	reason: string;
	54	}
	55
	56	/**
	57	* Format a list of commits for inclusion in the incident prompt / issue body.
	58	* Pure helper — kept separate so it can be unit-tested without any I/O.
	59	*/
	60	export function summariseCommitsForIncident(
	61	commits: { sha: string; message: string; author: string }[]
	62	): string {
	63	return commits
	64	.map((c) => {
	65	const sha7 = (c.sha \|\| "").slice(0, 7);
	66	const subject = (c.message \|\| "").split("\n")[0] \|\| "";
	67	const author = c.author \|\| "unknown";
	68	return `- ${sha7} ${subject} — ${author}`;
	69	})
	70	.join("\n");
	71	}
	72
	73	function truncate(s: string \| null \| undefined, max: number): string {
	74	if (!s) return "";
	75	if (s.length <= max) return s;
	76	return s.slice(0, max) + "\n…(truncated)";
	77	}
	78
	79	function renderIssueBody(args: {
	80	deploymentId: string;
	81	ref: string;
	82	shortSha: string;
	83	target: string;
	84	errorMessage: string;
	85	likelyCause: string;
	86	suspectedCommit: string \| null;
	87	remediation: string;
	88	}): string {
	89	const suspected = args.suspectedCommit \|\| "none";
	90	const safeError = args.errorMessage \|\| "(no error message captured)";
	91	return [
	92	`Automated by GlueCron incident responder after deployment ${args.deploymentId} failed.`,
	93	"",
	94	`Ref: \`${args.ref}\` (sha \`${args.shortSha}\`)`,
	95	`Target: ${args.target}`,
	96	"",
	97	"## Error",
	98	"```",
	99	safeError,
	100	"```",
	101	"",
	102	"## Likely cause",
	103	args.likelyCause,
	104	"",
	105	"## Suspected commit",
	106	suspected,
	107	"",
	108	"## Suggested remediation",
	109	args.remediation,
	110	"",
	111	"---",
	112	"_This issue was auto-generated. Edit or close if the analysis is off._",
	113	].join("\n");
	114	}
	115
	116	async function askClaudeForAnalysis(
	117	repoFullName: string,
	118	ref: string,
	119	shortSha: string,
	120	errorMessage: string,
	121	commitSummary: string
	122	): Promise<IncidentAnalysis \| null> {
	123	try {
	124	const client = getAnthropic();
	125	const message = await client.messages.create({
	126	model: MODEL_SONNET,
	127	max_tokens: 1024,
	128	messages: [
	129	{
	130	role: "user",
	131	content: `You are GlueCron's incident responder. A deployment just failed. Respond ONLY with JSON of the form:
	132
	133	{"title": "...", "likelyCause": "...", "suspectedCommit": "<sha or null>", "remediation": "..."}
	134
	135	Repository: ${repoFullName}
	136	Failing ref: ${ref} (sha ${shortSha})
	137
	138	Error message:
	139	\`\`\`
	140	${truncate(errorMessage, 4000)}
	141	\`\`\`
	142
	143	Recent commits (most recent first):
	144	${commitSummary \|\| "(no commits available)"}
	145
	146	Write a crisp issue title (prefixed with "Deploy failed:"), a plausible likelyCause (2-4 sentences), a suspectedCommit sha (or null if unclear), and concrete remediation steps (bullet list as a single string with \\n separators).`,
	147	},
	148	],
	149	});
	150	const parsed = parseJsonResponse<IncidentAnalysis>(extractText(message));
	151	if (!parsed) return null;
	152	const suspected =
	153	typeof parsed.suspectedCommit === "string" && parsed.suspectedCommit
	154	? parsed.suspectedCommit
	155	: null;
	156	return {
	157	title:
	158	typeof parsed.title === "string" && parsed.title.trim()
	159	? parsed.title.trim().slice(0, 200)
	160	: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	161	likelyCause:
	162	typeof parsed.likelyCause === "string" && parsed.likelyCause.trim()
	163	? parsed.likelyCause.trim()
	164	: "Unknown — see raw error above.",
	165	suspectedCommit: suspected,
	166	remediation:
	167	typeof parsed.remediation === "string" && parsed.remediation.trim()
	168	? parsed.remediation.trim()
	169	: "Inspect logs manually and re-run the deployment.",
	170	};
	171	} catch (err) {
	172	console.error("[ai-incident] analysis request failed:", err);
	173	return null;
	174	}
	175	}
	176
	177	/**
	178	* Entry point called by the post-receive hook and the retry-incident route.
	179	* Never throws.
	180	*/
	181	export async function onDeployFailure(
	182	args: OnDeployFailureArgs
	183	): Promise<OnDeployFailureResult> {
	184	const ref = args.ref \|\| "refs/heads/main";
	185	const commitSha = args.commitSha \|\| "";
	186	const shortSha = commitSha ? commitSha.slice(0, 7) : "unknown";
	187	const target = args.target \|\| "unknown";
	188	const errorMessage = args.errorMessage \|\| "";
	189
	190	try {
	191	// 1. Load repository + owner username for nice issue attribution.
	192	const [repoRow] = await db
	193	.select()
	194	.from(repositories)
	195	.where(eq(repositories.id, args.repositoryId))
	196	.limit(1);
	197	if (!repoRow) {
	198	return { issueNumber: null, reason: "repository not found" };
	199	}
	200	const [ownerRow] = await db
	201	.select()
	202	.from(users)
	203	.where(eq(users.id, repoRow.ownerId))
	204	.limit(1);
	205	if (!ownerRow) {
	206	return { issueNumber: null, reason: "repository owner not found" };
	207	}
	208	const repoFullName = `${ownerRow.username}/${repoRow.name}`;
	209
	210	// 2. Load up to ~10 recent commits leading to commitSha, with a fallback
	211	// to the default branch tip if the sha is missing / unresolvable.
	212	let commits: Array<{ sha: string; message: string; author: string }> = [];
	213	try {
	214	const refForLog =
	215	commitSha \|\|
	216	(await getDefaultBranch(ownerRow.username, repoRow.name)) \|\|
	217	repoRow.defaultBranch \|\|
	218	"main";
	219	const raw = await listCommits(
	220	ownerRow.username,
	221	repoRow.name,
	222	refForLog,
	223	10,
	224	0
	225	).catch(() => [] as Awaited<ReturnType<typeof listCommits>>);
	226	commits = raw.map((c) => ({
	227	sha: c.sha,
	228	message: c.message,
	229	author: c.author,
	230	}));
	231	} catch {
	232	commits = [];
	233	}
	234	const commitSummary = summariseCommitsForIncident(commits);
	235
	236	// 3. Ask Claude for an analysis, or fall back to a deterministic body.
	237	let analysis: IncidentAnalysis;
	238	if (isAiAvailable()) {
	239	const ai = await askClaudeForAnalysis(
	240	repoFullName,
	241	ref,
	242	shortSha,
	243	errorMessage,
	244	commitSummary
	245	);
	246	analysis = ai \|\| {
	247	title: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	248	likelyCause:
	249	"AI analysis unavailable — inspect logs manually.\n\nRecent commits:\n" +
	250	commitSummary,
	251	suspectedCommit: commits[0]?.sha \|\| null,
	252	remediation:
	253	"Inspect deployment logs and recent commits. Re-run the deploy once fixed.",
	254	};
	255	} else {
	256	analysis = {
	257	title: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	258	likelyCause:
	259	"AI analysis unavailable — inspect logs manually.\n\nRecent commits:\n" +
	260	(commitSummary \|\| "(none)"),
	261	suspectedCommit: commits[0]?.sha \|\| null,
	262	remediation:
	263	"Inspect deployment logs and recent commits. Re-run the deploy once fixed.",
	264	};
	265	}
	266
	267	// 4. Render the Markdown issue body.
	268	const body = renderIssueBody({
	269	deploymentId: args.deploymentId,
	270	ref,
	271	shortSha,
	272	target,
	273	errorMessage,
	274	likelyCause: analysis.likelyCause,
	275	suspectedCommit: analysis.suspectedCommit,
	276	remediation: analysis.remediation,
	277	});
	278
	279	// 5. Insert the issue row. `issues.number` is a `serial()` — Postgres
	280	// assigns the next number automatically, matching the pattern used
	281	// in src/routes/issues.tsx.
	282	let issueNumber: number \| null = null;
	283	try {
	284	const [inserted] = await db
	285	.insert(issues)
	286	.values({
	287	repositoryId: repoRow.id,
	288	authorId: repoRow.ownerId,
	289	title: analysis.title,
	290	body,
	291	state: "open",
	292	})
	293	.returning();
	294	issueNumber = inserted?.number ?? null;
	295
	296	// Best-effort: attach the "incident" label if one exists for the repo.
	297	if (inserted?.id) {
	298	try {
	299	const [incidentLabel] = await db
	300	.select()
	301	.from(labels)
	302	.where(
	303	and(
	304	eq(labels.repositoryId, repoRow.id),
	305	eq(labels.name, "incident")
	306	)
	307	)
	308	.limit(1);
	309	if (incidentLabel) {
	310	await db
	311	.insert(issueLabels)
	312	.values({ issueId: inserted.id, labelId: incidentLabel.id })
	313	.catch(() => {
	314	/* ignore — the unique constraint may reject duplicates */
	315	});
	316	}
	317	} catch {
	318	/* best-effort */
	319	}
	320	}
	321
	322	// Bump the repo's issue count so the UI stays in sync.
	323	try {
	324	await db
	325	.update(repositories)
	326	.set({ issueCount: (repoRow.issueCount \|\| 0) + 1 })
	327	.where(eq(repositories.id, repoRow.id));
	328	} catch {
	329	/* best-effort */
	330	}
	331	} catch (err) {
	332	return {
	333	issueNumber: null,
	334	reason: (err as Error).message \|\| "issue insert failed",
	335	};
	336	}
	337
	338	// 6. Update the deployment's blockedReason to link the auto-issue, but
	339	// only if the field is currently empty or looks like a raw error
	340	// (i.e. NOT an admin-edited note).
	341	if (issueNumber !== null) {
	342	try {
	343	const [depRow] = await db
	344	.select()
	345	.from(deployments)
	346	.where(eq(deployments.id, args.deploymentId))
	347	.limit(1);
	348	const current = depRow?.blockedReason \|\| "";
	349	const looksAutoEditable =
	350	!current \|\|
	351	current === errorMessage \|\|
	352	/^HTTP \d+/.test(current) \|\|
	353	/^auto-issue #/.test(current);
	354	if (depRow && looksAutoEditable) {
	355	await db
	356	.update(deployments)
	357	.set({ blockedReason: `auto-issue #${issueNumber}` })
	358	.where(eq(deployments.id, args.deploymentId));
	359	}
	360	} catch {
	361	/* best-effort */
	362	}
	363	}
	364
	365	return {
	366	issueNumber,
	367	reason: issueNumber !== null ? "ok" : "issue number unavailable",
	368	};
	369	} catch (err) {
	370	return {
	371	issueNumber: null,
	372	reason: (err as Error).message \|\| "unknown failure",
	373	};
	374	}
	375	}
	376
	377	// Re-exported for tests that want to inspect the most recent incident issue
	378	// for a repository without hitting the HTTP layer.
	379	export async function getLatestIncidentIssueForRepo(
	380	repositoryId: string
	381	): Promise<{ number: number; title: string } \| null> {
	382	try {
	383	const [row] = await db
	384	.select({ number: issues.number, title: issues.title })
	385	.from(issues)
	386	.where(eq(issues.repositoryId, repositoryId))
	387	.orderBy(desc(issues.createdAt))
	388	.limit(1);
	389	return row \|\| null;
	390	} catch {
	391	return null;
	392	}
	393	}