Blame · Line-by-line history

`ai-incident.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-incident.tsBlame403 lines · 1 contributor

Normal view Raw

1e162a8Claude	1	/**
	2	* Block D4 — AI Incident Responder.
	3	*
	4	* When a deployment fails, this module automatically opens an issue with an
	5	* AI-generated root-cause analysis. Invoked by the post-receive hook (from
	6	* `triggerCrontechDeploy`) whenever the Crontech deploy call returns a non-2xx
	7	* response or throws. Also retriggerable via the deployments route.
	8	*
	9	* Everything here degrades gracefully:
	10	* - Without `ANTHROPIC_API_KEY` we still open an issue, just with a
	11	* deterministic fallback body saying "AI analysis unavailable".
	12	* - Any DB/git/network failure is caught; the function never throws and
	13	* returns `{ issueNumber: null, reason: <err.message> }` instead.
	14	*/
	15
	16	import { and, desc, eq } from "drizzle-orm";
	17	import { db } from "../db";
	18	import {
	19	deployments,
	20	issueLabels,
	21	issues,
	22	labels,
	23	repositories,
	24	users,
	25	} from "../db/schema";
	26	import { getDefaultBranch, listCommits } from "../git/repository";
	27	import {
	28	MODEL_SONNET,
	29	extractText,
	30	getAnthropic,
	31	isAiAvailable,
	32	parseJsonResponse,
	33	} from "./ai-client";
	34
	35	export interface IncidentAnalysis {
	36	title: string;
	37	likelyCause: string;
	38	suspectedCommit: string \| null;
	39	remediation: string;
	40	}
	41
	42	export interface OnDeployFailureArgs {
	43	repositoryId: string;
	44	deploymentId: string;
	45	ref?: string \| null;
	46	commitSha?: string \| null;
	47	target?: string \| null;
	48	errorMessage?: string \| null;
	49	}
	50
	51	export interface OnDeployFailureResult {
	52	issueNumber: number \| null;
	53	reason: string;
	54	}
	55
	56	/**
	57	* Format a list of commits for inclusion in the incident prompt / issue body.
	58	* Pure helper — kept separate so it can be unit-tested without any I/O.
	59	*/
	60	export function summariseCommitsForIncident(
	61	commits: { sha: string; message: string; author: string }[]
	62	): string {
	63	return commits
	64	.map((c) => {
	65	const sha7 = (c.sha \|\| "").slice(0, 7);
	66	const subject = (c.message \|\| "").split("\n")[0] \|\| "";
	67	const author = c.author \|\| "unknown";
	68	return `- ${sha7} ${subject} — ${author}`;
	69	})
	70	.join("\n");
	71	}
	72
	73	function truncate(s: string \| null \| undefined, max: number): string {
	74	if (!s) return "";
	75	if (s.length <= max) return s;
	76	return s.slice(0, max) + "\n…(truncated)";
	77	}
	78
	79	function renderIssueBody(args: {
	80	deploymentId: string;
	81	ref: string;
	82	shortSha: string;
	83	target: string;
	84	errorMessage: string;
	85	likelyCause: string;
	86	suspectedCommit: string \| null;
	87	remediation: string;
	88	}): string {
	89	const suspected = args.suspectedCommit \|\| "none";
	90	const safeError = args.errorMessage \|\| "(no error message captured)";
	91	return [
	92	`Automated by GlueCron incident responder after deployment ${args.deploymentId} failed.`,
	93	"",
	94	`Ref: \`${args.ref}\` (sha \`${args.shortSha}\`)`,
	95	`Target: ${args.target}`,
	96	"",
	97	"## Error",
	98	"```",
	99	safeError,
	100	"```",
	101	"",
	102	"## Likely cause",
	103	args.likelyCause,
	104	"",
	105	"## Suspected commit",
	106	suspected,
	107	"",
	108	"## Suggested remediation",
	109	args.remediation,
	110	"",
	111	"---",
	112	"_This issue was auto-generated. Edit or close if the analysis is off._",
	113	].join("\n");
	114	}
	115
	116	async function askClaudeForAnalysis(
	117	repoFullName: string,
	118	ref: string,
	119	shortSha: string,
	120	errorMessage: string,
	121	commitSummary: string
	122	): Promise<IncidentAnalysis \| null> {
	123	try {
40e7738Claude	124	const { recordAi } = await import("./ai-flywheel");
1e162a8Claude	125	const client = getAnthropic();
40e7738Claude	126	const message = await recordAi(
	127	{
	128	actionType: "incident",
	129	model: MODEL_SONNET,
	130	summary: `incident analysis ${repoFullName}@${shortSha}`,
	131	commitSha: shortSha,
	132	metadata: { ref, errorMessage: errorMessage.slice(0, 200) },
	133	},
	134	() => client.messages.create({
1e162a8Claude	135	model: MODEL_SONNET,
	136	max_tokens: 1024,
	137	messages: [
	138	{
	139	role: "user",
	140	content: `You are GlueCron's incident responder. A deployment just failed. Respond ONLY with JSON of the form:
	141
	142	{"title": "...", "likelyCause": "...", "suspectedCommit": "<sha or null>", "remediation": "..."}
	143
	144	Repository: ${repoFullName}
	145	Failing ref: ${ref} (sha ${shortSha})
	146
	147	Error message:
	148	\`\`\`
	149	${truncate(errorMessage, 4000)}
	150	\`\`\`
	151
	152	Recent commits (most recent first):
	153	${commitSummary \|\| "(no commits available)"}
	154
	155	Write a crisp issue title (prefixed with "Deploy failed:"), a plausible likelyCause (2-4 sentences), a suspectedCommit sha (or null if unclear), and concrete remediation steps (bullet list as a single string with \\n separators).`,
	156	},
	157	],
40e7738Claude	158	})
	159	);
1e162a8Claude	160	const parsed = parseJsonResponse<IncidentAnalysis>(extractText(message));
	161	if (!parsed) return null;
	162	const suspected =
	163	typeof parsed.suspectedCommit === "string" && parsed.suspectedCommit
	164	? parsed.suspectedCommit
	165	: null;
	166	return {
	167	title:
	168	typeof parsed.title === "string" && parsed.title.trim()
	169	? parsed.title.trim().slice(0, 200)
	170	: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	171	likelyCause:
	172	typeof parsed.likelyCause === "string" && parsed.likelyCause.trim()
	173	? parsed.likelyCause.trim()
	174	: "Unknown — see raw error above.",
	175	suspectedCommit: suspected,
	176	remediation:
	177	typeof parsed.remediation === "string" && parsed.remediation.trim()
	178	? parsed.remediation.trim()
	179	: "Inspect logs manually and re-run the deployment.",
	180	};
	181	} catch (err) {
	182	console.error("[ai-incident] analysis request failed:", err);
	183	return null;
	184	}
	185	}
	186
	187	/**
	188	* Entry point called by the post-receive hook and the retry-incident route.
	189	* Never throws.
	190	*/
	191	export async function onDeployFailure(
	192	args: OnDeployFailureArgs
	193	): Promise<OnDeployFailureResult> {
	194	const ref = args.ref \|\| "refs/heads/main";
	195	const commitSha = args.commitSha \|\| "";
	196	const shortSha = commitSha ? commitSha.slice(0, 7) : "unknown";
	197	const target = args.target \|\| "unknown";
	198	const errorMessage = args.errorMessage \|\| "";
	199
	200	try {
	201	// 1. Load repository + owner username for nice issue attribution.
	202	const [repoRow] = await db
	203	.select()
	204	.from(repositories)
	205	.where(eq(repositories.id, args.repositoryId))
	206	.limit(1);
	207	if (!repoRow) {
	208	return { issueNumber: null, reason: "repository not found" };
	209	}
	210	const [ownerRow] = await db
	211	.select()
	212	.from(users)
	213	.where(eq(users.id, repoRow.ownerId))
	214	.limit(1);
	215	if (!ownerRow) {
	216	return { issueNumber: null, reason: "repository owner not found" };
	217	}
	218	const repoFullName = `${ownerRow.username}/${repoRow.name}`;
	219
	220	// 2. Load up to ~10 recent commits leading to commitSha, with a fallback
	221	// to the default branch tip if the sha is missing / unresolvable.
	222	let commits: Array<{ sha: string; message: string; author: string }> = [];
	223	try {
	224	const refForLog =
	225	commitSha \|\|
	226	(await getDefaultBranch(ownerRow.username, repoRow.name)) \|\|
	227	repoRow.defaultBranch \|\|
	228	"main";
	229	const raw = await listCommits(
	230	ownerRow.username,
	231	repoRow.name,
	232	refForLog,
	233	10,
	234	0
	235	).catch(() => [] as Awaited<ReturnType<typeof listCommits>>);
	236	commits = raw.map((c) => ({
	237	sha: c.sha,
	238	message: c.message,
	239	author: c.author,
	240	}));
	241	} catch {
	242	commits = [];
	243	}
	244	const commitSummary = summariseCommitsForIncident(commits);
	245
	246	// 3. Ask Claude for an analysis, or fall back to a deterministic body.
	247	let analysis: IncidentAnalysis;
	248	if (isAiAvailable()) {
	249	const ai = await askClaudeForAnalysis(
	250	repoFullName,
	251	ref,
	252	shortSha,
	253	errorMessage,
	254	commitSummary
	255	);
	256	analysis = ai \|\| {
	257	title: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	258	likelyCause:
	259	"AI analysis unavailable — inspect logs manually.\n\nRecent commits:\n" +
	260	commitSummary,
	261	suspectedCommit: commits[0]?.sha \|\| null,
	262	remediation:
	263	"Inspect deployment logs and recent commits. Re-run the deploy once fixed.",
	264	};
	265	} else {
	266	analysis = {
	267	title: `Deploy failed: ${repoFullName} @ ${shortSha}`,
	268	likelyCause:
	269	"AI analysis unavailable — inspect logs manually.\n\nRecent commits:\n" +
	270	(commitSummary \|\| "(none)"),
	271	suspectedCommit: commits[0]?.sha \|\| null,
	272	remediation:
	273	"Inspect deployment logs and recent commits. Re-run the deploy once fixed.",
	274	};
	275	}
	276
	277	// 4. Render the Markdown issue body.
	278	const body = renderIssueBody({
	279	deploymentId: args.deploymentId,
	280	ref,
	281	shortSha,
	282	target,
	283	errorMessage,
	284	likelyCause: analysis.likelyCause,
	285	suspectedCommit: analysis.suspectedCommit,
	286	remediation: analysis.remediation,
	287	});
	288
	289	// 5. Insert the issue row. `issues.number` is a `serial()` — Postgres
	290	// assigns the next number automatically, matching the pattern used
	291	// in src/routes/issues.tsx.
	292	let issueNumber: number \| null = null;
	293	try {
	294	const [inserted] = await db
	295	.insert(issues)
	296	.values({
	297	repositoryId: repoRow.id,
	298	authorId: repoRow.ownerId,
	299	title: analysis.title,
	300	body,
	301	state: "open",
	302	})
	303	.returning();
	304	issueNumber = inserted?.number ?? null;
	305
	306	// Best-effort: attach the "incident" label if one exists for the repo.
	307	if (inserted?.id) {
	308	try {
	309	const [incidentLabel] = await db
	310	.select()
	311	.from(labels)
	312	.where(
	313	and(
	314	eq(labels.repositoryId, repoRow.id),
	315	eq(labels.name, "incident")
	316	)
	317	)
	318	.limit(1);
	319	if (incidentLabel) {
	320	await db
	321	.insert(issueLabels)
	322	.values({ issueId: inserted.id, labelId: incidentLabel.id })
	323	.catch(() => {
	324	/* ignore — the unique constraint may reject duplicates */
	325	});
	326	}
	327	} catch {
	328	/* best-effort */
	329	}
	330	}
	331
	332	// Bump the repo's issue count so the UI stays in sync.
	333	try {
	334	await db
	335	.update(repositories)
	336	.set({ issueCount: (repoRow.issueCount \|\| 0) + 1 })
	337	.where(eq(repositories.id, repoRow.id));
	338	} catch {
	339	/* best-effort */
	340	}
	341	} catch (err) {
	342	return {
	343	issueNumber: null,
	344	reason: (err as Error).message \|\| "issue insert failed",
	345	};
	346	}
	347
	348	// 6. Update the deployment's blockedReason to link the auto-issue, but
	349	// only if the field is currently empty or looks like a raw error
	350	// (i.e. NOT an admin-edited note).
	351	if (issueNumber !== null) {
	352	try {
	353	const [depRow] = await db
	354	.select()
	355	.from(deployments)
	356	.where(eq(deployments.id, args.deploymentId))
	357	.limit(1);
	358	const current = depRow?.blockedReason \|\| "";
	359	const looksAutoEditable =
	360	!current \|\|
	361	current === errorMessage \|\|
	362	/^HTTP \d+/.test(current) \|\|
	363	/^auto-issue #/.test(current);
	364	if (depRow && looksAutoEditable) {
	365	await db
	366	.update(deployments)
	367	.set({ blockedReason: `auto-issue #${issueNumber}` })
	368	.where(eq(deployments.id, args.deploymentId));
	369	}
	370	} catch {
	371	/* best-effort */
	372	}
	373	}
	374
	375	return {
	376	issueNumber,
	377	reason: issueNumber !== null ? "ok" : "issue number unavailable",
	378	};
	379	} catch (err) {
	380	return {
	381	issueNumber: null,
	382	reason: (err as Error).message \|\| "unknown failure",
	383	};
	384	}
	385	}
	386
	387	// Re-exported for tests that want to inspect the most recent incident issue
	388	// for a repository without hitting the HTTP layer.
	389	export async function getLatestIncidentIssueForRepo(
	390	repositoryId: string
	391	): Promise<{ number: number; title: string } \| null> {
	392	try {
	393	const [row] = await db
	394	.select({ number: issues.number, title: issues.title })
	395	.from(issues)
	396	.where(eq(issues.repositoryId, repositoryId))
	397	.orderBy(desc(issues.createdAt))
	398	.limit(1);
	399	return row \|\| null;
	400	} catch {
	401	return null;
	402	}
	403	}