Blame · Line-by-line history

`ai-proactive-monitor.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

ai-proactive-monitor.test.tsBlame413 lines · 1 contributor

Normal view Raw

eead172Claude	1	/**
	2	* Tests for the AI Proactive Monitor (`src/lib/ai-proactive-monitor.ts`).
	3	*
	4	* Uses the dependency-injection seams on `aiProactiveMonitorTick` so we
	5	* never touch the DB or the Anthropic API. The Claude call is mocked
	6	* with canned findings; the issue-create + dedupe + audit side-effects
	7	* are observed via spy fakes.
	8	*
	9	* Covers:
	10	* - No-op when AI is unavailable.
	11	* - Info-severity findings are filtered out before issue creation.
	12	* - Warning + critical findings open issues with the proactive label.
	13	* - Dedupe (sha256 of title) skips repeat findings within 24h.
	14	* - Audit row is recorded for every considered finding.
	15	* - Per-finding errors are isolated (one bad finding doesn't wedge the rest).
	16	* - Hard cap on findings per tick (runaway protection).
	17	* - Body renderer embeds the dedupe marker so the lookup query can match.
	18	*/
	19
	20	import { describe, it, expect } from "bun:test";
	21	import {
	22	aiProactiveMonitorTick,
	23	dedupeKeyForTitle,
	24	renderFindingBody,
	25	PROACTIVE_LABEL_NAME,
	26	PROACTIVE_LOOKBACK_HOURS,
	27	PROACTIVE_DEDUPE_MARKER_PREFIX,
	28	__test as monitorInternals,
	29	type ProactiveFinding,
	30	type ProactiveTelemetry,
	31	} from "../lib/ai-proactive-monitor";
	32
	33	const EMPTY_TELEMETRY: ProactiveTelemetry = {
	34	auditLog: [],
	35	platformDeploys: [],
	36	workflowRuns: [],
	37	};
	38
	39	const REPO = { repositoryId: "repo-self", ownerId: "owner-1" };
	40
	41	function finding(
	42	partial: Partial<ProactiveFinding> = {}
	43	): ProactiveFinding {
	44	return {
	45	title: "Default finding",
	46	severity: "warning",
	47	body_markdown: "Something looks off.",
	48	target_url: null,
	49	...partial,
	50	};
	51	}
	52
	53	describe("ai-proactive-monitor — module surface", () => {
	54	it("exports the expected public functions + constants", () => {
	55	expect(typeof aiProactiveMonitorTick).toBe("function");
	56	expect(typeof dedupeKeyForTitle).toBe("function");
	57	expect(typeof renderFindingBody).toBe("function");
	58	expect(PROACTIVE_LABEL_NAME).toBe("ai:proactive-finding");
	59	expect(PROACTIVE_LOOKBACK_HOURS).toBe(24);
	60	});
	61	});
	62
	63	describe("dedupeKeyForTitle", () => {
	64	it("produces a deterministic 32-char hex digest", () => {
	65	const k1 = dedupeKeyForTitle("memory growth on workflow-runner");
	66	const k2 = dedupeKeyForTitle("memory growth on workflow-runner");
	67	expect(k1).toBe(k2);
	68	expect(k1).toMatch(/^[0-9a-f]{32}$/);
	69	});
	70
	71	it("is case-insensitive and trimmed", () => {
	72	expect(dedupeKeyForTitle(" Memory Growth ")).toBe(
	73	dedupeKeyForTitle("memory growth")
	74	);
	75	});
	76
	77	it("differs for different titles", () => {
	78	expect(dedupeKeyForTitle("a")).not.toBe(dedupeKeyForTitle("b"));
	79	});
	80	});
	81
	82	describe("renderFindingBody", () => {
	83	it("embeds the dedupe marker so the LIKE lookup matches", () => {
	84	const f = finding({ title: "deploy times creeping up" });
	85	const key = dedupeKeyForTitle(f.title);
	86	const body = renderFindingBody(f, key);
	87	expect(body).toContain(`${PROACTIVE_DEDUPE_MARKER_PREFIX}${key}`);
	88	expect(body).toContain("warning");
	89	expect(body).toContain("Something looks off.");
	90	});
	91
	92	it("uses the critical badge for critical findings", () => {
	93	const f = finding({ severity: "critical" });
	94	const body = renderFindingBody(f, "k");
	95	expect(body).toContain("critical");
	96	});
	97
	98	it("includes the target URL when provided", () => {
	99	const f = finding({ target_url: "https://gluecron.com/admin/deploys" });
	100	const body = renderFindingBody(f, "k");
	101	expect(body).toContain("https://gluecron.com/admin/deploys");
	102	});
	103	});
	104
	105	describe("aiProactiveMonitorTick — no-op paths", () => {
	106	it("returns zero summary when AI is unavailable", async () => {
	107	let askCalled = false;
	108	const summary = await aiProactiveMonitorTick({
	109	aiAvailable: () => false,
	110	askClaude: async () => {
	111	askCalled = true;
	112	return [];
	113	},
	114	});
	115	expect(askCalled).toBe(false);
	116	expect(summary).toEqual({
	117	considered: 0,
	118	opened: 0,
	119	skippedDedupe: 0,
	120	skippedSeverity: 0,
	121	errors: 0,
	122	});
	123	});
	124
	125	it("returns early when the self-host repo cannot be resolved", async () => {
	126	let askCalled = false;
	127	const summary = await aiProactiveMonitorTick({
	128	aiAvailable: () => true,
	129	resolveSelfHostRepo: async () => null,
	130	askClaude: async () => {
	131	askCalled = true;
	132	return [];
	133	},
	134	});
	135	expect(askCalled).toBe(false);
	136	expect(summary.opened).toBe(0);
	137	});
	138
	139	it("returns a clean summary when Claude finds nothing", async () => {
	140	const summary = await aiProactiveMonitorTick({
	141	aiAvailable: () => true,
	142	resolveSelfHostRepo: async () => REPO,
	143	loadTelemetry: async () => EMPTY_TELEMETRY,
	144	askClaude: async () => [],
	145	isDuplicate: async () => false,
	146	createFindingIssue: async () => 1,
	147	recordAudit: async () => {},
	148	});
	149	expect(summary).toEqual({
	150	considered: 0,
	151	opened: 0,
	152	skippedDedupe: 0,
	153	skippedSeverity: 0,
	154	errors: 0,
	155	});
	156	});
	157	});
	158
	159	describe("aiProactiveMonitorTick — issue creation", () => {
	160	it("opens an issue per warning/critical finding and skips info-severity", async () => {
	161	const created: Array<{ title: string; body: string }> = [];
	162	const audited: Array<{ title: string; issueNumber: number \| null }> = [];
	163
	164	const summary = await aiProactiveMonitorTick({
	165	aiAvailable: () => true,
	166	resolveSelfHostRepo: async () => REPO,
	167	loadTelemetry: async () => EMPTY_TELEMETRY,
	168	askClaude: async () => [
	169	finding({ title: "Memory growth on workflow-runner", severity: "critical" }),
	170	finding({ title: "Deploy times creeping up", severity: "warning" }),
	171	finding({ title: "Just FYI: nothing wrong", severity: "info" }),
	172	],
	173	isDuplicate: async () => false,
	174	createFindingIssue: async (args) => {
	175	created.push({ title: args.title, body: args.body });
	176	return created.length; // 1, 2, ...
	177	},
	178	recordAudit: async (f, _repo, n) => {
	179	audited.push({ title: f.title, issueNumber: n });
	180	},
	181	});
	182
	183	expect(summary.considered).toBe(3);
	184	expect(summary.opened).toBe(2);
	185	expect(summary.skippedSeverity).toBe(1);
	186	expect(summary.skippedDedupe).toBe(0);
	187	expect(summary.errors).toBe(0);
	188	expect(created.map((c) => c.title)).toEqual([
	189	"Memory growth on workflow-runner",
	190	"Deploy times creeping up",
	191	]);
	192	// Each created body must carry the dedupe marker so future ticks dedupe.
	193	for (const c of created) {
	194	const key = dedupeKeyForTitle(c.title);
	195	expect(c.body).toContain(`${PROACTIVE_DEDUPE_MARKER_PREFIX}${key}`);
	196	}
	197	// Audit fires for the 2 non-info findings (the info one is skipped before audit).
	198	expect(audited.map((a) => a.title)).toEqual([
	199	"Memory growth on workflow-runner",
	200	"Deploy times creeping up",
	201	]);
	202	});
	203
	204	it("truncates over-long titles to 200 chars when inserting", async () => {
	205	const longTitle = "x".repeat(500);
	206	let receivedTitle = "";
	207	await aiProactiveMonitorTick({
	208	aiAvailable: () => true,
	209	resolveSelfHostRepo: async () => REPO,
	210	loadTelemetry: async () => EMPTY_TELEMETRY,
	211	askClaude: async () => [finding({ title: longTitle, severity: "warning" })],
	212	isDuplicate: async () => false,
	213	createFindingIssue: async (args) => {
	214	receivedTitle = args.title;
	215	return 1;
	216	},
	217	recordAudit: async () => {},
	218	});
	219	expect(receivedTitle.length).toBe(200);
	220	});
	221	});
	222
	223	describe("aiProactiveMonitorTick — dedupe", () => {
	224	it("does not double-fire on the same title", async () => {
	225	const created: string[] = [];
	226	let dedupeChecks = 0;
	227
	228	const summary = await aiProactiveMonitorTick({
	229	aiAvailable: () => true,
	230	resolveSelfHostRepo: async () => REPO,
	231	loadTelemetry: async () => EMPTY_TELEMETRY,
	232	askClaude: async () => [
	233	finding({ title: "Memory growth on workflow-runner", severity: "warning" }),
	234	],
	235	isDuplicate: async (_repoId, _key, _hours) => {
	236	dedupeChecks += 1;
	237	return true; // pretend we already filed it
	238	},
	239	createFindingIssue: async (args) => {
	240	created.push(args.title);
	241	return 1;
	242	},
	243	recordAudit: async () => {},
	244	});
	245
	246	expect(dedupeChecks).toBe(1);
	247	expect(created).toEqual([]);
	248	expect(summary.skippedDedupe).toBe(1);
	249	expect(summary.opened).toBe(0);
	250	});
	251
	252	it("uses the lookback window from the constant", async () => {
	253	let observedHours = -1;
	254	await aiProactiveMonitorTick({
	255	aiAvailable: () => true,
	256	resolveSelfHostRepo: async () => REPO,
	257	loadTelemetry: async () => EMPTY_TELEMETRY,
	258	askClaude: async () => [finding({ severity: "warning" })],
	259	isDuplicate: async (_r, _k, hours) => {
	260	observedHours = hours;
	261	return true;
	262	},
	263	createFindingIssue: async () => 1,
	264	recordAudit: async () => {},
	265	});
	266	expect(observedHours).toBe(PROACTIVE_LOOKBACK_HOURS);
	267	});
	268
	269	it("passes the sha256 dedupe key to the duplicate check", async () => {
	270	let observedKey = "";
	271	await aiProactiveMonitorTick({
	272	aiAvailable: () => true,
	273	resolveSelfHostRepo: async () => REPO,
	274	loadTelemetry: async () => EMPTY_TELEMETRY,
	275	askClaude: async () => [
	276	finding({ title: "Suspicious admin pattern", severity: "critical" }),
	277	],
	278	isDuplicate: async (_r, key) => {
	279	observedKey = key;
	280	return true;
	281	},
	282	createFindingIssue: async () => 1,
	283	recordAudit: async () => {},
	284	});
	285	expect(observedKey).toBe(dedupeKeyForTitle("Suspicious admin pattern"));
	286	});
	287	});
	288
	289	describe("aiProactiveMonitorTick — robustness", () => {
	290	it("isolates per-finding failures — one bad issue insert does not stop the rest", async () => {
	291	let attempts = 0;
	292	const created: string[] = [];
	293
	294	const summary = await aiProactiveMonitorTick({
	295	aiAvailable: () => true,
	296	resolveSelfHostRepo: async () => REPO,
	297	loadTelemetry: async () => EMPTY_TELEMETRY,
	298	askClaude: async () => [
	299	finding({ title: "first", severity: "warning" }),
	300	finding({ title: "second", severity: "critical" }),
	301	],
	302	isDuplicate: async () => false,
	303	createFindingIssue: async (args) => {
	304	attempts += 1;
	305	if (args.title === "first") {
	306	throw new Error("DB blew up");
	307	}
	308	created.push(args.title);
	309	return attempts;
	310	},
	311	recordAudit: async () => {},
	312	});
	313
	314	expect(attempts).toBe(2);
	315	expect(created).toEqual(["second"]);
	316	expect(summary.opened).toBe(1);
	317	expect(summary.errors).toBe(1);
	318	});
	319
	320	it("returns errors=1 when askClaude throws (does not propagate)", async () => {
	321	const summary = await aiProactiveMonitorTick({
	322	aiAvailable: () => true,
	323	resolveSelfHostRepo: async () => REPO,
	324	loadTelemetry: async () => EMPTY_TELEMETRY,
	325	askClaude: async () => {
	326	throw new Error("anthropic 500");
	327	},
	328	});
	329	expect(summary.errors).toBe(1);
	330	expect(summary.opened).toBe(0);
	331	});
	332
	333	it("returns errors=1 when loadTelemetry throws (does not propagate)", async () => {
	334	const summary = await aiProactiveMonitorTick({
	335	aiAvailable: () => true,
	336	resolveSelfHostRepo: async () => REPO,
	337	loadTelemetry: async () => {
	338	throw new Error("DB down");
	339	},
	340	askClaude: async () => [],
	341	});
	342	expect(summary.errors).toBe(1);
	343	});
	344
	345	it("caps the number of findings opened per tick", async () => {
	346	const created: string[] = [];
	347	const tooMany: ProactiveFinding[] = [];
	348	for (let i = 0; i < 10; i++) {
	349	tooMany.push(finding({ title: `finding-${i}`, severity: "warning" }));
	350	}
	351	const summary = await aiProactiveMonitorTick({
	352	aiAvailable: () => true,
	353	resolveSelfHostRepo: async () => REPO,
	354	loadTelemetry: async () => EMPTY_TELEMETRY,
	355	askClaude: async () => tooMany,
	356	isDuplicate: async () => false,
	357	createFindingIssue: async (args) => {
	358	created.push(args.title);
	359	return created.length;
	360	},
	361	recordAudit: async () => {},
	362	maxFindings: 3,
	363	});
	364	expect(created.length).toBe(3);
	365	expect(summary.opened).toBe(3);
	366	expect(summary.considered).toBe(3);
	367	// The 7 we didn't even look at are counted in the skipped overflow.
	368	expect(summary.skippedSeverity).toBe(7);
	369	});
	370	});
	371
	372	describe("ai-proactive-monitor — prompt summary helper", () => {
	373	it("renders an empty telemetry block without throwing", () => {
	374	const out = monitorInternals.summariseTelemetryForPrompt(EMPTY_TELEMETRY);
	375	expect(out).toContain("Audit log");
	376	expect(out).toContain("Platform deploys");
	377	expect(out).toContain("Workflow runs");
	378	});
	379
	380	it("groups audit rows by action and counts them", () => {
	381	const out = monitorInternals.summariseTelemetryForPrompt({
	382	...EMPTY_TELEMETRY,
	383	auditLog: [
	384	{
	385	action: "repo.create",
	386	targetType: null,
	387	targetId: null,
	388	userId: null,
	389	repositoryId: null,
	390	createdAt: new Date(),
	391	},
	392	{
	393	action: "repo.create",
	394	targetType: null,
	395	targetId: null,
	396	userId: null,
	397	repositoryId: null,
	398	createdAt: new Date(),
	399	},
	400	{
	401	action: "auto_merge.merged",
	402	targetType: null,
	403	targetId: null,
	404	userId: null,
	405	repositoryId: null,
	406	createdAt: new Date(),
	407	},
	408	],
	409	});
	410	expect(out).toContain("repo.create: 2");
	411	expect(out).toContain("auto_merge.merged: 1");
	412	});
	413	});