Blame · Line-by-line history

`gate-security-scan-availability.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

gate-security-scan-availability.test.tsBlame155 lines · 1 contributor

Normal view Raw

e2206a6Test User	1	/**
	2	* aiSecurityScanSafe() / gate.ts's runSecretAndSecurityScan — availability
	3	* vs verdict, mirroring gate-gatetest-availability.test.ts's pattern for
	4	* the exact same class of bug in a different gate.
	5	*
	6	* Before this fix, aiSecurityScan() returned `[]` on an AI-provider error
	7	* (timeout, rate limit, unparseable response) — indistinguishable from a
	8	* genuinely clean scan. gate.ts then reported "No security issues found",
	9	* silently converting a merge-blocking security gate into an automatic
	10	* pass on any Anthropic outage. Fixed the same way runGateTestScan's
	11	* outage bug was fixed (commit 6930df0): don't hard-block on a third-party
	12	* outage, but never claim "clean" when the scan didn't actually run.
	13	*/
	14
	15	import { afterEach, beforeEach, describe, expect, it } from "bun:test";
	16	import { runSecretAndSecurityScan } from "../lib/gate";
	17	import { config } from "../lib/config";
	18	import { __resetAnthropicClientForTests } from "../lib/ai-client";
	19
	20	const origFetch = globalThis.fetch;
	21	const origKey = process.env.ANTHROPIC_API_KEY;
	22
	23	function mockFetch(responder: () => Response \| Promise<Response>): void {
	24	// @ts-expect-error — override global fetch for the test
	25	globalThis.fetch = async (): Promise<Response> => responder();
	26	}
	27
	28	function anthropicResponse(text: string): Response {
	29	return new Response(
	30	JSON.stringify({
	31	id: "msg_test",
	32	type: "message",
	33	role: "assistant",
	34	model: "claude-sonnet-4-6",
	35	content: [{ type: "text", text }],
	36	stop_reason: "end_turn",
	37	stop_sequence: null,
	38	usage: { input_tokens: 10, output_tokens: 10 },
	39	}),
	40	{ status: 200, headers: { "content-type": "application/json" } }
	41	);
	42	}
	43
	44	beforeEach(() => {
	45	process.env.ANTHROPIC_API_KEY = "sk-ant-test-key";
	46	// The Anthropic SDK client is a module-level singleton (ai-client.ts's
	47	// `_client`) that captures a fetch reference at construction time rather
	48	// than reading globalThis.fetch fresh per call — without resetting it,
	49	// only the FIRST test's globalThis.fetch mock would ever actually be
	50	// used, and every later test would silently reuse it (confirmed: this
	51	// leaked a 429-mock response into an unrelated later test before this
	52	// reset was added).
	53	__resetAnthropicClientForTests();
	54	});
	55
	56	afterEach(() => {
	57	globalThis.fetch = origFetch;
	58	if (origKey === undefined) delete process.env.ANTHROPIC_API_KEY;
	59	else process.env.ANTHROPIC_API_KEY = origKey;
	60	__resetAnthropicClientForTests();
	61	});
	62
	63	describe("runSecretAndSecurityScan — security scan availability vs verdict", () => {
	64	it("is skipped (not blocking) when the AI provider errors — was previously silently 'clean'", async () => {
	65	mockFetch(() => new Response("rate limited", { status: 429 }));
	66	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	67	scanSecrets: false,
	68	scanSecurity: true,
	69	diffText: "diff --git a/x.ts b/x.ts\n+ eval(userInput)",
	70	});
	71	expect(result.securityResult.skipped).toBe(true);
	72	expect(result.securityResult.passed).toBe(true);
	73	expect(result.securityResult.details).not.toBe("No security issues found");
	74	expect(result.securityResult.details).toContain("unavailable");
	75	});
	76
	77	it("is skipped (not blocking) when the request throws — network error / timeout", async () => {
	78	mockFetch(() => {
	79	throw new Error("ETIMEDOUT");
	80	});
	81	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	82	scanSecrets: false,
	83	scanSecurity: true,
	84	diffText: "some diff",
	85	});
	86	expect(result.securityResult.skipped).toBe(true);
	87	expect(result.securityResult.passed).toBe(true);
	88	// The Anthropic SDK wraps a thrown-during-fetch error into its own
	89	// connection/timeout error type rather than preserving the original
	90	// message verbatim — assert on the outcome (skipped, not the clean
	91	// string), not the SDK's exact wording.
	92	expect(result.securityResult.details).not.toBe("No security issues found");
	93	expect(result.securityResult.details).toContain("unavailable");
	94	});
	95
	96	it("is skipped (not blocking) when the AI response is unparseable", async () => {
	97	mockFetch(() => anthropicResponse("not json at all, just prose"));
	98	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	99	scanSecrets: false,
	100	scanSecurity: true,
	101	diffText: "some diff",
	102	});
	103	expect(result.securityResult.skipped).toBe(true);
	104	expect(result.securityResult.passed).toBe(true);
	105	expect(result.securityResult.details).not.toBe("No security issues found");
	106	});
	107
	108	// Regression guard: proves the two states are now actually distinguishable
	109	// — a genuine clean scan must still say "No security issues found" and
	110	// must NOT be marked skipped.
	111	it("is NOT skipped and reports a genuine clean verdict on success with zero findings", async () => {
	112	mockFetch(() => anthropicResponse('```json\n{"findings": []}\n```'));
	113	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	114	scanSecrets: false,
	115	scanSecurity: true,
	116	diffText: "some diff",
	117	});
	118	expect(result.securityResult.skipped).toBe(false);
	119	expect(result.securityResult.passed).toBe(true);
	120	expect(result.securityResult.details).toBe("No security issues found");
	121	});
	122
	123	it("blocks on a genuine finding — real verdict, not an outage", async () => {
	124	mockFetch(() =>
	125	anthropicResponse(
	126	'```json\n{"findings": [{"type": "Command Injection", "file": "x.ts", "line": 1, "severity": "critical", "description": "eval on user input"}]}\n```'
	127	)
	128	);
	129	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	130	scanSecrets: false,
	131	scanSecurity: true,
	132	diffText: "some diff",
	133	});
	134	expect(result.securityResult.skipped).toBe(false);
	135	expect(result.securityResult.passed).toBe(false);
	136	expect(result.securityIssues).toHaveLength(1);
	137	});
	138
	139	it("is skipped with 'no diff provided' when scanSecurity is requested but no diffText is given", async () => {
	140	const result = await runSecretAndSecurityScan("o", "r", "refs/heads/main", "a".repeat(40), {
	141	scanSecrets: false,
	142	scanSecurity: true,
	143	});
	144	expect(result.securityResult.skipped).toBe(true);
	145	expect(result.securityResult.passed).toBe(true);
	146	expect(result.securityResult.details).toBe("Skipped — no diff provided");
	147	});
	148	});
	149
	150	// Keep config import referenced — sanity that config.anthropicApiKey reflects env for this test file.
	151	describe("config.anthropicApiKey reflects ANTHROPIC_API_KEY for this suite", () => {
	152	it("resolves to the mocked value while set", () => {
	153	expect(config.anthropicApiKey).toBe("sk-ant-test-key");
	154	});
	155	});