fix(merge): survive a lapsed Anthropic bill — the outage's three dominoes #5615
5 changed files+266−16
Addedsrc/__tests__/merge-outage-2026-09-01.test.ts+130−0View fileUnifiedSplit
@@ -0,0 +1,130 @@
1/**
2 * The 2026-09-01 merge outage, pinned end to end.
3 *
4 * With the platform's Anthropic balance exhausted, every merge failed —
5 * REST 500, MCP surfacing a raw 400 "credit balance too low" — while pushes,
6 * CI and deploys kept working. Located via the server log after five guarded
7 * seams were each verified NOT to be the cause. The chain:
8 *
9 * 1. A PR's diff contained the security scanner's own test fixtures —
10 * an eval call on user input as a rule input, credential-shaped strings proving
11 * detectors fire — marked with the `gluecron:allow-secret` pragma, the
12 * route the push gate's own rejection message instructs.
13 * 2. The push gate honoured the pragma. The MERGE-time scanners did not:
14 * scanForSecrets carried a drifted copy of the placeholder list missing
15 * that one alternation, and staticSecurityScan honoured no marker at
16 * all. The gate marked the checks failed.
17 * 3. Failed checks armed auto-repair, whose three tiers were each
18 * `try { …messages.create(…) } finally { cleanup }` — no catch — so the
19 * Anthropic 400 rethrew through runAllGateChecks and performGatedMerge
20 * to the top-level handler.
21 *
22 * Three dominoes, three fixes, each asserted below. The invariant they
23 * protect is owner-stated: "we shouldn't need credit balance to ship a
24 * product."
25 */
26import { describe, expect, it } from "bun:test";
27import {
28 ALLOW_SECRET_PRAGMA,
29 SECRET_PLACEHOLDER_RE,
30} from "../lib/secret-placeholders";
31import { scanForSecrets, staticSecurityScan } from "../lib/security-scan";
32
33describe("domino 1+2 — the pragma means the same thing at push and at merge", () => {
34 // Fixtures are assembled at runtime and carry the pragma, for the same
35 // reason the incident PR's did: content that LOOKS like a credential is the
36 // point, and both this repo's own push gate and GitHub's push protection
37 // have (correctly) rejected such fixtures written as bare literals.
38 const fakeAwsKey = "AKIA" + "IOSFODNN7" + "EXAMPLE7X"; // gluecron:allow-secret
39
40 it("scanForSecrets honours the pragma the push gate honours", () => {
41 const marked = `const k = "${fakeAwsKey}"; // ${ALLOW_SECRET_PRAGMA}`;
42 expect(
43 scanForSecrets([{ path: "src/a.ts", content: marked }])
44 ).toEqual([]);
45 });
46
47 it("scanForSecrets still fires without the pragma — parity, not a hole", () => {
48 // The fix must close the drift, not widen the skip. The same content
49 // unmarked is still a finding. (No placeholder word in the line: the
50 // value is built to avoid them.)
51 const secretish = "ak" + "ia".toUpperCase();
52 const line = `const k = "${"AKIA" + "ZZZZQQQQ9999XXXX"}";`;
53 void secretish;
54 expect(
55 scanForSecrets([{ path: "src/a.ts", content: line }]).length
56 ).toBeGreaterThan(0);
57 });
58
59 it("the in-process list and the hook list are the same object, not twins", async () => {
60 // The drift existed BECAUSE there were two lists. push-policy embeds the
61 // shared ERE string into the hook script; if it ever stops consuming the
62 // shared module, this fails before the next outage does.
63 const src = await Bun.file("src/lib/push-policy.ts").text();
64 expect(src).toContain('from "./secret-placeholders"');
65 expect(src).toContain("const HOOK_PLACEHOLDER_RE = SECRET_PLACEHOLDER_ERE;");
66 // And the compiled form recognises the pragma.
67 expect(SECRET_PLACEHOLDER_RE.test(`x // ${ALLOW_SECRET_PRAGMA}`)).toBe(true);
68 });
69
70 it("staticSecurityScan honours the pragma — and ONLY the pragma", () => {
71 // The dangerous token is assembled at runtime so THIS file's own diff
72 // cannot trip the deployed gate. That is not paranoia: the incident PR
73 // was blocked by exactly this — its scanner fixtures, in plain text in
74 // the diff, armed auto-repair at merge time. A fix for that outage whose
75 // own diff re-triggers it would be the outage with better documentation.
76 const evalCall = "ev" + "al(userInput)";
77 const diff = [
78 "diff --git a/src/__tests__/x.test.ts b/src/__tests__/x.test.ts",
79 "+++ b/src/__tests__/x.test.ts",
80 "@@ -0,0 +1,2 @@",
81 `+const fixture = "${evalCall}"; // ${ALLOW_SECRET_PRAGMA}`,
82 `+const real = ${evalCall}; // an example of what not to do`,
83 ].join("\n");
84 const findings = staticSecurityScan(diff);
85 // The marked line is skipped; the unmarked one still fires even though
86 // its line contains the word "example" — injection rules must not adopt
87 // the broad placeholder words, because "example" and "fake" appear in
88 // real code and skipping on them would quietly blind the gate.
89 expect(findings.some((f) => f.line === 1)).toBe(false);
90 expect(findings.some((f) => f.line === 2 && f.type === "code-execution")).toBe(true);
91 });
92});
93
94describe("domino 3 — a repair failure is a result, not an exception", () => {
95 it("every repair tier catches before its finally", async () => {
96 // The defect was structural — `try/finally` with no catch, in all three
97 // tiers — so the pin is structural: every `finally { cleanupWorktree`
98 // in the module must be preceded by a catch in the same block. A
99 // behavioural test would need a live Anthropic failure to reproduce;
100 // this fails the moment someone removes a catch, which is how the shape
101 // regresses.
102 const src = await Bun.file("src/lib/auto-repair.ts").text();
103 const finallies = [...src.matchAll(/\} finally \{\s*\n\s*await cleanupWorktree/g)];
104 expect(finallies.length).toBe(3);
105 // Adjacency, not proximity: each finally must be entered FROM the catch
106 // block — the failed-RepairResult return sits immediately before it. (A
107 // last-catch-before-last-try heuristic breaks on the inner try blocks
108 // these functions legitimately contain.)
109 const guarded = [
110 ...src.matchAll(
111 /\} catch \(err\) \{[^]{0,1600}?success: false,[^]{0,400}?\} finally \{\s*\n\s*await cleanupWorktree/g
112 ),
113 ];
114 expect(guarded.length).toBe(3);
115 });
116
117 it("the catch returns a failed RepairResult, never a success", async () => {
118 // Degrading must not flatter: a failed repair leaves the check failed and
119 // the merge refused with a reason — the same honest state as repair
120 // disabled — never a green check it did not earn.
121 const src = await Bun.file("src/lib/auto-repair.ts").text();
122 const catches = [...src.matchAll(/\[auto-repair\] repair attempt failed/g)];
123 expect(catches.length).toBe(3);
124 for (const m of catches) {
125 const after = src.slice(m.index!, m.index! + 400);
126 expect(after).toContain("success: false");
127 expect(after).toContain("attempted: true");
128 }
129 });
130});
Modifiedsrc/lib/auto-repair.ts+69−0View fileUnifiedSplit
@@ -274,6 +274,29 @@ ACTION REQUIRED: these credentials must be rotated — they remain visible in gi
274274 filesChanged: result.filesChanged,
275275 summary: `Redacted ${findings.length} secret${findings.length === 1 ? "" : "s"} across ${patches.length} file${patches.length === 1 ? "" : "s"}`,
276276 };
277 } catch (err) {
278 // THE CATCH THAT WAS MISSING — found via the server log for the
279 // 2026-09-01 merge outage. All three repair tiers were
280 // `try { …messages.create(…) } finally { cleanup }` with no catch, so an
281 // Anthropic failure cleaned up the worktree and then RETHREW, straight
282 // through runAllGateChecks and performGatedMerge to the top-level
283 // handler: REST answered 500, MCP surfaced the raw 400 "credit balance
284 // too low", and no merge on the platform could complete.
285 //
286 // A repair is an optional accelerant. Its failure mode is "the check
287 // stays failed and a human fixes it" — the same honest refusal the gate
288 // gives when repair is disabled — never "nobody can merge anything".
289 // This is the owner-stated ship-path invariant: no credit balance
290 // required to ship. Every sibling AI seam in the merge chain already
291 // degraded this way; these three were the ones that did not.
292 console.error("[auto-repair] repair attempt failed:", err);
293 return {
294 attempted: true,
295 success: false,
296 filesChanged: [],
297 summary: "repair failed",
298 error: err instanceof Error ? err.message : String(err),
299 };
277300 } finally {
278301 await cleanupWorktree(repoDir, wt.path);
279302 }
@@ -400,6 +423,29 @@ ${patches.map((p) => `- ${p.path}: ${p.reason}`).join("\n")}
400423 filesChanged: result.filesChanged,
401424 summary: `Repaired ${findings.length} security issue${findings.length === 1 ? "" : "s"} in ${patches.length} file${patches.length === 1 ? "" : "s"}`,
402425 };
426 } catch (err) {
427 // THE CATCH THAT WAS MISSING — found via the server log for the
428 // 2026-09-01 merge outage. All three repair tiers were
429 // `try { …messages.create(…) } finally { cleanup }` with no catch, so an
430 // Anthropic failure cleaned up the worktree and then RETHREW, straight
431 // through runAllGateChecks and performGatedMerge to the top-level
432 // handler: REST answered 500, MCP surfaced the raw 400 "credit balance
433 // too low", and no merge on the platform could complete.
434 //
435 // A repair is an optional accelerant. Its failure mode is "the check
436 // stays failed and a human fixes it" — the same honest refusal the gate
437 // gives when repair is disabled — never "nobody can merge anything".
438 // This is the owner-stated ship-path invariant: no credit balance
439 // required to ship. Every sibling AI seam in the merge chain already
440 // degraded this way; these three were the ones that did not.
441 console.error("[auto-repair] repair attempt failed:", err);
442 return {
443 attempted: true,
444 success: false,
445 filesChanged: [],
446 summary: "repair failed",
447 error: err instanceof Error ? err.message : String(err),
448 };
403449 } finally {
404450 await cleanupWorktree(repoDir, wt.path);
405451 }
@@ -590,6 +636,29 @@ ${parsed.patches.map((p) => `- ${p.path}: ${p.reason}`).join("\n")}
590636 filesChanged: result.filesChanged,
591637 summary: parsed.summary,
592638 };
639 } catch (err) {
640 // THE CATCH THAT WAS MISSING — found via the server log for the
641 // 2026-09-01 merge outage. All three repair tiers were
642 // `try { …messages.create(…) } finally { cleanup }` with no catch, so an
643 // Anthropic failure cleaned up the worktree and then RETHREW, straight
644 // through runAllGateChecks and performGatedMerge to the top-level
645 // handler: REST answered 500, MCP surfaced the raw 400 "credit balance
646 // too low", and no merge on the platform could complete.
647 //
648 // A repair is an optional accelerant. Its failure mode is "the check
649 // stays failed and a human fixes it" — the same honest refusal the gate
650 // gives when repair is disabled — never "nobody can merge anything".
651 // This is the owner-stated ship-path invariant: no credit balance
652 // required to ship. Every sibling AI seam in the merge chain already
653 // degraded this way; these three were the ones that did not.
654 console.error("[auto-repair] repair attempt failed:", err);
655 return {
656 attempted: true,
657 success: false,
658 filesChanged: [],
659 summary: "repair failed",
660 error: err instanceof Error ? err.message : String(err),
661 };
593662 } finally {
594663 await cleanupWorktree(repoDir, wt.path);
595664 }
Modifiedsrc/lib/push-policy.ts+7−10View fileUnifiedSplit
@@ -35,6 +35,7 @@
3535 * Postgres hiccups.
3636 */
3737
38import { SECRET_PLACEHOLDER_ERE } from "./secret-placeholders";
3839import { mkdtemp, writeFile, rm } from "fs/promises";
3940import { join } from "path";
4041import { tmpdir } from "os";
@@ -108,16 +109,12 @@ export function hookSecretPatternTypes(): string[] {
108109
109110const HOOK_SKIP_PATH_RE =
110111 "(^|/)(\\.git|node_modules|vendor|dist|build|\\.next|\\.cache)/|\\.(png|jpe?g|gif|webp|ico|svg|pdf|mp4|mov|wasm|woff2?|ttf|eot|map)$|(^|/)(bun\\.lockb?|package-lock\\.json|yarn\\.lock|pnpm-lock\\.yaml)$";
111// Lines matching this are never treated as a leaked secret.
112//
113// `gluecron:allow-secret` is an explicit, reviewable opt-out pragma (the same
114// idea as gitleaks:allow). It exists because this repository's own test suite
115// must contain realistic-looking credentials in order to prove the scanner
116// detects them — without a pragma the scanner rejects its own fixtures and the
117// suite becomes unpushable. Prefer this over skip-listing whole directories,
118// which would silently stop scanning real code under those paths.
119const HOOK_PLACEHOLDER_RE =
120 "gluecron:allow-secret|example|placeholder|fake|dummy|your[-_]?api|xxxxx|testkey|changeme";
112// Lines matching this are never treated as a leaked secret. Single-sourced
113// from secret-placeholders.ts: this hook and the merge-time scanner drifted
114// apart once (the allow-secret pragma existed here and not there), and the
115// merge gate rejecting fixtures the push gate had sanctioned was the first
116// domino of the 2026-09-01 merge outage documented in that module.
117const HOOK_PLACEHOLDER_RE = SECRET_PLACEHOLDER_ERE;
121118
122119/**
123120 * JS source for the per-push evaluator that the pre-receive hook calls once
Addedsrc/lib/secret-placeholders.ts+45−0View fileUnifiedSplit
@@ -0,0 +1,45 @@
1/**
2 * One definition of "this credential-looking line is a sanctioned fixture".
3 *
4 * WHY THIS FILE EXISTS. The pre-receive push gate and the merge-time gate
5 * each carried their own copy of the placeholder list, and the copies had
6 * drifted: push-policy.ts included the `gluecron:allow-secret` pragma —
7 * documented as THE sanctioned route for this repository's own test fixtures,
8 * "prefer this over skip-listing whole directories" — while scanForSecrets in
9 * security-scan.ts had every other alternation and not that one.
10 *
11 * So the pragma passed the push gate and was ignored at merge. A PR whose
12 * fixtures were marked exactly as instructed pushed cleanly, then failed the
13 * merge-time scan, which marked the check failed, which triggered
14 * auto-repair, which called Anthropic — and when the account had no credits,
15 * that unguarded call became the 500 that blocked every merge on the
16 * platform. The drift was not cosmetic; it was the first domino of the
17 * 2026-09-01 merge outage.
18 *
19 * This is the state-filters.ts lesson again, and the GateTest engine hit the
20 * identical shape the same day: its placeholder allow-list existed twice
21 * character-for-character under a comment claiming single-sourcing, with the
22 * copies diverged on the `i` flag. A comment asserting single-sourcing is not
23 * single-sourcing. A module is.
24 */
25
26/**
27 * The explicit, reviewable opt-out pragma (the same idea as gitleaks:allow).
28 *
29 * It exists because this repository's own test suite must contain
30 * realistic-looking credentials to prove the scanner detects them — without
31 * a pragma the scanner rejects its own fixtures and the suite becomes
32 * unpushable.
33 */
34export const ALLOW_SECRET_PRAGMA = "gluecron:allow-secret";
35
36/**
37 * ERE source string — embedded verbatim into the pre-receive hook script,
38 * which is why this is a string and not a RegExp.
39 */
40export const SECRET_PLACEHOLDER_ERE =
41 ALLOW_SECRET_PRAGMA +
42 "|example|placeholder|fake|dummy|your[-_]?api|xxxxx|testkey|changeme";
43
44/** The same expression, compiled, for in-process scanners. */
45export const SECRET_PLACEHOLDER_RE = new RegExp(SECRET_PLACEHOLDER_ERE, "i");
Modifiedsrc/lib/security-scan.ts+15−6View fileUnifiedSplit
@@ -5,6 +5,7 @@
55 * for risky patterns (SSRF, SQL injection, XSS, unsafe deserialisation, etc).
66 */
77
8import { ALLOW_SECRET_PRAGMA, SECRET_PLACEHOLDER_RE } from "./secret-placeholders";
89import { getAnthropic, MODEL_SONNET, extractText, parseJsonResponse, isAiAvailable } from "./ai-client";
910
1011export interface SecretFinding {
@@ -101,12 +102,12 @@ export function scanForSecrets(
101102 const lines = file.content.split("\n");
102103 for (let i = 0; i < lines.length; i++) {
103104 const line = lines[i];
104 // Skip lines that look like placeholders / tests
105 if (
106 /example|placeholder|fake|dummy|your[-_]?api|xxxxx|testkey|changeme/i.test(
107 line
108 )
109 ) {
105 // Skip lines that look like placeholders / tests. Shared with the
106 // pre-receive hook: this copy was missing the `gluecron:allow-secret`
107 // pragma the hook honours, so a fixture marked exactly as the push
108 // rejection instructs sailed through the push and then failed HERE —
109 // which is what armed auto-repair during the 2026-09-01 merge outage.
110 if (SECRET_PLACEHOLDER_RE.test(line)) {
110111 continue;
111112 }
112113 for (const pattern of SECRET_PATTERNS) {
@@ -475,6 +476,14 @@ export function staticSecurityScan(diffText: string): SecurityFinding[] {
475476 const findings: SecurityFinding[] = [];
476477 for (const { file, line, text } of addedLinesFromDiff(diffText)) {
477478 if (shouldSkipPath(file)) continue;
479 // Only the explicit pragma skips here — NOT the broad placeholder list.
480 // These are injection rules, and words like "example" or "fake" appear in
481 // real code; skipping on them would quietly blind the gate. The pragma is
482 // a deliberate, reviewable act, and it is how the repository's own
483 // security-scanner fixtures (an eval call written out as a test input, a fake
484 // AWS key proving the detector fires) get past the gate that would
485 // otherwise arm auto-repair against them.
486 if (text.includes(ALLOW_SECRET_PRAGMA)) continue;
478487 for (const rule of STATIC_SECURITY_RULES) {
479488 if (!rule.pattern.test(text)) continue;
480489 findings.push({
481490
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts