Blame · Line-by-line history

`hosted-loop-spend-floor.test.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

hosted-loop-spend-floor.test.tsBlame166 lines · 1 contributor

Normal view Raw

68eccaaccantynz-alt	1	/**
	2	* Regression guard: a hosted loop's monthly budget must not be bypassable by
	3	* simply declining to report usage.
	4	*
	5	* The cap itself was fine — invokeLoop short-circuits with `budget_exceeded`
	6	* once monthly spend reaches monthlyBudgetCents. What fed it was not. Spend
	7	* came entirely from extractUsageFromStdout, i.e. from a `usage` block
	8	* printed by the snippet, and loop.sourceCode is arbitrary user-supplied
	9	* code that any registered account can run (the file says so itself, above
	10	* the env block).
	11	*
	12	* So a snippet that never prints a usage block costed 0 cents, its monthly
	13	* spend never advanced, and the cap could never stop it: unlimited
	14	* invocations, each burning up to LOOP_EXEC_TIMEOUT_MS of platform compute,
	15	* and platform Anthropic credits wherever HOSTED_LOOPS_PLATFORM_KEY=1.
	16	*
	17	* The fix charges a floor derived from the executor's own measured duration,
	18	* which the snippet cannot forge. Self-reported usage still wins when it is
	19	* higher — it is the better estimate of real token spend and nobody has an
	20	* incentive to overstate it. The floor only removes the option of reporting
	21	* nothing.
	22	*
	23	* NOTE this is a deliberate billing behaviour change, not purely internal:
	24	* runs that previously cost 0 now cost at least the per-run minimum. Both
	25	* knobs are env-tunable and read at access time.
	26	*/
	27
	28	import { describe, it, expect, afterEach } from "bun:test";
	29	import { readFileSync } from "node:fs";
	30	import { join } from "node:path";
	31	import {
	32	invocationFloorCents,
	33	extractUsageFromStdout,
	34	LOOP_EXEC_TIMEOUT_MS,
	35	} from "../lib/hosted-claude-loop";
	36
	37	const ENV_MIN = "HOSTED_LOOP_MIN_CENTS_PER_RUN";
	38	const ENV_RATE = "HOSTED_LOOP_MS_PER_CENT";
	39
	40	afterEach(() => {
	41	delete process.env[ENV_MIN];
	42	delete process.env[ENV_RATE];
	43	});
	44
	45	describe("invocationFloorCents", () => {
	46	it("charges at least one cent even for an instant run", () => {
	47	// This is the whole point: a snippet reporting nothing is not free.
	48	expect(invocationFloorCents(0)).toBe(1);
	49	expect(invocationFloorCents(1)).toBe(1);
	50	});
	51
	52	it("scales with measured wall clock", () => {
	53	expect(invocationFloorCents(10_000)).toBe(1);
	54	expect(invocationFloorCents(10_001)).toBe(2);
	55	expect(invocationFloorCents(25_000)).toBe(3);
	56	});
	57
	58	it("bounds a maximum-length run sensibly", () => {
	59	// Execution is hard-capped, so the floor per run is bounded too.
	60	const maxFloor = invocationFloorCents(LOOP_EXEC_TIMEOUT_MS);
	61	expect(maxFloor).toBeGreaterThan(0);
	62	expect(maxFloor).toBeLessThanOrEqual(5);
	63	});
	64
	65	it("never returns a negative or non-finite charge", () => {
	66	for (const bad of [-1, -100_000, NaN, Infinity, -Infinity]) {
	67	const got = invocationFloorCents(bad as number);
	68	expect(Number.isFinite(got)).toBe(true);
	69	expect(got).toBeGreaterThanOrEqual(1);
	70	}
	71	});
	72
	73	it("honours the env knobs at access time", () => {
	74	process.env[ENV_MIN] = "5";
	75	process.env[ENV_RATE] = "1000";
	76	expect(invocationFloorCents(0)).toBe(5);
	77	expect(invocationFloorCents(9_000)).toBe(9);
	78	});
	79
	80	it("falls back to safe defaults when the env is nonsense", () => {
	81	process.env[ENV_MIN] = "not-a-number";
	82	process.env[ENV_RATE] = "0";
	83	expect(invocationFloorCents(0)).toBe(1);
	84	expect(invocationFloorCents(10_000)).toBe(1);
	85	});
	86
	87	it("allows the floor to be disabled deliberately", () => {
	88	// Explicit zero is a real operator choice, not nonsense — it must be
	89	// honoured rather than silently coerced back to the default.
	90	process.env[ENV_MIN] = "0";
	91	expect(invocationFloorCents(0)).toBe(0);
	92	});
	93	});
	94
	95	describe("extractUsageFromStdout is the untrusted input", () => {
	96	it("reports nothing for a snippet that prints nothing", () => {
	97	// The exact shape that used to cost zero and never advance the cap.
	98	expect(extractUsageFromStdout("")).toEqual({
	99	inputTokens: 0,
	100	outputTokens: 0,
	101	model: null,
	102	});
	103	expect(extractUsageFromStdout("hello, no json here")).toEqual({
	104	inputTokens: 0,
	105	outputTokens: 0,
	106	model: null,
	107	});
	108	});
	109
	110	it("still reads a well-formed usage block", () => {
	111	const got = extractUsageFromStdout(
	112	JSON.stringify({ usage: { input_tokens: 100, output_tokens: 50 } })
	113	);
	114	expect(got.inputTokens).toBe(100);
	115	expect(got.outputTokens).toBe(50);
	116	});
	117	});
	118
	119	// --- the floor must actually be applied ------------------------------------
	120	//
	121	// Asserting the identifier merely APPEARS is not asserting it is invoked —
	122	// `void fn;` would satisfy that. Assert the call form. Strip ONLY line
	123	// comments.
	124
	125	describe("invokeLoop applies the floor to what it charges", () => {
	126	const src = (() => {
	127	const raw = readFileSync(
	128	join(import.meta.dir, "..", "lib", "hosted-claude-loop.ts"),
	129	"utf8"
	130	);
	131	const stripped = raw
	132	.split("\n")
	133	.filter((l) => !l.trim().startsWith("//") && !l.trim().startsWith("*"))
	134	.join("\n");
	135	expect(stripped.length).toBeGreaterThan(0);
	136	return stripped;
	137	})();
	138
	139	it("takes the max of self-reported cost and the measured floor", () => {
	140	expect(src).toContain("invocationFloorCents(execResult.durationMs)");
	141	expect(src).toContain("Math.max(");
	142	});
	143
	144	it("charges the floor before the run row is written", () => {
	145	// centsEstimate on the run row, the loop rollup and the agent charge all
	146	// read the same `cents`, so it has to be floored before any of them.
	147	const at = src.indexOf("invocationFloorCents(execResult.durationMs)");
	148	expect(at).toBeGreaterThan(-1);
	149	const after = src.slice(at);
	150	expect(after.length).toBeGreaterThan(0);
	151	expect(after).toContain("centsEstimate: cents");
	152	expect(after).toContain("bumpLoopTotals(loop.id, cents)");
	153	});
	154
	155	it("still short-circuits over-budget loops before executing", () => {
	156	// The floor is worthless if the cap check itself goes away.
	157	const at = src.indexOf("getLoopMonthlySpendCents(loop.id)");
	158	expect(at).toBeGreaterThan(-1);
	159	const window = src.slice(at, at + 400);
	160	expect(window.length).toBeGreaterThan(0);
	161	expect(window).toContain("loop.monthlyBudgetCents");
	162	expect(window).toContain("budget_exceeded");
	163	// ...and it happens before the executor is invoked.
	164	expect(at).toBeLessThan(src.indexOf("await exec({"));
	165	});
	166	});