Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

advancement-scanner.test.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/__tests__/advancement-scanner.test.ts
advancement-scanner.test.tsBlame597 lines · 1 contributor
Normal viewRaw
d199847Claude1/**
2 * Tests for src/lib/advancement-scanner.ts.
3 *
4 * Uses the dependency-injection seams on `runAdvancementScan` so we
5 * never hit the DB or the Anthropic API. The Claude call is mocked
6 * with canned findings; the issue-create + dedupe + audit side-effects
7 * are observed via spy fakes.
8 *
9 * Coverage:
10 * - Public surface exports
11 * - dedupe key + body marker
12 * - Pure model-suggestion logic (newer / cheaper variants)
13 * - Issue creation for canned Claude findings
14 * - Dedupe (same sha256 title) skips re-filing within 30d
15 * - Stack bumps route to the migration-assistant kickoff, not openIssue
16 * - Per-finding errors are isolated
17 * - Hard cap on findings per scan
18 * - audit_log gets the scan-complete row
19 *
20 * DB-touching paths (the defaults) are gated behind HAS_DB so the
21 * suite stays green on machines without Postgres.
22 */
23
24import { describe, it, expect } from "bun:test";
25import {
26 ADVANCEMENT_AUDIT_ACTION,
27 ADVANCEMENT_DEDUPE_DAYS,
28 ADVANCEMENT_DEDUPE_MARKER_PREFIX,
29 ADVANCEMENT_LABEL_NAME,
30 ADVANCEMENT_SCAN_COMPLETE_ACTION,
31 KNOWN_CLAUDE_MODELS,
32 MAX_ADVANCEMENT_FINDINGS_PER_SCAN,
33 STACK_KEYSTONE_DEPS,
34 TRENDING_FEATURE_CATALOGUE,
35 __test as scannerInternals,
36 advancementDedupeKey,
37 renderAdvancementBody,
38 runAdvancementScan,
39 suggestModelUpgrades,
40 type AdvancementFinding,
41 type ClaudeModelEntry,
42} from "../lib/advancement-scanner";
43
44const HAS_DB = Boolean(process.env.DATABASE_URL);
45
46const REPO = {
47 repositoryId: "repo-self",
48 ownerId: "owner-1",
49 ownerName: "ccantynz",
50 repoName: "Gluecron.com",
51 defaultBranch: "main",
52};
53
54function finding(
55 partial: Partial<AdvancementFinding> = {}
56): AdvancementFinding {
57 return {
58 kind: "self_improvement",
59 title: "Default finding",
60 urgency: "medium",
61 suggested_action: "Do the thing.",
62 ...partial,
63 };
64}
65
66// ---------------------------------------------------------------------------
67// Public surface
68// ---------------------------------------------------------------------------
69
70describe("advancement-scanner — module surface", () => {
71 it("exports the expected public functions + constants", () => {
72 expect(typeof runAdvancementScan).toBe("function");
73 expect(typeof advancementDedupeKey).toBe("function");
74 expect(typeof renderAdvancementBody).toBe("function");
75 expect(typeof suggestModelUpgrades).toBe("function");
76 expect(ADVANCEMENT_LABEL_NAME).toBe("ai:advancement");
77 expect(ADVANCEMENT_AUDIT_ACTION).toBe("ai.advancement.finding");
78 expect(ADVANCEMENT_SCAN_COMPLETE_ACTION).toBe(
79 "ai.advancement.scan_complete"
80 );
81 expect(ADVANCEMENT_DEDUPE_DAYS).toBe(30);
82 expect(MAX_ADVANCEMENT_FINDINGS_PER_SCAN).toBeGreaterThan(0);
83 expect(STACK_KEYSTONE_DEPS).toContain("hono");
84 expect(STACK_KEYSTONE_DEPS).toContain("drizzle-orm");
85 expect(STACK_KEYSTONE_DEPS).toContain("@anthropic-ai/sdk");
86 expect(TRENDING_FEATURE_CATALOGUE.length).toBeGreaterThan(0);
87 expect(KNOWN_CLAUDE_MODELS.length).toBeGreaterThan(0);
88 });
89});
90
91describe("advancementDedupeKey", () => {
92 it("produces a deterministic 32-char hex digest", () => {
93 const k1 = advancementDedupeKey("Bump hono 4 → 5");
94 const k2 = advancementDedupeKey("Bump hono 4 → 5");
95 expect(k1).toBe(k2);
96 expect(k1).toMatch(/^[0-9a-f]{32}$/);
97 });
98
99 it("is case-insensitive + trimmed", () => {
100 expect(advancementDedupeKey(" Bump HONO ")).toBe(
101 advancementDedupeKey("bump hono")
102 );
103 });
104
105 it("differs across distinct titles", () => {
106 expect(advancementDedupeKey("a")).not.toBe(advancementDedupeKey("b"));
107 });
108});
109
110describe("renderAdvancementBody", () => {
111 it("embeds the dedupe marker so LIKE lookup matches", () => {
112 const f = finding({ title: "Upgrade Sonnet 4 → 4.7" });
113 const key = advancementDedupeKey(f.title);
114 const body = renderAdvancementBody(f, key);
115 expect(body).toContain(`${ADVANCEMENT_DEDUPE_MARKER_PREFIX}${key}`);
116 expect(body).toContain("Suggested action");
117 expect(body).toContain(f.suggested_action);
118 });
119
120 it("uses a high-urgency badge for high findings", () => {
121 const body = renderAdvancementBody(finding({ urgency: "high" }), "k");
122 expect(body).toContain("high");
123 });
124
125 it("labels each kind correctly", () => {
126 expect(
127 renderAdvancementBody(finding({ kind: "model_release" }), "k")
128 ).toContain("Model release");
129 expect(
130 renderAdvancementBody(finding({ kind: "stack_bump" }), "k")
131 ).toContain("Stack version bump");
132 expect(
133 renderAdvancementBody(finding({ kind: "trending_feature" }), "k")
134 ).toContain("Trending feature");
135 });
136});
137
138// ---------------------------------------------------------------------------
139// suggestModelUpgrades — pure
140// ---------------------------------------------------------------------------
141
142describe("suggestModelUpgrades", () => {
143 const fixture: ClaudeModelEntry[] = [
144 { id: "sonnet-old", family: "sonnet", generation: 4, capability: 80, cost: 30, label: "Sonnet old" },
145 { id: "sonnet-new", family: "sonnet", generation: 4.7, capability: 92, cost: 30, label: "Sonnet new" },
146 { id: "sonnet-cheap-better", family: "sonnet", generation: 4.7, capability: 92, cost: 20, label: "Sonnet cheap" },
147 { id: "haiku-old", family: "haiku", generation: 4.5, capability: 60, cost: 10, label: "Haiku old" },
148 ];
149
150 it("suggests a newer same-family model when one exists", () => {
151 const out = suggestModelUpgrades("sonnet-old", fixture);
152 const titles = out.map((f) => f.title);
153 expect(titles.some((t) => t.includes("→"))).toBe(true);
154 expect(out.every((f) => f.kind === "model_release")).toBe(true);
155 });
156
157 it("suggests a cheaper-better variant when both exist", () => {
158 const out = suggestModelUpgrades("sonnet-old", fixture);
159 expect(out.some((f) => f.title.toLowerCase().includes("cost"))).toBe(true);
160 });
161
162 it("returns empty when configured model is already best in family", () => {
163 // strip the cheaper variant so only one current-best exists.
164 const trimmed = fixture.filter((m) => m.id !== "sonnet-cheap-better");
165 const out = suggestModelUpgrades("sonnet-new", trimmed);
166 expect(out).toEqual([]);
167 });
168
169 it("falls back to best-in-family when configured id is unknown", () => {
170 const out = suggestModelUpgrades("sonnet-legacy-2024", fixture);
171 expect(out.length).toBeGreaterThan(0);
172 expect(out[0].kind).toBe("model_release");
173 });
174
175 it("returns empty when family can't be guessed", () => {
176 const out = suggestModelUpgrades("unrecognized-model", fixture);
177 expect(out).toEqual([]);
178 });
179});
180
181// ---------------------------------------------------------------------------
182// runAdvancementScan — issue creation
183// ---------------------------------------------------------------------------
184
185describe("runAdvancementScan — issue creation", () => {
186 it("opens an issue per Claude finding and skips info-grade defaults", async () => {
187 const created: Array<{ title: string; body: string }> = [];
188 const audited: Array<{ title: string; issueNumber: number | null }> = [];
189
190 const result = await runAdvancementScan({
191 aiAvailable: () => true,
192 // Force the model probe to be empty so the assertion below targets only the Claude finding.
193 configuredModels: () => [],
194 // No stack-bump probe.
195 loadPackageJson: async () => null,
196 fetchLatestVersion: async () => null,
197 askSelfImprovement: async () => [
198 finding({ title: "Improve cold-start time", urgency: "high" }),
199 ],
200 askTrending: async () => [
201 finding({
202 kind: "trending_feature",
203 title: "Add per-PR observability dashboard",
204 urgency: "medium",
205 suggested_action: "Plumb a per-PR dashboard the way Vercel does.",
206 }),
207 ],
208 resolveSelfHostRepo: async () => REPO,
209 isDuplicate: async () => false,
210 openIssue: async (args) => {
211 created.push({ title: args.title, body: args.body });
212 return created.length;
213 },
214 recordAudit: async (f, _r, n) => {
215 audited.push({ title: f.title, issueNumber: n });
216 },
217 proposeBumpPr: async () => null,
218 recordScanComplete: async () => {},
219 });
220
221 expect(result.openedIssues).toBe(2);
222 expect(result.openedPrs).toBe(0);
223 expect(result.skippedDedupe).toBe(0);
224 expect(result.errors).toBe(0);
225 expect(created.map((c) => c.title).sort()).toEqual(
226 ["Add per-PR observability dashboard", "Improve cold-start time"].sort()
227 );
228 for (const c of created) {
229 const key = advancementDedupeKey(c.title);
230 expect(c.body).toContain(`${ADVANCEMENT_DEDUPE_MARKER_PREFIX}${key}`);
231 }
232 expect(audited).toHaveLength(2);
233 });
234
235 it("creates an issue for each model-release suggestion", async () => {
236 const created: string[] = [];
237 const result = await runAdvancementScan({
238 aiAvailable: () => false, // Skip Claude probes.
239 configuredModels: () => [KNOWN_CLAUDE_MODELS[0]!.id],
240 loadPackageJson: async () => null,
241 fetchLatestVersion: async () => null,
242 resolveSelfHostRepo: async () => REPO,
243 isDuplicate: async () => false,
244 openIssue: async (args) => {
245 created.push(args.title);
246 return created.length;
247 },
248 recordAudit: async () => {},
249 proposeBumpPr: async () => null,
250 recordScanComplete: async () => {},
251 });
252 // The curated catalogue has newer/cheaper entries vs the first model,
253 // so we expect at least one issue.
254 expect(result.openedIssues).toBeGreaterThan(0);
255 expect(created.every((t) => t.toLowerCase().includes("sonnet") || t.toLowerCase().includes("upgrade") || t.toLowerCase().includes("save"))).toBe(true);
256 });
257});
258
259// ---------------------------------------------------------------------------
260// Dedupe
261// ---------------------------------------------------------------------------
262
263describe("runAdvancementScan — dedupe", () => {
264 it("does not double-fire when isDuplicate returns true", async () => {
265 const created: string[] = [];
266 let dedupeChecks = 0;
267
268 const result = await runAdvancementScan({
269 aiAvailable: () => true,
270 configuredModels: () => [],
271 loadPackageJson: async () => null,
272 fetchLatestVersion: async () => null,
273 askSelfImprovement: async () => [
274 finding({ title: "Improve cold-start", urgency: "high" }),
275 ],
276 askTrending: async () => [],
277 resolveSelfHostRepo: async () => REPO,
278 isDuplicate: async () => {
279 dedupeChecks += 1;
280 return true;
281 },
282 openIssue: async (args) => {
283 created.push(args.title);
284 return 1;
285 },
286 recordAudit: async () => {},
287 proposeBumpPr: async () => null,
288 recordScanComplete: async () => {},
289 });
290
291 expect(dedupeChecks).toBe(1);
292 expect(created).toEqual([]);
293 expect(result.skippedDedupe).toBe(1);
294 expect(result.openedIssues).toBe(0);
295 });
296
297 it("passes the 30-day window to the duplicate lookup", async () => {
298 let observed = -1;
299 await runAdvancementScan({
300 aiAvailable: () => true,
301 configuredModels: () => [],
302 loadPackageJson: async () => null,
303 fetchLatestVersion: async () => null,
304 askSelfImprovement: async () => [finding({ urgency: "high" })],
305 askTrending: async () => [],
306 resolveSelfHostRepo: async () => REPO,
307 isDuplicate: async (_r, _k, days) => {
308 observed = days;
309 return true;
310 },
311 openIssue: async () => 1,
312 recordAudit: async () => {},
313 proposeBumpPr: async () => null,
314 recordScanComplete: async () => {},
315 });
316 expect(observed).toBe(ADVANCEMENT_DEDUPE_DAYS);
317 });
318
319 it("passes the sha256 dedupe key to the duplicate check", async () => {
320 let observed = "";
321 await runAdvancementScan({
322 aiAvailable: () => true,
323 configuredModels: () => [],
324 loadPackageJson: async () => null,
325 fetchLatestVersion: async () => null,
326 askSelfImprovement: async () => [
327 finding({ title: "Improve cold-start time" }),
328 ],
329 askTrending: async () => [],
330 resolveSelfHostRepo: async () => REPO,
331 isDuplicate: async (_r, key) => {
332 observed = key;
333 return true;
334 },
335 openIssue: async () => 1,
336 recordAudit: async () => {},
337 proposeBumpPr: async () => null,
338 recordScanComplete: async () => {},
339 });
340 expect(observed).toBe(advancementDedupeKey("Improve cold-start time"));
341 });
342});
343
344// ---------------------------------------------------------------------------
345// Stack bumps route to the migration assistant
346// ---------------------------------------------------------------------------
347
348describe("runAdvancementScan — stack bumps", () => {
349 it("hands a major-version bump to the migration assistant instead of opening an issue", async () => {
350 const bumps: Array<{ dep: string; from: string; to: string }> = [];
351 const issues: string[] = [];
352
353 const result = await runAdvancementScan({
354 aiAvailable: () => false,
355 configuredModels: () => [],
356 loadPackageJson: async () =>
357 JSON.stringify({
358 dependencies: { hono: "^3.0.0" },
359 }),
360 fetchLatestVersion: async (name) => (name === "hono" ? "4.5.0" : null),
361 resolveSelfHostRepo: async () => REPO,
362 isDuplicate: async () => false,
363 openIssue: async (args) => {
364 issues.push(args.title);
365 return issues.length;
366 },
367 recordAudit: async () => {},
368 proposeBumpPr: async (args) => {
369 bumps.push({
370 dep: args.dependency,
371 from: args.fromVersion,
372 to: args.toVersion,
373 });
374 return { branch: "ai-migration/hono", prNumber: 7 };
375 },
376 resolveBaseSha: async () => "deadbeefcafefeedfacefeeddeadbeefcafefeed",
377 recordScanComplete: async () => {},
378 });
379
380 expect(bumps).toEqual([
381 { dep: "hono", from: "^3.0.0", to: "4.5.0" },
382 ]);
383 // Migration assistant accepted -> no fallback issue.
384 expect(issues).toEqual([]);
385 expect(result.openedPrs).toBe(1);
386 expect(result.openedIssues).toBe(0);
387 });
388
389 it("falls back to an issue when the migration assistant declines", async () => {
390 const issues: string[] = [];
391 const result = await runAdvancementScan({
392 aiAvailable: () => false,
393 configuredModels: () => [],
394 loadPackageJson: async () =>
395 JSON.stringify({ dependencies: { hono: "^3.0.0" } }),
396 fetchLatestVersion: async () => "4.5.0",
397 resolveSelfHostRepo: async () => REPO,
398 isDuplicate: async () => false,
399 openIssue: async (args) => {
400 issues.push(args.title);
401 return issues.length;
402 },
403 recordAudit: async () => {},
404 proposeBumpPr: async () => null, // decline
405 resolveBaseSha: async () => "deadbeefcafefeedfacefeeddeadbeefcafefeed",
406 recordScanComplete: async () => {},
407 });
408 expect(issues.length).toBe(1);
409 expect(issues[0]).toContain("Bump hono");
410 expect(result.openedIssues).toBe(1);
411 expect(result.openedPrs).toBe(0);
412 });
413});
414
415// ---------------------------------------------------------------------------
416// Robustness
417// ---------------------------------------------------------------------------
418
419describe("runAdvancementScan — robustness", () => {
420 it("isolates per-finding failures — one bad issue insert does not stop the rest", async () => {
421 const created: string[] = [];
422
423 const result = await runAdvancementScan({
424 aiAvailable: () => true,
425 configuredModels: () => [],
426 loadPackageJson: async () => null,
427 fetchLatestVersion: async () => null,
428 askSelfImprovement: async () => [
429 finding({ title: "first", urgency: "high" }),
430 finding({ title: "second", urgency: "high" }),
431 ],
432 askTrending: async () => [],
433 resolveSelfHostRepo: async () => REPO,
434 isDuplicate: async () => false,
435 openIssue: async (args) => {
436 if (args.title === "first") throw new Error("DB blew up");
437 created.push(args.title);
438 return created.length;
439 },
440 recordAudit: async () => {},
441 proposeBumpPr: async () => null,
442 recordScanComplete: async () => {},
443 });
444
445 expect(created).toEqual(["second"]);
446 expect(result.openedIssues).toBe(1);
447 expect(result.errors).toBe(1);
448 });
449
450 it("returns a clean summary when AI is unavailable", async () => {
451 const result = await runAdvancementScan({
452 aiAvailable: () => false,
453 configuredModels: () => [],
454 loadPackageJson: async () => null,
455 fetchLatestVersion: async () => null,
456 resolveSelfHostRepo: async () => REPO,
457 isDuplicate: async () => false,
458 openIssue: async () => null,
459 recordAudit: async () => {},
460 proposeBumpPr: async () => null,
461 recordScanComplete: async () => {},
462 });
463 expect(result.findings).toEqual([]);
464 expect(result.openedIssues).toBe(0);
465 expect(result.openedPrs).toBe(0);
466 expect(result.errors).toBe(0);
467 });
468
469 it("caps total findings persisted per scan", async () => {
470 const created: string[] = [];
471 const tooMany: AdvancementFinding[] = [];
472 for (let i = 0; i < 20; i++) {
473 tooMany.push(finding({ title: `finding-${i}`, urgency: "high" }));
474 }
475 const result = await runAdvancementScan({
476 aiAvailable: () => true,
477 configuredModels: () => [],
478 loadPackageJson: async () => null,
479 fetchLatestVersion: async () => null,
480 askSelfImprovement: async () => tooMany,
481 askTrending: async () => [],
482 resolveSelfHostRepo: async () => REPO,
483 isDuplicate: async () => false,
484 openIssue: async (args) => {
485 created.push(args.title);
486 return created.length;
487 },
488 recordAudit: async () => {},
489 proposeBumpPr: async () => null,
490 recordScanComplete: async () => {},
491 maxFindings: 4,
492 });
493 expect(created.length).toBe(4);
494 expect(result.openedIssues).toBe(4);
495 expect(result.findings.length).toBe(4);
496 });
497
498 it("records a scan-complete audit row at the end", async () => {
499 let scanCompleteCalled = false;
500 await runAdvancementScan({
501 aiAvailable: () => false,
502 configuredModels: () => [],
503 loadPackageJson: async () => null,
504 fetchLatestVersion: async () => null,
505 resolveSelfHostRepo: async () => REPO,
506 isDuplicate: async () => false,
507 openIssue: async () => 1,
508 recordAudit: async () => {},
509 proposeBumpPr: async () => null,
510 recordScanComplete: async () => {
511 scanCompleteCalled = true;
512 },
513 });
514 expect(scanCompleteCalled).toBe(true);
515 });
516
517 it("does not crash when the self-host repo is missing", async () => {
518 const result = await runAdvancementScan({
519 aiAvailable: () => false,
520 configuredModels: () => [KNOWN_CLAUDE_MODELS[0]!.id],
521 loadPackageJson: async () => null,
522 fetchLatestVersion: async () => null,
523 resolveSelfHostRepo: async () => null,
524 isDuplicate: async () => false,
525 openIssue: async () => 1,
526 recordAudit: async () => {},
527 proposeBumpPr: async () => null,
528 recordScanComplete: async () => {},
529 });
530 // Model-release findings still surface in the result, but no issues
531 // are opened because the repo couldn't be resolved.
532 expect(result.findings.length).toBeGreaterThan(0);
533 expect(result.openedIssues).toBe(0);
534 });
535});
536
537// ---------------------------------------------------------------------------
538// Internal helpers
539// ---------------------------------------------------------------------------
540
541describe("internal helpers", () => {
542 it("countByKind tallies findings", () => {
543 const out = scannerInternals.countByKind([
544 finding({ kind: "model_release" }),
545 finding({ kind: "model_release" }),
546 finding({ kind: "trending_feature" }),
547 ]);
548 expect(out.model_release).toBe(2);
549 expect(out.trending_feature).toBe(1);
550 expect(out.stack_bump).toBe(0);
551 });
552
553 it("isPlausibleClaudeFinding rejects malformed rows", () => {
554 expect(scannerInternals.isPlausibleClaudeFinding({})).toBe(false);
555 expect(
556 scannerInternals.isPlausibleClaudeFinding({
557 title: "",
558 urgency: "high",
559 suggested_action: "x",
560 })
561 ).toBe(false);
562 expect(
563 scannerInternals.isPlausibleClaudeFinding({
564 title: "ok",
565 urgency: "spicy",
566 suggested_action: "x",
567 })
568 ).toBe(false);
569 expect(
570 scannerInternals.isPlausibleClaudeFinding({
571 title: "ok",
572 urgency: "high",
573 suggested_action: "x",
574 })
575 ).toBe(true);
576 });
577});
578
579// ---------------------------------------------------------------------------
580// DB-backed smoke (only runs when DATABASE_URL is set)
581// ---------------------------------------------------------------------------
582
583describe.skipIf(!HAS_DB)("runAdvancementScan — DB-backed smoke", () => {
584 it("calls the default recordAudit/openIssue when no overrides given", async () => {
585 // Just verifies the wiring — uses an unresolved repo so nothing
586 // actually inserts. Confirms the lib doesn't throw when default deps
587 // hit the live DB.
588 const result = await runAdvancementScan({
589 aiAvailable: () => false,
590 configuredModels: () => [],
591 loadPackageJson: async () => null,
592 fetchLatestVersion: async () => null,
593 resolveSelfHostRepo: async () => null,
594 });
595 expect(result.errors).toBeLessThanOrEqual(1);
596 });
597});