Blame · Line-by-line history
ai-tests.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 1e162a8 | 1 | /** |
| 2 | * Block D8 — AI-generated test suite helper. | |
| 3 | * | |
| 4 | * Given a source file from a repo, produces a *failing* test stub that | |
| 5 | * exercises the public surface of the file using whatever test framework | |
| 6 | * the repository appears to be using (bun:test, vitest, jest, pytest, | |
| 7 | * go test, etc.). | |
| 8 | * | |
| 9 | * The HTTP glue lives in `routes/ai-tests.tsx`; this module only exposes | |
| 10 | * pure helpers and an AI wrapper that NEVER throws. When Claude isn't | |
| 11 | * available (no API key, transport error, etc.) `generateTestStub` returns | |
| 12 | * an empty body and `framework: "fallback"` so the route can render a | |
| 13 | * "couldn't generate" message without crashing. | |
| 14 | */ | |
| 15 | ||
| 16 | import { | |
| 17 | MODEL_SONNET, | |
| 18 | extractText, | |
| 19 | getAnthropic, | |
| 20 | isAiAvailable, | |
| 21 | } from "./ai-client"; | |
| 22 | ||
| 23 | // --------------------------------------------------------------------------- | |
| 24 | // Language detection | |
| 25 | // --------------------------------------------------------------------------- | |
| 26 | ||
| 27 | export type TestLanguage = | |
| 28 | | "typescript" | |
| 29 | | "javascript" | |
| 30 | | "python" | |
| 31 | | "go" | |
| 32 | | "rust" | |
| 33 | | "java" | |
| 34 | | "ruby" | |
| 35 | | "other"; | |
| 36 | ||
| 37 | /** | |
| 38 | * Detect a coarse language bucket from a file path's extension. | |
| 39 | * Unknown / non-code paths return "other". | |
| 40 | */ | |
| 41 | export function detectLanguage(path: string): TestLanguage { | |
| 42 | const lower = path.toLowerCase(); | |
| 43 | const dot = lower.lastIndexOf("."); | |
| 44 | if (dot < 0) return "other"; | |
| 45 | const ext = lower.slice(dot + 1); | |
| 46 | switch (ext) { | |
| 47 | case "ts": | |
| 48 | case "tsx": | |
| 49 | case "mts": | |
| 50 | case "cts": | |
| 51 | return "typescript"; | |
| 52 | case "js": | |
| 53 | case "jsx": | |
| 54 | case "mjs": | |
| 55 | case "cjs": | |
| 56 | return "javascript"; | |
| 57 | case "py": | |
| 58 | return "python"; | |
| 59 | case "go": | |
| 60 | return "go"; | |
| 61 | case "rs": | |
| 62 | return "rust"; | |
| 63 | case "java": | |
| 64 | case "kt": | |
| 65 | return "java"; | |
| 66 | case "rb": | |
| 67 | return "ruby"; | |
| 68 | default: | |
| 69 | return "other"; | |
| 70 | } | |
| 71 | } | |
| 72 | ||
| 73 | // --------------------------------------------------------------------------- | |
| 74 | // Framework detection | |
| 75 | // --------------------------------------------------------------------------- | |
| 76 | ||
| 77 | /** | |
| 78 | * Given a language bucket and a flat list of repository files (paths, not | |
| 79 | * contents), return a short framework identifier that matches the | |
| 80 | * conventions the repo already appears to be using. | |
| 81 | * | |
| 82 | * The returned string is what gets plumbed into Claude prompts and is used | |
| 83 | * to compute the suggested test file path. | |
| 84 | */ | |
| 85 | export function detectTestFramework( | |
| 86 | language: TestLanguage, | |
| 87 | repoFiles: string[] | |
| 88 | ): string { | |
| 89 | const files = repoFiles.map((f) => f.toLowerCase()); | |
| 90 | const has = (needle: string | RegExp): boolean => { | |
| 91 | if (typeof needle === "string") return files.some((f) => f === needle); | |
| 92 | return files.some((f) => needle.test(f)); | |
| 93 | }; | |
| 94 | ||
| eb3b81a | 95 | // Python always uses pytest (most widely adopted test runner). |
| 96 | if (language === "python") return "pytest"; | |
| 1e162a8 | 97 | |
| 98 | if (language === "go") return "go test"; | |
| 99 | if (language === "rust") return "cargo test"; | |
| 100 | if (language === "java") return "junit"; | |
| 101 | if (language === "ruby") return has("gemfile") ? "rspec" : "minitest"; | |
| 102 | ||
| 103 | // JavaScript / TypeScript — multiple competing frameworks. | |
| 104 | if (language === "typescript" || language === "javascript" || language === "other") { | |
| 105 | if (has(/vitest\.config\.(ts|js|mjs|cjs)$/) || has("vitest.config.ts")) | |
| 106 | return "vitest"; | |
| 107 | if (has(/jest\.config(\..+)?$/)) return "jest"; | |
| 108 | if (has(/\.mocharc(\..+)?$/) || has("mocha.opts")) return "mocha"; | |
| 109 | if (has("playwright.config.ts") || has("playwright.config.js")) | |
| 110 | return "playwright"; | |
| 111 | ||
| 112 | const usesBun = | |
| 113 | has("bun.lockb") || | |
| 114 | has("bunfig.toml") || | |
| 115 | files.some((f) => f.endsWith("package.json")); | |
| 116 | if (usesBun) return "bun:test"; | |
| 117 | } | |
| 118 | ||
| 119 | return "bun:test"; | |
| 120 | } | |
| 121 | ||
| 122 | // --------------------------------------------------------------------------- | |
| 123 | // Suggested test path | |
| 124 | // --------------------------------------------------------------------------- | |
| 125 | ||
| 126 | /** | |
| 127 | * Compute a deterministic path for the generated test file. The rules are | |
| 128 | * conventional rather than exhaustive — reviewers can always move the file | |
| 129 | * after generation. | |
| 130 | */ | |
| 131 | export function suggestedTestPath( | |
| 132 | sourcePath: string, | |
| 133 | language: TestLanguage, | |
| 134 | framework: string | |
| 135 | ): string { | |
| 136 | const parts = sourcePath.split("/"); | |
| 137 | const file = parts[parts.length - 1] || sourcePath; | |
| 138 | const dir = parts.slice(0, -1).join("/"); | |
| 139 | const dotIdx = file.lastIndexOf("."); | |
| 140 | const base = dotIdx > 0 ? file.slice(0, dotIdx) : file; | |
| 141 | const ext = dotIdx > 0 ? file.slice(dotIdx) : ""; | |
| 142 | ||
| 143 | // Python: siblings `test_foo.py` is near-universal. | |
| 144 | if (language === "python" || framework === "pytest") { | |
| 145 | return dir ? `${dir}/test_${base}.py` : `test_${base}.py`; | |
| 146 | } | |
| 147 | ||
| 148 | // Go: `foo_test.go` adjacent to source. | |
| 149 | if (language === "go" || framework === "go test") { | |
| 150 | return dir ? `${dir}/${base}_test.go` : `${base}_test.go`; | |
| 151 | } | |
| 152 | ||
| 153 | // Rust: convention is `#[cfg(test)] mod tests` inline, but for a standalone | |
| 154 | // stub we drop a file into `tests/`. | |
| 155 | if (language === "rust" || framework === "cargo test") { | |
| 156 | return `tests/${base}_test.rs`; | |
| 157 | } | |
| 158 | ||
| 159 | // Java / Kotlin | |
| 160 | if (language === "java" || framework === "junit") { | |
| 161 | const klass = base.charAt(0).toUpperCase() + base.slice(1); | |
| 162 | return dir | |
| 163 | ? `${dir.replace(/\/main\//, "/test/")}/${klass}Test${ext || ".java"}` | |
| 164 | : `${klass}Test${ext || ".java"}`; | |
| 165 | } | |
| 166 | ||
| 167 | // Ruby | |
| 168 | if (language === "ruby") { | |
| 169 | if (framework === "rspec") { | |
| 170 | return dir ? `spec/${dir}/${base}_spec.rb` : `spec/${base}_spec.rb`; | |
| 171 | } | |
| 172 | return dir ? `test/${dir}/${base}_test.rb` : `test/${base}_test.rb`; | |
| 173 | } | |
| 174 | ||
| 175 | // JS/TS with bun / jest / vitest — prefer `__tests__/<name>.test.<ext>`. | |
| 176 | const testExt = ext === ".tsx" ? ".test.ts" : ext.replace(/^\./, ".test."); | |
| 177 | const safeExt = testExt || ".test.ts"; | |
| 178 | ||
| 179 | if (framework === "bun:test") { | |
| 180 | // If the source is already under src/, put tests under src/__tests__/. | |
| 181 | if (dir.startsWith("src/")) { | |
| 182 | return `src/__tests__/${base}${safeExt}`; | |
| 183 | } | |
| 184 | if (dir === "src") { | |
| 185 | return `src/__tests__/${base}${safeExt}`; | |
| 186 | } | |
| 187 | if (dir) return `${dir}/__tests__/${base}${safeExt}`; | |
| 188 | return `__tests__/${base}${safeExt}`; | |
| 189 | } | |
| 190 | ||
| 191 | // vitest / jest default to sibling `.test.` file. | |
| 192 | if (dir) return `${dir}/${base}${safeExt}`; | |
| 193 | return `${base}${safeExt}`; | |
| 194 | } | |
| 195 | ||
| 196 | // --------------------------------------------------------------------------- | |
| 197 | // Prompt | |
| 198 | // --------------------------------------------------------------------------- | |
| 199 | ||
| 200 | export interface TestGenOpts { | |
| 201 | path: string; | |
| 202 | language: string; | |
| 203 | framework: string; | |
| 204 | sourceCode: string; | |
| 205 | apiHints?: string; | |
| 206 | } | |
| 207 | ||
| 208 | /** | |
| 209 | * Build the user prompt that instructs Claude to emit a failing test stub. | |
| 210 | * Returning the prompt as a pure function keeps it easy to test. | |
| 211 | */ | |
| 212 | export function buildTestsPrompt(opts: TestGenOpts): string { | |
| 213 | const { path, language, framework, sourceCode, apiHints } = opts; | |
| 214 | const trimmed = sourceCode.length > 40_000 | |
| 215 | ? sourceCode.slice(0, 40_000) + "\n// ... (truncated)" | |
| 216 | : sourceCode; | |
| 217 | ||
| 218 | return `You are writing an initial failing test suite for an open-source project. | |
| 219 | ||
| 220 | Source file path: \`${path}\` | |
| 221 | Detected language: ${language} | |
| 222 | Detected test framework: ${framework} | |
| 223 | ||
| 224 | Write a *failing* test stub — the tests should compile / import cleanly where possible, but assertions MUST fail (or use explicit \`fail()\` / \`todo\` / \`skip\` markers where the framework supports them) so a developer is forced to review, fill in expected values, and confirm the intended behaviour. Prefer realistic \`expect(...)\` calls with placeholder expected values that are obviously wrong (like \`TODO\`) rather than empty bodies. | |
| 225 | ||
| 226 | Rules: | |
| 227 | - Exercise every exported / public symbol you can see in the source. | |
| 228 | - Use only idioms native to "${framework}". | |
| 229 | - No explanations, no Markdown, no surrounding prose — return ONLY the test file body. | |
| 230 | - If imports are needed, compute paths relative to the source file at \`${path}\`. | |
| 231 | - Leave a top-of-file comment that this stub was generated by gluecron's AI test helper and MUST be reviewed before being committed. | |
| 232 | ${apiHints ? `\nAdditional hints about the public API:\n${apiHints}\n` : ""} | |
| 233 | Source file contents: | |
| 234 | \`\`\` | |
| 235 | ${trimmed} | |
| 236 | \`\`\` | |
| 237 | `; | |
| 238 | } | |
| 239 | ||
| 240 | // --------------------------------------------------------------------------- | |
| 241 | // Claude wrapper | |
| 242 | // --------------------------------------------------------------------------- | |
| 243 | ||
| 244 | export interface TestStubResult { | |
| 245 | code: string; | |
| 246 | suggestedPath: string; | |
| 247 | framework: string; | |
| 248 | language: string; | |
| 249 | } | |
| 250 | ||
| 251 | /** | |
| 252 | * Strip a leading/trailing Markdown fence (```lang ... ```) that Claude will | |
| 253 | * sometimes add around the returned body, even when told not to. | |
| 254 | */ | |
| 255 | function stripCodeFences(raw: string): string { | |
| 256 | let text = raw.trim(); | |
| 257 | const fence = text.match(/^```[a-zA-Z0-9_+-]*\n?([\s\S]*?)\n?```\s*$/); | |
| 258 | if (fence) return fence[1].trim(); | |
| 259 | // Partial fences (just the opener) — drop them. | |
| 260 | if (text.startsWith("```")) { | |
| 261 | const nl = text.indexOf("\n"); | |
| 262 | if (nl > -1) text = text.slice(nl + 1); | |
| 263 | } | |
| 264 | if (text.endsWith("```")) text = text.slice(0, -3); | |
| 265 | return text.trim(); | |
| 266 | } | |
| 267 | ||
| 268 | /** | |
| 269 | * Ask Claude Sonnet to produce a failing test stub for the given source. | |
| 270 | * Never throws. On any error (AI unavailable, network failure, empty | |
| 271 | * response) returns `{ code: "", framework: "fallback", ... }`. | |
| 272 | */ | |
| 273 | export async function generateTestStub( | |
| 274 | opts: TestGenOpts | |
| 275 | ): Promise<TestStubResult> { | |
| 276 | const lang = (opts.language as TestLanguage) || "other"; | |
| 277 | const suggestedPath = suggestedTestPath(opts.path, lang, opts.framework); | |
| 278 | ||
| 279 | if (!isAiAvailable()) { | |
| 280 | return { | |
| 281 | code: "", | |
| 282 | suggestedPath, | |
| 283 | framework: "fallback", | |
| 284 | language: opts.language, | |
| 285 | }; | |
| 286 | } | |
| 287 | ||
| 288 | try { | |
| 289 | const client = getAnthropic(); | |
| 290 | const prompt = buildTestsPrompt(opts); | |
| 291 | const message = await client.messages.create({ | |
| 292 | model: MODEL_SONNET, | |
| 293 | max_tokens: 2048, | |
| 294 | messages: [{ role: "user", content: prompt }], | |
| 295 | }); | |
| 296 | const raw = extractText(message); | |
| 297 | const code = stripCodeFences(raw); | |
| 298 | if (!code) { | |
| 299 | return { | |
| 300 | code: "", | |
| 301 | suggestedPath, | |
| 302 | framework: "fallback", | |
| 303 | language: opts.language, | |
| 304 | }; | |
| 305 | } | |
| 306 | return { | |
| 307 | code, | |
| 308 | suggestedPath, | |
| 309 | framework: opts.framework, | |
| 310 | language: opts.language, | |
| 311 | }; | |
| 312 | } catch { | |
| 313 | return { | |
| 314 | code: "", | |
| 315 | suggestedPath, | |
| 316 | framework: "fallback", | |
| 317 | language: opts.language, | |
| 318 | }; | |
| 319 | } | |
| 320 | } | |
| 321 | ||
| 322 | // --------------------------------------------------------------------------- | |
| 323 | // Content types for ?format=raw | |
| 324 | // --------------------------------------------------------------------------- | |
| 325 | ||
| 326 | /** | |
| 327 | * Return a suitable `Content-Type` header for a generated test file, based | |
| 328 | * on the language bucket. Defaults to `text/plain; charset=utf-8`. | |
| 329 | */ | |
| 330 | export function contentTypeFor(language: string): string { | |
| 331 | switch (language) { | |
| 332 | case "typescript": | |
| 333 | return "application/typescript; charset=utf-8"; | |
| 334 | case "javascript": | |
| 335 | return "application/javascript; charset=utf-8"; | |
| 336 | case "python": | |
| 337 | return "text/x-python; charset=utf-8"; | |
| 338 | case "go": | |
| 339 | return "text/x-go; charset=utf-8"; | |
| 340 | case "rust": | |
| 341 | return "text/x-rust; charset=utf-8"; | |
| 342 | case "java": | |
| 343 | return "text/x-java-source; charset=utf-8"; | |
| 344 | case "ruby": | |
| 345 | return "text/x-ruby; charset=utf-8"; | |
| 346 | default: | |
| 347 | return "text/plain; charset=utf-8"; | |
| 348 | } | |
| 349 | } |