CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history
heal-loop.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 01e93d2 | 1 | /** |
| 2 | * Self-healing loop — GateTest integration. | |
| 3 | * | |
| 4 | * Continuously runs tests on a branch, interprets failures, auto-repairs, | |
| 5 | * resubmits, and repeats until green or max attempts reached. This is the | |
| 6 | * "nothing broken ships" guarantee taken to its logical conclusion. | |
| 7 | * | |
| 8 | * Flow: | |
| 9 | * 1. Run GateTest scan (or local test suite) | |
| 10 | * 2. If green → done | |
| 11 | * 3. If red → parse failure output | |
| 12 | * 4. Send failure context to Claude for repair | |
| 13 | * 5. Apply patches, commit, update ref | |
| 14 | * 6. Repeat from step 1 (max 3 attempts) | |
| 15 | * | |
| 16 | * The loop runs asynchronously and broadcasts SSE events at each step | |
| 17 | * so the UI can show live progress. | |
| 18 | */ | |
| 19 | ||
| 20 | import { spawn } from "bun"; | |
| 21 | import { readFile, writeFile, mkdir } from "fs/promises"; | |
| 22 | import { join } from "path"; | |
| 23 | import { getRepoPath } from "../git/repository"; | |
| 24 | import { broadcast } from "./sse"; | |
| 25 | import { config } from "./config"; | |
| 26 | import { db } from "../db"; | |
| 27 | import { gateRuns, repositories, users } from "../db/schema"; | |
| 28 | import { eq, and } from "drizzle-orm"; | |
| 29 | ||
| 30 | const MAX_HEAL_ATTEMPTS = 3; | |
| 31 | ||
| 32 | interface HealResult { | |
| 33 | success: boolean; | |
| 34 | attempts: number; | |
| 35 | repairs: Array<{ | |
| 36 | attempt: number; | |
| 37 | failureType: string; | |
| 38 | filesChanged: string[]; | |
| 39 | commitSha?: string; | |
| 40 | }>; | |
| 41 | finalStatus: "green" | "red" | "max_attempts" | "error"; | |
| 42 | error?: string; | |
| 43 | } | |
| 44 | ||
| 45 | interface TestFailure { | |
| 46 | type: "test" | "lint" | "typecheck" | "build" | "security" | "unknown"; | |
| 47 | message: string; | |
| 48 | file?: string; | |
| 49 | line?: number; | |
| 50 | details: string; | |
| 51 | } | |
| 52 | ||
| 53 | async function exec( | |
| 54 | cmd: string[], | |
| 55 | opts?: { cwd?: string; env?: Record<string, string>; timeout?: number } | |
| 56 | ): Promise<{ stdout: string; stderr: string; exitCode: number }> { | |
| 57 | const proc = spawn(cmd, { | |
| 58 | cwd: opts?.cwd, | |
| 59 | env: { ...process.env, ...opts?.env }, | |
| 60 | stdout: "pipe", | |
| 61 | stderr: "pipe", | |
| 62 | }); | |
| 63 | ||
| 64 | let timeoutId: ReturnType<typeof setTimeout> | null = null; | |
| 65 | if (opts?.timeout) { | |
| 66 | timeoutId = setTimeout(() => proc.kill(), opts.timeout); | |
| 67 | } | |
| 68 | ||
| 69 | const [stdout, stderr] = await Promise.all([ | |
| 70 | new Response(proc.stdout).text(), | |
| 71 | new Response(proc.stderr).text(), | |
| 72 | ]); | |
| 73 | const exitCode = await proc.exited; | |
| 74 | if (timeoutId) clearTimeout(timeoutId); | |
| 75 | return { stdout, stderr, exitCode }; | |
| 76 | } | |
| 77 | ||
| 78 | const AUTHOR_ENV = { | |
| 79 | GIT_AUTHOR_NAME: "GlueCron AI", | |
| 80 | GIT_AUTHOR_EMAIL: "ai@gluecron.com", | |
| 81 | GIT_COMMITTER_NAME: "GlueCron AI", | |
| 82 | GIT_COMMITTER_EMAIL: "ai@gluecron.com", | |
| 83 | }; | |
| 84 | ||
| 85 | /** | |
| 86 | * Parse test/build output to identify specific failures. | |
| 87 | */ | |
| 88 | function parseFailures(stdout: string, stderr: string): TestFailure[] { | |
| 89 | const combined = stdout + "\n" + stderr; | |
| 90 | const failures: TestFailure[] = []; | |
| 91 | ||
| 92 | // TypeScript errors: src/file.ts(10,5): error TS1234: message | |
| 93 | const tsErrors = combined.matchAll(/^(.+?)\((\d+),\d+\):\s*error\s+TS\d+:\s*(.+)$/gm); | |
| 94 | for (const m of tsErrors) { | |
| 95 | failures.push({ | |
| 96 | type: "typecheck", | |
| 97 | message: m[3], | |
| 98 | file: m[1], | |
| 99 | line: parseInt(m[2]), | |
| 100 | details: m[0], | |
| 101 | }); | |
| 102 | } | |
| 103 | ||
| 104 | // Test failures: ✗ test name ... expected X received Y | |
| 105 | const testFails = combined.matchAll(/(?:FAIL|✗|×)\s+(.+?)(?:\n[\s\S]*?(?:expected|Error|assert)[\s\S]*?)(?=\n(?:FAIL|✗|×|PASS|✓|\d+ pass)|\n\n)/gm); | |
| 106 | for (const m of testFails) { | |
| 107 | failures.push({ | |
| 108 | type: "test", | |
| 109 | message: m[1].trim(), | |
| 110 | details: m[0].slice(0, 500), | |
| 111 | }); | |
| 112 | } | |
| 113 | ||
| 114 | // ESLint errors | |
| 115 | const lintErrors = combined.matchAll(/^(.+?):(\d+):\d+\s+error\s+(.+?)\s+/gm); | |
| 116 | for (const m of lintErrors) { | |
| 117 | failures.push({ | |
| 118 | type: "lint", | |
| 119 | message: m[3], | |
| 120 | file: m[1], | |
| 121 | line: parseInt(m[2]), | |
| 122 | details: m[0], | |
| 123 | }); | |
| 124 | } | |
| 125 | ||
| 126 | // Build errors | |
| 127 | if (combined.includes("Build failed") || combined.includes("Cannot find module")) { | |
| 128 | const buildMatch = combined.match(/(?:Build failed|Cannot find module[^\n]+)/); | |
| 129 | failures.push({ | |
| 130 | type: "build", | |
| 131 | message: buildMatch?.[0] || "Build error", | |
| 132 | details: combined.slice(0, 500), | |
| 133 | }); | |
| 134 | } | |
| 135 | ||
| 136 | if (failures.length === 0 && (stdout.includes("fail") || stderr.includes("error"))) { | |
| 137 | failures.push({ | |
| 138 | type: "unknown", | |
| 139 | message: "Unrecognized failure pattern", | |
| 140 | details: combined.slice(0, 500), | |
| 141 | }); | |
| 142 | } | |
| 143 | ||
| 144 | return failures; | |
| 145 | } | |
| 146 | ||
| 147 | /** | |
| 148 | * Ask Claude to generate repair patches for identified failures. | |
| 149 | */ | |
| 150 | async function generateRepairPatches( | |
| 151 | worktreePath: string, | |
| 152 | failures: TestFailure[] | |
| 153 | ): Promise<Array<{ path: string; content: string; reason: string }>> { | |
| 154 | if (!config.anthropicApiKey) return []; | |
| 155 | ||
| 156 | const { getAnthropic, MODEL_SONNET, extractText } = await import("./ai-client"); | |
| 157 | const client = getAnthropic(); | |
| 158 | ||
| 159 | // Group failures by file | |
| 160 | const byFile = new Map<string, TestFailure[]>(); | |
| 161 | for (const f of failures) { | |
| 162 | const file = f.file || "unknown"; | |
| 163 | if (!byFile.has(file)) byFile.set(file, []); | |
| 164 | byFile.get(file)!.push(f); | |
| 165 | } | |
| 166 | ||
| 167 | const patches: Array<{ path: string; content: string; reason: string }> = []; | |
| 168 | ||
| 169 | for (const [file, fileFailures] of byFile) { | |
| 170 | if (file === "unknown") continue; | |
| 171 | ||
| 172 | let original: string; | |
| 173 | try { | |
| 174 | original = await readFile(join(worktreePath, file), "utf8"); | |
| 175 | } catch { | |
| 176 | continue; | |
| 177 | } | |
| 178 | ||
| 179 | const failureText = fileFailures | |
| 180 | .map((f, i) => `${i + 1}. [${f.type}] ${f.message}${f.line ? ` (line ${f.line})` : ""}\n ${f.details}`) | |
| 181 | .join("\n\n"); | |
| 182 | ||
| 183 | try { | |
| 184 | const message = await client.messages.create({ | |
| 185 | model: MODEL_SONNET, | |
| 186 | max_tokens: 8192, | |
| 187 | messages: [ | |
| 188 | { | |
| 189 | role: "user", | |
| 190 | content: `Fix the following failures in "${file}". Output ONLY the corrected file content — no prose, no code fences, no explanation. | |
| 191 | ||
| 192 | Failures: | |
| 193 | ${failureText} | |
| 194 | ||
| 195 | Current file content: | |
| 196 | ${original.slice(0, 50000)}`, | |
| 197 | }, | |
| 198 | ], | |
| 199 | }); | |
| 200 | ||
| 201 | const fixed = extractText(message).replace(/^```[\w]*\n?/, "").replace(/\n?```$/, ""); | |
| 202 | if (fixed && fixed !== original && fixed.length > 10) { | |
| 203 | patches.push({ | |
| 204 | path: file, | |
| 205 | content: fixed, | |
| 206 | reason: `Fix ${fileFailures.length} ${fileFailures[0].type} failure${fileFailures.length > 1 ? "s" : ""}`, | |
| 207 | }); | |
| 208 | } | |
| 209 | } catch (err) { | |
| 210 | console.error(`[heal-loop] Claude repair failed for ${file}:`, err); | |
| 211 | } | |
| 212 | } | |
| 213 | ||
| 214 | return patches; | |
| 215 | } | |
| 216 | ||
| 217 | /** | |
| 218 | * Run the test suite in a worktree. | |
| 219 | */ | |
| 220 | async function runTests( | |
| 221 | worktreePath: string | |
| 222 | ): Promise<{ passed: boolean; stdout: string; stderr: string }> { | |
| 223 | // Check if there's a package.json with test script | |
| 224 | try { | |
| 225 | const pkg = JSON.parse(await readFile(join(worktreePath, "package.json"), "utf8")); | |
| 226 | const testCmd = pkg.scripts?.test; | |
| 227 | if (testCmd) { | |
| 228 | const result = await exec(["bun", "test"], { cwd: worktreePath, timeout: 120_000 }); | |
| 229 | return { passed: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; | |
| 230 | } | |
| 231 | } catch {} | |
| 232 | ||
| 233 | // Fallback: try bun test directly | |
| 234 | const result = await exec(["bun", "test"], { cwd: worktreePath, timeout: 120_000 }); | |
| 235 | return { passed: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; | |
| 236 | } | |
| 237 | ||
| 238 | /** | |
| 239 | * Run the self-healing loop on a branch. | |
| 240 | * | |
| 241 | * Called after a push or PR merge attempt that failed gate checks. | |
| 242 | * The loop attempts to fix failures and push repairs back to the branch. | |
| 243 | */ | |
| 244 | export async function runHealLoop( | |
| 245 | owner: string, | |
| 246 | repo: string, | |
| 247 | branch: string, | |
| 248 | opts: { | |
| 249 | repositoryId?: string; | |
| 250 | pullRequestId?: string; | |
| 251 | triggerSource?: string; | |
| 252 | } = {} | |
| 253 | ): Promise<HealResult> { | |
| 254 | const repoDir = getRepoPath(owner, repo); | |
| 255 | const sseChannel = opts.repositoryId ? `gate:${opts.repositoryId}` : null; | |
| 256 | ||
| 257 | const result: HealResult = { | |
| 258 | success: false, | |
| 259 | attempts: 0, | |
| 260 | repairs: [], | |
| 261 | finalStatus: "error", | |
| 262 | }; | |
| 263 | ||
| 264 | for (let attempt = 1; attempt <= MAX_HEAL_ATTEMPTS; attempt++) { | |
| 265 | result.attempts = attempt; | |
| 266 | ||
| 267 | // Broadcast SSE: healing attempt starting | |
| 268 | if (sseChannel) { | |
| 269 | broadcast(sseChannel, "heal:attempt", { | |
| 270 | attempt, | |
| 271 | maxAttempts: MAX_HEAL_ATTEMPTS, | |
| 272 | branch, | |
| 273 | status: "running", | |
| 274 | }); | |
| 275 | } | |
| 276 | ||
| 277 | // Create worktree | |
| 278 | const wtPath = join(repoDir, `_heal_${Date.now()}_${attempt}`); | |
| 279 | const wt = await exec(["git", "worktree", "add", wtPath, branch], { cwd: repoDir }); | |
| 280 | if (wt.exitCode !== 0) { | |
| 281 | result.error = `Worktree creation failed: ${wt.stderr}`; | |
| 282 | result.finalStatus = "error"; | |
| 283 | break; | |
| 284 | } | |
| 285 | ||
| 286 | try { | |
| 287 | // Install deps if needed | |
| 288 | const pkgExists = await readFile(join(wtPath, "package.json"), "utf8").catch(() => null); | |
| 289 | if (pkgExists) { | |
| 290 | await exec(["bun", "install", "--frozen-lockfile"], { cwd: wtPath, timeout: 60_000 }); | |
| 291 | } | |
| 292 | ||
| 293 | // Run tests | |
| 294 | const testResult = await runTests(wtPath); | |
| 295 | ||
| 296 | if (testResult.passed) { | |
| 297 | result.success = true; | |
| 298 | result.finalStatus = "green"; | |
| 299 | ||
| 300 | if (sseChannel) { | |
| 301 | broadcast(sseChannel, "heal:green", { | |
| 302 | attempt, | |
| 303 | branch, | |
| 304 | totalRepairs: result.repairs.length, | |
| 305 | }); | |
| 306 | } | |
| 307 | break; | |
| 308 | } | |
| 309 | ||
| 310 | // Tests failed — parse failures | |
| 311 | const failures = parseFailures(testResult.stdout, testResult.stderr); | |
| 312 | console.log(`[heal-loop] Attempt ${attempt}: ${failures.length} failures detected`); | |
| 313 | ||
| 314 | if (failures.length === 0) { | |
| 315 | result.finalStatus = "red"; | |
| 316 | result.error = "Tests failed but no parseable failures found"; | |
| 317 | ||
| 318 | if (sseChannel) { | |
| 319 | broadcast(sseChannel, "heal:unparseable", { attempt, branch }); | |
| 320 | } | |
| 321 | break; | |
| 322 | } | |
| 323 | ||
| 324 | // Generate repair patches | |
| 325 | const patches = await generateRepairPatches(wtPath, failures); | |
| 326 | if (patches.length === 0) { | |
| 327 | result.finalStatus = "red"; | |
| 328 | result.error = "Could not generate repair patches"; | |
| 329 | ||
| 330 | if (sseChannel) { | |
| 331 | broadcast(sseChannel, "heal:no_patches", { attempt, branch, failures: failures.length }); | |
| 332 | } | |
| 333 | break; | |
| 334 | } | |
| 335 | ||
| 336 | // Apply patches | |
| 337 | for (const patch of patches) { | |
| 338 | const fullPath = join(wtPath, patch.path); | |
| 339 | await mkdir(join(fullPath, ".."), { recursive: true }).catch(() => {}); | |
| 340 | await writeFile(fullPath, patch.content, "utf8"); | |
| 341 | } | |
| 342 | ||
| 343 | // Commit repair | |
| 344 | await exec(["git", "add", "-A"], { cwd: wtPath }); | |
| 345 | const commitMsg = `fix(heal): auto-repair attempt ${attempt}/${MAX_HEAL_ATTEMPTS}\n\n${patches.map((p) => `- ${p.path}: ${p.reason}`).join("\n")}\n\n[GlueCron self-healing loop]`; | |
| 346 | const commit = await exec(["git", "commit", "-m", commitMsg], { cwd: wtPath, env: AUTHOR_ENV }); | |
| 347 | ||
| 348 | if (commit.exitCode !== 0) { | |
| 349 | result.error = `Commit failed: ${commit.stderr}`; | |
| 350 | continue; | |
| 351 | } | |
| 352 | ||
| 353 | const { stdout: sha } = await exec(["git", "rev-parse", "HEAD"], { cwd: wtPath }); | |
| 354 | await exec(["git", "update-ref", `refs/heads/${branch}`, sha.trim()], { cwd: repoDir }); | |
| 355 | ||
| 356 | result.repairs.push({ | |
| 357 | attempt, | |
| 358 | failureType: failures[0].type, | |
| 359 | filesChanged: patches.map((p) => p.path), | |
| 360 | commitSha: sha.trim(), | |
| 361 | }); | |
| 362 | ||
| 363 | if (sseChannel) { | |
| 364 | broadcast(sseChannel, "heal:repaired", { | |
| 365 | attempt, | |
| 366 | branch, | |
| 367 | sha: sha.trim(), | |
| 368 | filesChanged: patches.map((p) => p.path), | |
| 369 | }); | |
| 370 | } | |
| 371 | } finally { | |
| 372 | // Cleanup worktree | |
| 373 | await exec(["git", "worktree", "remove", "--force", wtPath], { cwd: repoDir }).catch(() => {}); | |
| 374 | } | |
| 375 | } | |
| 376 | ||
| 377 | if (!result.success && result.attempts >= MAX_HEAL_ATTEMPTS) { | |
| 378 | result.finalStatus = "max_attempts"; | |
| 379 | if (sseChannel) { | |
| 380 | broadcast(sseChannel, "heal:exhausted", { | |
| 381 | attempts: result.attempts, | |
| 382 | branch, | |
| 383 | repairs: result.repairs.length, | |
| 384 | }); | |
| 385 | } | |
| 386 | } | |
| 387 | ||
| 388 | // Record the heal loop result in gate_runs | |
| 389 | if (opts.repositoryId) { | |
| 390 | try { | |
| 391 | await db.insert(gateRuns).values({ | |
| 392 | repositoryId: opts.repositoryId, | |
| 393 | pullRequestId: opts.pullRequestId, | |
| 394 | commitSha: "heal-loop", | |
| 395 | ref: `refs/heads/${branch}`, | |
| 396 | gateName: "Self-heal loop", | |
| 397 | status: result.success ? "passed" : "failed", | |
| 398 | summary: `${result.attempts} attempt${result.attempts === 1 ? "" : "s"}, ${result.repairs.length} repair${result.repairs.length === 1 ? "" : "s"} — ${result.finalStatus}`, | |
| 399 | details: JSON.stringify(result), | |
| 400 | repairAttempted: result.repairs.length > 0, | |
| 401 | repairSucceeded: result.success, | |
| 402 | repairCommitSha: result.repairs[result.repairs.length - 1]?.commitSha, | |
| 403 | completedAt: new Date(), | |
| 404 | }); | |
| 405 | } catch (err) { | |
| 406 | console.error("[heal-loop] Failed to record gate run:", err); | |
| 407 | } | |
| 408 | } | |
| 409 | ||
| 410 | console.log(`[heal-loop] Complete: ${result.finalStatus} after ${result.attempts} attempts, ${result.repairs.length} repairs`); | |
| 411 | return result; | |
| 412 | } | |
| 413 | ||
| 414 | /** | |
| 415 | * Quick check: is the heal loop available? (needs AI for repair generation) | |
| 416 | */ | |
| 417 | export function isHealLoopEnabled(): boolean { | |
| 418 | return !!config.anthropicApiKey; | |
| 419 | } |