Pre-launch — Gluecron is in final validation. Public signups and git hosting for non-owner users open after launch review.
ccantynz/Gluecron.com
CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history

deploy-watcher.ts

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

Gluecron.com/src/lib/agents/deploy-watcher.ts
deploy-watcher.tsBlame455 lines · 1 contributor
Normal viewRaw
055ebc4Claude1/**
2 * Block K7 — Deploy Watcher Agent.
3 *
4 * Fires after Crontech reports a deployment for a commit. Watches the
5 * deployment and consumes prod error signals (ingested via K9) for the same
6 * commit. If the deploy fails outright, OR if the error-signal count exceeds
7 * ERROR_THRESHOLD within the watch window, the watcher calls Crontech's
8 * `rollbackDeployment` and opens a P0 incident issue on the repo.
9 *
10 * Design rules (mirrors heal-bot.ts + fix-agent.ts):
11 * - Never throws. Every DB / network error returns `{ ok, summary }`.
12 * - Runs inside K1's `executeAgentRun` wrapper.
13 * - Cost: 2¢ flat per watch (Crontech compute proxy).
14 * - No new tables — consumes `deployments`, `prod_signals`, `issues`.
15 */
16
17import { and, eq, gte, sql } from "drizzle-orm";
18import { db } from "../../db";
19import {
20 issues,
21 repositories,
22 users,
23} from "../../db/schema";
24import {
25 rollbackDeployment,
26 watchDeployment,
27 type DeployWatchResult,
28} from "../crontech-client";
29import {
30 executeAgentRun,
31 startAgentRun,
32 type AgentExecutorContext,
33} from "../agent-runtime";
34
35// ---------------------------------------------------------------------------
36// Types
37// ---------------------------------------------------------------------------
38
39export interface RunDeployWatcherArgs {
40 repositoryId: string;
41 deployId: string;
42 commitSha: string;
43 triggerBy?: string | null;
44}
45
46export interface RunDeployWatcherResult {
47 ok: boolean;
48 summary: string;
49 runId: string | null;
50 rolledBack: boolean;
51 incidentIssueNumber: number | null;
52}
53
54// ---------------------------------------------------------------------------
55// Constants
56// ---------------------------------------------------------------------------
57
58export const DEPLOY_WATCHER_COST_CENTS = 2;
59
60/** Error signals of (error|critical) severity needed to auto-rollback. */
61export const DEPLOY_WATCHER_ERROR_THRESHOLD = 5;
62
63/** How long to watch before declaring the deploy healthy. */
64export const DEPLOY_WATCHER_WINDOW_MS = 15 * 60 * 1000;
65
66/** Poll interval inside the window. */
67export const DEPLOY_WATCHER_POLL_MS = 30 * 1000;
68
69export const DEPLOY_WATCHER_SLUG = "agent-deploy-watcher";
70export const DEPLOY_WATCHER_BOT_USERNAME = "agent-deploy-watcher[bot]";
71
72// ---------------------------------------------------------------------------
73// Pure helpers (unit-testable)
74// ---------------------------------------------------------------------------
75
76/**
77 * Is the given deploy-watch result a rollback trigger? Either the deploy
78 * itself failed, or the error-signal threshold was breached.
79 */
80export function shouldRollback(params: {
81 watchResult: DeployWatchResult;
82 errorSignalCount: number;
83 threshold: number;
84}): { rollback: boolean; reason: string } {
85 if (params.watchResult.finalStatus === "failed") {
86 return { rollback: true, reason: "deploy reported status=failed" };
87 }
88 if (params.watchResult.finalStatus === "rolled_back") {
89 return { rollback: false, reason: "deploy already rolled back" };
90 }
91 if (params.errorSignalCount >= params.threshold) {
92 return {
93 rollback: true,
94 reason: `${params.errorSignalCount} error signals ≥ threshold ${params.threshold}`,
95 };
96 }
97 return { rollback: false, reason: "deploy healthy" };
98}
99
100/**
101 * Deterministic P0 incident issue body. Lists the top error hashes + a link
102 * back to the commit.
103 */
104export function renderIncidentIssueBody(params: {
105 commitSha: string;
106 ownerUsername: string;
107 repoName: string;
108 deployId: string;
109 reason: string;
110 topErrors: Array<{ hash: string; message: string; count: number }>;
111}): string {
112 const {
113 commitSha,
114 ownerUsername,
115 repoName,
116 deployId,
117 reason,
118 topErrors,
119 } = params;
120 const shortSha = commitSha.slice(0, 7);
121 const lines: string[] = [];
122 lines.push(`**Automatic regression rollback triggered by the deploy-watcher agent.**`);
123 lines.push("");
124 lines.push(`- **Commit:** [\`${shortSha}\`](/${ownerUsername}/${repoName}/commit/${commitSha})`);
125 lines.push(`- **Deployment:** \`${deployId}\``);
126 lines.push(`- **Reason:** ${reason}`);
127 lines.push("");
128 if (topErrors.length > 0) {
129 lines.push(`### Top errors`);
130 lines.push("");
131 lines.push("| Hash | Count | Message |");
132 lines.push("| --- | ---: | --- |");
133 for (const e of topErrors.slice(0, 10)) {
134 const safeMsg = e.message.replace(/\|/g, "\\|").slice(0, 120);
135 lines.push(`| \`${e.hash}\` | ${e.count} | ${safeMsg} |`);
136 }
137 lines.push("");
138 }
139 lines.push(`_Generated by ${DEPLOY_WATCHER_BOT_USERNAME}._`);
140 return lines.join("\n");
141}
142
143export function buildDeployWatcherSummary(params: {
144 offline: boolean;
145 rolledBack: boolean;
146 reason: string;
147 incidentIssueNumber: number | null;
148 watchedForMs: number;
149 errorSignalCount: number;
150}): string {
151 if (params.offline) return "crontech offline; watch skipped";
152 if (params.rolledBack) {
153 const issueLabel =
154 params.incidentIssueNumber !== null
155 ? `#${params.incidentIssueNumber}`
156 : "(unknown issue)";
157 return `ROLLED BACK — ${params.reason}; opened ${issueLabel}`;
158 }
159 const secs = Math.round(params.watchedForMs / 1000);
160 return `deploy healthy — watched ${secs}s, ${params.errorSignalCount} signal(s)`;
161}
162
163// ---------------------------------------------------------------------------
164// Internal DB helpers — defensive, never throw.
165// ---------------------------------------------------------------------------
166
167interface RepoContext {
168 repoId: string;
169 repoName: string;
170 ownerUsername: string;
171 ownerId: string;
172}
173
174async function resolveRepoContext(
175 repositoryId: string
176): Promise<RepoContext | null> {
177 try {
178 const rows = await db
179 .select({
180 repoId: repositories.id,
181 repoName: repositories.name,
182 ownerUsername: users.username,
183 ownerId: repositories.ownerId,
184 })
185 .from(repositories)
186 .innerJoin(users, eq(users.id, repositories.ownerId))
187 .where(eq(repositories.id, repositoryId))
188 .limit(1);
189 const row = rows[0];
190 if (!row) return null;
191 return row;
192 } catch (err) {
193 console.error("[deploy-watcher] resolveRepoContext:", err);
194 return null;
195 }
196}
197
198/**
199 * Pick the author_id for the incident issue. Preference:
200 * 1. a "bot" user named DEPLOY_WATCHER_BOT_USERNAME,
201 * 2. the repo owner.
202 */
203async function resolveWatcherAuthorId(ownerId: string): Promise<string | null> {
204 try {
205 const [bot] = await db
206 .select({ id: users.id })
207 .from(users)
208 .where(eq(users.username, DEPLOY_WATCHER_BOT_USERNAME))
209 .limit(1);
210 if (bot?.id) return bot.id;
211 } catch (err) {
212 console.error("[deploy-watcher] bot author lookup:", err);
213 }
214 return ownerId;
215}
216
217interface SignalAggregate {
218 hash: string;
219 message: string;
220 count: number;
221 severity: string;
222}
223
224/**
225 * Count error/critical signals for the given commit that have been recorded
226 * since `since`, and return the top-N aggregated signals for the incident
227 * body.
228 *
229 * Uses raw sql because we filter on severity in SQL for efficiency.
230 */
231async function countAndTopErrors(
232 repositoryId: string,
233 commitSha: string,
234 since: Date
235): Promise<{ count: number; top: SignalAggregate[] }> {
236 try {
237 const rows = (await db.execute(sql`
238 SELECT error_hash, message, count, severity
239 FROM prod_signals
240 WHERE repository_id = ${repositoryId}
241 AND commit_sha = ${commitSha.toLowerCase()}
242 AND severity IN ('error', 'critical')
243 AND last_seen >= ${since.toISOString()}
244 ORDER BY count DESC
245 LIMIT 25
246 `)) as unknown as Array<Record<string, unknown>>;
247 if (!Array.isArray(rows)) return { count: 0, top: [] };
248 const list = rows.map((r) => ({
249 hash: String(r.error_hash ?? ""),
250 message: String(r.message ?? ""),
251 count: Number(r.count ?? 1),
252 severity: String(r.severity ?? "error"),
253 }));
254 // Total count = sum of counts (each signal row is one distinct error,
255 // bumped per occurrence).
256 const total = list.reduce((acc, r) => acc + r.count, 0);
257 return { count: total, top: list };
258 } catch (err) {
259 console.error("[deploy-watcher] countAndTopErrors:", err);
260 return { count: 0, top: [] };
261 }
262}
263
264// ---------------------------------------------------------------------------
265// Entry point
266// ---------------------------------------------------------------------------
267
268/**
269 * Run the deploy watcher for a single deployment. Never throws.
270 */
271export async function runDeployWatcher(
272 args: RunDeployWatcherArgs
273): Promise<RunDeployWatcherResult> {
274 const emptyResult = (summary: string): RunDeployWatcherResult => ({
275 ok: false,
276 summary,
277 runId: null,
278 rolledBack: false,
279 incidentIssueNumber: null,
280 });
281
282 if (
283 !args ||
284 typeof args.repositoryId !== "string" ||
285 !args.repositoryId ||
286 typeof args.deployId !== "string" ||
287 !args.deployId ||
288 typeof args.commitSha !== "string" ||
289 !args.commitSha
290 ) {
291 return emptyResult("invalid args: missing repositoryId/deployId/commitSha");
292 }
293
294 const trigger = args.triggerBy ? "manual" : "scheduled";
295
296 const run = await startAgentRun({
297 repositoryId: args.repositoryId,
298 kind: "deploy_watcher",
299 trigger,
300 triggerRef: args.deployId,
301 });
302 if (!run) {
303 return emptyResult("could not open agent_runs row");
304 }
305
306 let finalSummary = "not started";
307 let rolledBack = false;
308 let incidentIssueNumber: number | null = null;
309
310 await executeAgentRun(run.id, async (ctx: AgentExecutorContext) => {
311 await ctx.appendLog(
312 `[deploy-watcher] starting watch for deploy ${args.deployId} @ ${args.commitSha.slice(0, 7)} (trigger=${trigger})`
313 );
314
315 const repo = await resolveRepoContext(args.repositoryId);
316 if (!repo) {
317 finalSummary = "repo lookup failed";
318 await ctx.appendLog("[deploy-watcher] repo lookup failed; aborting");
319 return { ok: false, summary: finalSummary };
320 }
321
322 const repoSlug = `${repo.ownerUsername}/${repo.repoName}`;
323 const watchStart = new Date();
324
325 await ctx.appendLog(
326 `[deploy-watcher] calling crontech.watchDeployment for ${repoSlug}`
327 );
328 const watchResult = await watchDeployment({
329 repo: repoSlug,
330 deployId: args.deployId,
331 maxWaitMs: DEPLOY_WATCHER_WINDOW_MS,
332 pollIntervalMs: DEPLOY_WATCHER_POLL_MS,
333 });
334 await ctx.recordCost(0, 0, DEPLOY_WATCHER_COST_CENTS);
335
336 if (watchResult.offline) {
337 finalSummary = buildDeployWatcherSummary({
338 offline: true,
339 rolledBack: false,
340 reason: "",
341 incidentIssueNumber: null,
342 watchedForMs: 0,
343 errorSignalCount: 0,
344 });
345 await ctx.appendLog(`[deploy-watcher] ${finalSummary}`);
346 return { ok: true, summary: finalSummary };
347 }
348
349 const { count: errorSignalCount, top: topErrors } = await countAndTopErrors(
350 args.repositoryId,
351 args.commitSha,
352 watchStart
353 );
354 await ctx.appendLog(
355 `[deploy-watcher] watch complete: finalStatus=${watchResult.finalStatus}, errorSignals=${errorSignalCount}, watchedForMs=${watchResult.watchedForMs}`
356 );
357
358 const decision = shouldRollback({
359 watchResult,
360 errorSignalCount,
361 threshold: DEPLOY_WATCHER_ERROR_THRESHOLD,
362 });
363
364 if (!decision.rollback) {
365 finalSummary = buildDeployWatcherSummary({
366 offline: false,
367 rolledBack: false,
368 reason: decision.reason,
369 incidentIssueNumber: null,
370 watchedForMs: watchResult.watchedForMs,
371 errorSignalCount,
372 });
373 await ctx.appendLog(`[deploy-watcher] ${decision.reason}`);
374 return { ok: true, summary: finalSummary };
375 }
376
377 await ctx.appendLog(
378 `[deploy-watcher] ROLLBACK TRIGGERED: ${decision.reason}`
379 );
380
381 const rollbackOk = await rollbackDeployment({
382 repo: repoSlug,
383 deployId: args.deployId,
384 });
385 rolledBack = rollbackOk;
386 if (!rollbackOk) {
387 await ctx.appendLog(
388 `[deploy-watcher] crontech rollback failed; opening incident anyway`
389 );
390 }
391
392 // Open a P0 incident issue.
393 const authorId = await resolveWatcherAuthorId(repo.ownerId);
394 if (!authorId) {
395 finalSummary = `rollback ${rollbackOk ? "ok" : "failed"}; no author for incident`;
396 await ctx.appendLog(`[deploy-watcher] ${finalSummary}`);
397 return { ok: false, summary: finalSummary };
398 }
399
400 const body = renderIncidentIssueBody({
401 commitSha: args.commitSha,
402 ownerUsername: repo.ownerUsername,
403 repoName: repo.repoName,
404 deployId: args.deployId,
405 reason: decision.reason,
406 topErrors,
407 });
408
409 try {
410 const [inserted] = await db
411 .insert(issues)
412 .values({
413 repositoryId: repo.repoId,
414 authorId,
415 title: `P0: Regression in ${args.commitSha.slice(0, 7)}`,
416 body,
417 })
418 .returning();
419 incidentIssueNumber = inserted?.number ?? null;
420 } catch (err) {
421 await ctx.appendLog(
422 `[deploy-watcher] incident issue insert failed: ${(err as Error).message}`
423 );
424 }
425
426 finalSummary = buildDeployWatcherSummary({
427 offline: false,
428 rolledBack,
429 reason: decision.reason,
430 incidentIssueNumber,
431 watchedForMs: watchResult.watchedForMs,
432 errorSignalCount,
433 });
434 await ctx.appendLog(`[deploy-watcher] ${finalSummary}`);
435 return { ok: rolledBack, summary: finalSummary };
436 });
437
438 return {
439 ok: true,
440 summary: finalSummary,
441 runId: run.id,
442 rolledBack,
443 incidentIssueNumber,
444 };
445}
446
447export const __internal = {
448 DEPLOY_WATCHER_COST_CENTS,
449 DEPLOY_WATCHER_ERROR_THRESHOLD,
450 DEPLOY_WATCHER_WINDOW_MS,
451 DEPLOY_WATCHER_POLL_MS,
452 resolveRepoContext,
453 resolveWatcherAuthorId,
454 countAndTopErrors,
455};