Blame · Line-by-line history

`flywheel.ts`

Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.

flywheel.tsBlame489 lines · 1 contributor

Normal view Raw

b2ff5c7Claude	1	/**
	2	* Flywheel — the learning engine that makes every AI review smarter.
	3	*
	4	* Aggregates review_outcomes and gate_runs into patterns, then injects
	5	* those patterns into future AI prompts so the system learns from every
	6	* review, every merge, every failure.
	7	*/
	8
	9	import { eq, and, desc, gte, sql } from "drizzle-orm";
	10	import { db } from "../db";
	11	import {
	12	reviewOutcomes,
	13	reviewPatterns,
	14	gateRuns,
	15	gateMetrics,
	16	prComments,
	17	repositories,
	18	users,
	19	} from "../db/schema";
	20	import type { ReviewPattern } from "../db/schema";
	21	import { config } from "./config";
	22
	23	// ── Outcome Recording ───────────────────────────────────────────────────────
	24
	25	/**
	26	* Record how a developer responded to an AI review comment.
	27	* Called when a PR comment is resolved, dismissed, or the PR is merged.
	28	*/
	29	export async function recordReviewOutcome(opts: {
	30	repositoryId: string;
	31	pullRequestId: string;
	32	commentId: string;
	33	outcome: "accepted" \| "dismissed" \| "modified" \| "ignored";
	34	category: string;
	35	filePath?: string;
	36	language?: string;
	37	wasUseful?: boolean;
	38	}): Promise<void> {
	39	try {
	40	await db.insert(reviewOutcomes).values({
	41	repositoryId: opts.repositoryId,
	42	pullRequestId: opts.pullRequestId,
	43	commentId: opts.commentId,
	44	outcome: opts.outcome,
	45	category: opts.category,
	46	filePath: opts.filePath,
	47	language: opts.language ?? deriveLanguage(opts.filePath),
	48	wasUseful: opts.wasUseful,
	49	});
	50	} catch (err) {
	51	console.error("[flywheel] recordReviewOutcome failed:", err);
	52	}
	53	}
	54
	55	function deriveLanguage(filePath?: string): string \| null {
	56	if (!filePath) return null;
	57	const ext = filePath.split(".").pop()?.toLowerCase();
	58	const map: Record<string, string> = {
	59	ts: "typescript",
	60	tsx: "typescript",
	61	js: "javascript",
	62	jsx: "javascript",
	63	py: "python",
	64	rb: "ruby",
	65	go: "go",
	66	rs: "rust",
	67	java: "java",
	68	kt: "kotlin",
	69	cs: "csharp",
	70	cpp: "cpp",
	71	c: "c",
	72	swift: "swift",
	73	php: "php",
	74	sql: "sql",
	75	sh: "shell",
	76	yml: "yaml",
	77	yaml: "yaml",
	78	json: "json",
	79	toml: "toml",
	80	};
	81	return map[ext ?? ""] ?? ext ?? null;
	82	}
	83
	84	// ── Pattern Extraction ──────────────────────────────────────────────────────
	85
	86	/**
	87	* Extract patterns from recent review outcomes. Looks for:
	88	* - Categories that are consistently accepted (boost confidence)
	89	* - Categories that are consistently dismissed (reduce confidence / deactivate)
	90	* - New recurring patterns that should become rules
	91	*
	92	* Called periodically (e.g. after every N merges) or on demand.
	93	*/
	94	export async function extractPatterns(
	95	repositoryId?: string
	96	): Promise<{ created: number; updated: number; deactivated: number }> {
	97	const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
	98	let stats = { created: 0, updated: 0, deactivated: 0 };
	99
	100	try {
	101	const whereClause = repositoryId
	102	? and(
	103	eq(reviewOutcomes.repositoryId, repositoryId),
	104	gte(reviewOutcomes.createdAt, thirtyDaysAgo)
	105	)
	106	: gte(reviewOutcomes.createdAt, thirtyDaysAgo);
	107
	108	const outcomes = await db
	109	.select({
	110	category: reviewOutcomes.category,
	111	language: reviewOutcomes.language,
	112	outcome: reviewOutcomes.outcome,
	113	count: sql<number>`count(*)::int`,
	114	})
	115	.from(reviewOutcomes)
	116	.where(whereClause)
	117	.groupBy(
	118	reviewOutcomes.category,
	119	reviewOutcomes.language,
	120	reviewOutcomes.outcome
	121	);
	122
	123	// Group by category+language to compute acceptance rates
	124	const grouped = new Map<
	125	string,
	126	{ accepted: number; dismissed: number; total: number; language: string \| null; category: string }
	127	>();
	128
	129	for (const row of outcomes) {
	130	const key = `${row.category}:${row.language ?? "all"}`;
	131	const entry = grouped.get(key) ?? {
	132	accepted: 0,
	133	dismissed: 0,
	134	total: 0,
	135	language: row.language,
	136	category: row.category,
	137	};
	138	entry.total += row.count;
	139	if (row.outcome === "accepted" \|\| row.outcome === "modified") {
	140	entry.accepted += row.count;
	141	} else if (row.outcome === "dismissed") {
	142	entry.dismissed += row.count;
	143	}
	144	grouped.set(key, entry);
	145	}
	146
	147	for (const [, data] of grouped) {
	148	if (data.total < 3) continue; // need minimum evidence
	149
	150	const acceptRate = data.accepted / data.total;
	151	const confidence = Math.round(acceptRate * 100);
	152
	153	const scope = repositoryId ? "repo" : "global";
	154	const existing = await db
	155	.select()
	156	.from(reviewPatterns)
	157	.where(
	158	and(
	159	eq(reviewPatterns.scope, scope),
	160	eq(reviewPatterns.category, data.category),
	161	repositoryId
	162	? eq(reviewPatterns.repositoryId, repositoryId)
	163	: sql`${reviewPatterns.repositoryId} IS NULL`,
	164	data.language
	165	? eq(reviewPatterns.language, data.language)
	166	: sql`${reviewPatterns.language} IS NULL`
	167	)
	168	)
	169	.limit(1);
	170
	171	if (existing.length > 0) {
	172	const pattern = existing[0];
	173	if (confidence < 20 && data.total >= 5) {
	174	// Consistently dismissed — deactivate this pattern
	175	await db
	176	.update(reviewPatterns)
	177	.set({ active: false, confidence, evidenceCount: data.total, lastSeenAt: new Date() })
	178	.where(eq(reviewPatterns.id, pattern.id));
	179	stats.deactivated++;
	180	} else {
	181	await db
	182	.update(reviewPatterns)
	183	.set({ confidence, evidenceCount: data.total, lastSeenAt: new Date() })
	184	.where(eq(reviewPatterns.id, pattern.id));
	185	stats.updated++;
	186	}
	187	} else if (confidence >= 40) {
	188	// New pattern worth tracking
	189	const patternText = generatePatternDescription(data.category, data.language, acceptRate);
	190	await db.insert(reviewPatterns).values({
	191	repositoryId: repositoryId ?? null,
	192	scope,
	193	language: data.language,
	194	category: data.category,
	195	pattern: patternText,
	196	confidence,
	197	evidenceCount: data.total,
	198	});
	199	stats.created++;
	200	}
	201	}
	202	} catch (err) {
	203	console.error("[flywheel] extractPatterns failed:", err);
	204	}
	205
	206	return stats;
	207	}
	208
	209	function generatePatternDescription(
	210	category: string,
	211	language: string \| null,
	212	acceptRate: number
	213	): string {
	214	const langSuffix = language ? ` in ${language} files` : "";
	215	const emphasis = acceptRate > 0.8 ? "high priority" : "moderate priority";
	216
	217	const descriptions: Record<string, string> = {
	218	bug: `Bug detection${langSuffix} — ${emphasis}. Developers consistently fix flagged bugs.`,
	219	security: `Security vulnerability detection${langSuffix} — ${emphasis}. Flag injection, auth bypass, and data exposure.`,
	220	perf: `Performance issue detection${langSuffix} — ${emphasis}. Watch for N+1 queries, blocking I/O, unnecessary allocations.`,
	221	logic: `Logic error detection${langSuffix} — ${emphasis}. Check for off-by-one, null derefs, race conditions.`,
	222	breaking: `Breaking change detection${langSuffix} — ${emphasis}. Flag API contract violations and removed public interfaces.`,
	223	};
	224
	225	return descriptions[category] ?? `${category} detection${langSuffix} — ${emphasis}.`;
	226	}
	227
	228	// ── Gate Metrics Aggregation ────────────────────────────────────────────────
	229
	230	/**
	231	* Aggregate gate_runs into monthly metrics for trend analysis.
	232	* Called after gate checks complete.
	233	*/
	234	export async function updateGateMetrics(
	235	repositoryId: string,
	236	gateName: string,
	237	status: "passed" \| "failed" \| "skipped" \| "repaired",
	238	durationMs?: number
	239	): Promise<void> {
	240	const period = new Date().toISOString().slice(0, 7); // YYYY-MM
	241
	242	try {
	243	const [existing] = await db
	244	.select()
	245	.from(gateMetrics)
	246	.where(
	247	and(
	248	eq(gateMetrics.repositoryId, repositoryId),
	249	eq(gateMetrics.gateName, gateName),
	250	eq(gateMetrics.period, period)
	251	)
	252	)
	253	.limit(1);
	254
	255	if (existing) {
	256	const updates: Record<string, unknown> = {
	257	totalRuns: existing.totalRuns + 1,
	258	updatedAt: new Date(),
	259	};
	260
	261	if (status === "passed") updates.passed = existing.passed + 1;
	262	else if (status === "failed") updates.failed = existing.failed + 1;
	263	else if (status === "repaired") updates.repaired = existing.repaired + 1;
	264	else if (status === "skipped") updates.skipped = existing.skipped + 1;
	265
	266	if (durationMs != null && existing.avgDurationMs != null) {
	267	updates.avgDurationMs = Math.round(
	268	(existing.avgDurationMs * existing.totalRuns + durationMs) / (existing.totalRuns + 1)
	269	);
	270	} else if (durationMs != null) {
	271	updates.avgDurationMs = durationMs;
	272	}
	273
	274	await db
	275	.update(gateMetrics)
	276	.set(updates)
	277	.where(eq(gateMetrics.id, existing.id));
	278	} else {
	279	await db.insert(gateMetrics).values({
	280	repositoryId,
	281	gateName,
	282	period,
	283	totalRuns: 1,
	284	passed: status === "passed" ? 1 : 0,
	285	failed: status === "failed" ? 1 : 0,
	286	repaired: status === "repaired" ? 1 : 0,
	287	skipped: status === "skipped" ? 1 : 0,
	288	avgDurationMs: durationMs ?? null,
	289	});
	290	}
	291	} catch (err) {
	292	console.error("[flywheel] updateGateMetrics failed:", err);
	293	}
	294	}
	295
	296	// ── Context Injection (the actual "learning") ───────────────────────────────
	297
	298	/**
	299	* Build a context block for AI review prompts based on learned patterns.
	300	* This is what makes the flywheel turn — historical patterns inform future reviews.
	301	*/
	302	export async function buildReviewContext(
	303	repositoryId: string \| null,
	304	language?: string
	305	): Promise<string> {
	306	try {
	307	const patterns: ReviewPattern[] = [];
	308
	309	// Fetch repo-specific patterns
	310	if (repositoryId) {
	311	const repoPatterns = await db
	312	.select()
	313	.from(reviewPatterns)
	314	.where(
	315	and(
	316	eq(reviewPatterns.repositoryId, repositoryId),
	317	eq(reviewPatterns.active, true)
	318	)
	319	)
	320	.orderBy(desc(reviewPatterns.confidence))
	321	.limit(10);
	322	patterns.push(...repoPatterns);
	323	}
	324
	325	// Fetch global patterns
	326	const globalPatterns = await db
	327	.select()
	328	.from(reviewPatterns)
	329	.where(
	330	and(
	331	sql`${reviewPatterns.repositoryId} IS NULL`,
	332	eq(reviewPatterns.scope, "global"),
	333	eq(reviewPatterns.active, true)
	334	)
	335	)
	336	.orderBy(desc(reviewPatterns.confidence))
	337	.limit(10);
	338	patterns.push(...globalPatterns);
	339
	340	// Fetch language-specific patterns
	341	if (language) {
	342	const langPatterns = await db
	343	.select()
	344	.from(reviewPatterns)
	345	.where(
	346	and(
	347	eq(reviewPatterns.language, language),
	348	eq(reviewPatterns.active, true)
	349	)
	350	)
	351	.orderBy(desc(reviewPatterns.confidence))
	352	.limit(5);
	353	patterns.push(...langPatterns);
	354	}
	355
	356	if (patterns.length === 0) return "";
	357
	358	// Deduplicate by id
	359	const seen = new Set<string>();
	360	const unique = patterns.filter((p) => {
	361	if (seen.has(p.id)) return false;
	362	seen.add(p.id);
	363	return true;
	364	});
	365
	366	// Build context string
	367	const lines = unique
	368	.sort((a, b) => b.confidence - a.confidence)
	369	.slice(0, 15)
	370	.map(
	371	(p) =>
	372	`- [${p.category}] (confidence: ${p.confidence}%) ${p.pattern}`
	373	);
	374
	375	return `\n\nBased on historical review data for this codebase, pay extra attention to:\n${lines.join("\n")}`;
	376	} catch (err) {
	377	console.error("[flywheel] buildReviewContext failed:", err);
	378	return "";
	379	}
	380	}
	381
	382	// ── Repo-level Stats ────────────────────────────────────────────────────────
	383
	384	export interface FlywheelStats {
	385	totalReviews: number;
	386	acceptedRate: number;
	387	topCategories: Array<{ category: string; count: number; acceptRate: number }>;
	388	gateHealth: Array<{
	389	gateName: string;
	390	passRate: number;
	391	avgDurationMs: number;
	392	totalRuns: number;
	393	}>;
	394	activePatterns: number;
	395	}
	396
	397	/**
	398	* Get flywheel learning stats for a repository.
	399	* Useful for the settings/dashboard page.
	400	*/
	401	export async function getFlywheelStats(
	402	repositoryId: string
	403	): Promise<FlywheelStats> {
	404	try {
	405	const [outcomeStats, metrics, patternCount] = await Promise.all([
	406	db
	407	.select({
	408	category: reviewOutcomes.category,
	409	outcome: reviewOutcomes.outcome,
	410	count: sql<number>`count(*)::int`,
	411	})
	412	.from(reviewOutcomes)
	413	.where(eq(reviewOutcomes.repositoryId, repositoryId))
	414	.groupBy(reviewOutcomes.category, reviewOutcomes.outcome),
	415	db
	416	.select()
	417	.from(gateMetrics)
	418	.where(eq(gateMetrics.repositoryId, repositoryId))
	419	.orderBy(desc(gateMetrics.period))
	420	.limit(20),
	421	db
	422	.select({ count: sql<number>`count(*)::int` })
	423	.from(reviewPatterns)
	424	.where(
	425	and(
	426	eq(reviewPatterns.repositoryId, repositoryId),
	427	eq(reviewPatterns.active, true)
	428	)
	429	),
	430	]);
	431
	432	// Aggregate outcome stats
	433	let totalReviews = 0;
	434	let totalAccepted = 0;
	435	const catMap = new Map<string, { count: number; accepted: number }>();
	436
	437	for (const row of outcomeStats) {
	438	totalReviews += row.count;
	439	if (row.outcome === "accepted" \|\| row.outcome === "modified") {
	440	totalAccepted += row.count;
	441	}
	442	const cat = catMap.get(row.category) ?? { count: 0, accepted: 0 };
	443	cat.count += row.count;
	444	if (row.outcome === "accepted" \|\| row.outcome === "modified") {
	445	cat.accepted += row.count;
	446	}
	447	catMap.set(row.category, cat);
	448	}
	449
	450	const topCategories = Array.from(catMap.entries())
	451	.map(([category, data]) => ({
	452	category,
	453	count: data.count,
	454	acceptRate: data.count > 0 ? data.accepted / data.count : 0,
	455	}))
	456	.sort((a, b) => b.count - a.count)
	457	.slice(0, 5);
	458
	459	// Aggregate gate health (most recent period per gate)
	460	const latestGates = new Map<string, (typeof metrics)[0]>();
	461	for (const m of metrics) {
	462	if (!latestGates.has(m.gateName)) latestGates.set(m.gateName, m);
	463	}
	464
	465	const gateHealth = Array.from(latestGates.values()).map((m) => ({
	466	gateName: m.gateName,
	467	passRate: m.totalRuns > 0 ? (m.passed + m.repaired) / m.totalRuns : 1,
	468	avgDurationMs: m.avgDurationMs ?? 0,
	469	totalRuns: m.totalRuns,
	470	}));
	471
	472	return {
	473	totalReviews,
	474	acceptedRate: totalReviews > 0 ? totalAccepted / totalReviews : 0,
	475	topCategories,
	476	gateHealth,
	477	activePatterns: patternCount[0]?.count ?? 0,
	478	};
	479	} catch (err) {
	480	console.error("[flywheel] getFlywheelStats failed:", err);
	481	return {
	482	totalReviews: 0,
	483	acceptedRate: 0,
	484	topCategories: [],
	485	gateHealth: [],
	486	activePatterns: 0,
	487	};
	488	}
	489	}