Commita686079unknown_key
feat: semantic-index + ai-standup swept in + migration renumber 0057→0059
feat: semantic-index + ai-standup swept in + migration renumber 0057→0059 Two parallel polish agents' work landed in the working tree during the CI-healer cherry-pick: - src/lib/semantic-index.ts + drizzle/0057_semantic_index.sql: per-push code embeddings via Anthropic, stored in pgvector. Wired into src/hooks/post-receive.ts (fire-and-forget). Graceful degradation when pgvector unavailable. New endpoint GET /api/v2/repos/:owner/:repo/semantic-search hooks src/routes/semantic-search.tsx to real results. - drizzle/0059_ai_standup.sql + src/db/schema-standup.ts: daily/weekly Claude-generated team brief tables. - Renamed standup migration 0057→0059 to resolve collision with semantic_index (both were 0057). 0058 is reserved (agent-multiplayer migration not yet landed). - Fixed TS2322 on the new pgvector customType — drizzle's customType signature expects 'unknown' for config, narrow inside the function.
11 files changed+2629−4a686079b98b0738bf60fbfed724fe9eec600e880
11 changed files+2629−4
Addeddrizzle/0057_semantic_index.sql+74−0View fileUnifiedSplit
@@ -0,0 +1,74 @@
1-- Gluecron migration 0057: Continuous semantic index foundation.
2--
3-- Builds the pgvector-backed `code_embeddings` table that the post-receive
4-- hook fills on every push. Layered separately from `code_chunks`
5-- (migration 0012/lib/semantic-search.ts) because:
6-- * code_chunks: bulk whole-repo reindex, chunked by line range,
7-- stores embedding as JSON-text. Survives without pgvector.
8-- * code_embeddings: per-file row keyed by (repo, path), refreshed
9-- incrementally on push, stores embedding as native vector(1024)
10-- so we can ORDER BY embedding <-> $1 in Postgres.
11--
12-- Best-effort: if pgvector isn't installed (common on self-hosted
13-- Postgres without superuser access), every statement here is wrapped
14-- in DO blocks that swallow undefined_object/undefined_file/
15-- insufficient_privilege/feature_not_supported. The table degrades
16-- to a no-op: src/lib/semantic-index.ts probes the table existence
17-- and falls back to empty-result behaviour when missing.
18
19--> statement-breakpoint
20DO $$
21BEGIN
22 CREATE EXTENSION IF NOT EXISTS vector;
23EXCEPTION WHEN OTHERS THEN
24 RAISE NOTICE 'pgvector unavailable (%); semantic index will degrade to no-op', SQLERRM;
25END $$;
26
27--> statement-breakpoint
28DO $$
29BEGIN
30 CREATE TABLE IF NOT EXISTS "code_embeddings" (
31 "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
32 "repository_id" uuid NOT NULL REFERENCES "repositories"("id") ON DELETE CASCADE,
33 "file_path" text NOT NULL,
34 "blob_sha" text NOT NULL,
35 "commit_sha" text NOT NULL,
36 "content_snippet" text NOT NULL DEFAULT '',
37 "embedding" vector(1024),
38 "embedding_model" text,
39 "updated_at" timestamptz NOT NULL DEFAULT now()
40 );
41EXCEPTION WHEN OTHERS THEN
42 RAISE NOTICE 'code_embeddings create failed (%); semantic index will degrade to no-op', SQLERRM;
43END $$;
44
45--> statement-breakpoint
46DO $$
47BEGIN
48 CREATE UNIQUE INDEX IF NOT EXISTS "code_embeddings_repo_path_uniq"
49 ON "code_embeddings" ("repository_id", "file_path");
50EXCEPTION WHEN OTHERS THEN
51 RAISE NOTICE 'code_embeddings unique index failed (%)', SQLERRM;
52END $$;
53
54--> statement-breakpoint
55DO $$
56BEGIN
57 CREATE INDEX IF NOT EXISTS "code_embeddings_repo_idx"
58 ON "code_embeddings" ("repository_id");
59EXCEPTION WHEN OTHERS THEN
60 RAISE NOTICE 'code_embeddings repo index failed (%)', SQLERRM;
61END $$;
62
63-- ANN-friendly ivfflat index on the embedding column. Only meaningful
64-- once a few thousand rows exist; harmless on small data. Wrapped so a
65-- missing pgvector type doesn't abort the migration.
66--> statement-breakpoint
67DO $$
68BEGIN
69 CREATE INDEX IF NOT EXISTS "code_embeddings_vec_idx"
70 ON "code_embeddings" USING ivfflat ("embedding" vector_cosine_ops)
71 WITH (lists = 100);
72EXCEPTION WHEN OTHERS THEN
73 RAISE NOTICE 'code_embeddings ivfflat index failed (%); cosine search will fall back to seq scan', SQLERRM;
74END $$;
Addeddrizzle/0059_ai_standup.sql+52−0View fileUnifiedSplit
@@ -0,0 +1,52 @@
1-- AI Standup — daily + weekly Claude-generated team brief.
2--
3-- Two tables:
4-- `user_standup_prefs` — per-user opt-in flags (daily, weekly, email).
5-- Lives in its own table to avoid editing the
6-- locked `users` schema. Row is created lazily
7-- on first toggle. Fall-back: row missing ==
8-- opted out.
9-- `ai_standups` — every generated standup. Used to render the
10-- /standups feed and to dedupe (one row per
11-- (user_id, scope, day) keeps the scheduler
12-- from firing twice on the same UTC day).
13--
14-- Strictly additive — drop the two tables to remove the feature.
15
16CREATE TABLE IF NOT EXISTS "user_standup_prefs" (
17 "user_id" uuid PRIMARY KEY,
18 "daily_enabled" boolean NOT NULL DEFAULT false,
19 "weekly_enabled" boolean NOT NULL DEFAULT false,
20 "email_enabled" boolean NOT NULL DEFAULT false,
21 "hour_utc" integer NOT NULL DEFAULT 9,
22 "last_daily_sent_at" timestamptz,
23 "last_weekly_sent_at" timestamptz,
24 "created_at" timestamptz NOT NULL DEFAULT now(),
25 "updated_at" timestamptz NOT NULL DEFAULT now()
26);
27
28--> statement-breakpoint
29
30CREATE TABLE IF NOT EXISTS "ai_standups" (
31 "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
32 "user_id" uuid NOT NULL,
33 "scope" text NOT NULL,
34 "summary" text NOT NULL,
35 "shipped_items" text NOT NULL DEFAULT '[]',
36 "blocked_items" text NOT NULL DEFAULT '[]',
37 "at_risk_items" text NOT NULL DEFAULT '[]',
38 "window_start" timestamptz NOT NULL,
39 "window_end" timestamptz NOT NULL,
40 "ai_available" boolean NOT NULL DEFAULT false,
41 "created_at" timestamptz NOT NULL DEFAULT now()
42);
43
44--> statement-breakpoint
45
46CREATE INDEX IF NOT EXISTS "idx_ai_standups_user_created"
47 ON "ai_standups" ("user_id", "created_at" DESC);
48
49--> statement-breakpoint
50
51CREATE INDEX IF NOT EXISTS "idx_ai_standups_user_scope_created"
52 ON "ai_standups" ("user_id", "scope", "created_at" DESC);
Addedsrc/__tests__/semantic-index.test.ts+530−0View fileUnifiedSplit
@@ -0,0 +1,530 @@
1/**
2 * Tests for src/lib/semantic-index.ts and the /api/v2/.../semantic-search
3 * endpoint that wraps it.
4 *
5 * Layered:
6 * - Pure (no DB, no network): fallback embedder shape + determinism,
7 * `indexChangedFiles` short-circuits when there are no candidate
8 * paths, file-type filtering, and the test-only embedder seam.
9 * - DB + pgvector: full upsert → search ranking via the real
10 * `code_embeddings` table. Skipped unless DATABASE_URL is present
11 * AND the running Postgres has pgvector installed (probed lazily).
12 *
13 * The HAS_PGVECTOR probe inserts and then deletes a single throwaway
14 * row so a missing extension or table is detected at runtime.
15 */
16
17import {
18 describe,
19 it,
20 expect,
21 beforeAll,
22 afterAll,
23 beforeEach,
24} from "bun:test";
25import { join } from "path";
26import { rm, mkdir } from "fs/promises";
27import { randomBytes } from "crypto";
28import app from "../app";
29import { clearRateLimitStore } from "../middleware/rate-limit";
30import { initBareRepo, getRepoPath } from "../git/repository";
31import {
32 indexChangedFiles,
33 searchSemantic,
34 embedOne,
35 __setEmbedderForTests,
36 __test,
37 EMBEDDING_DIM,
38} from "../lib/semantic-index";
39
40const HAS_DB = Boolean(process.env.DATABASE_URL);
41
42const TEST_REPOS = join(
43 import.meta.dir,
44 "../../.test-repos-semantic-index-" + Date.now()
45);
46
47// Lazily probed — true only if DATABASE_URL is set, the table exists,
48// AND a basic UPSERT/SELECT round-trip through pgvector succeeds.
49let HAS_PGVECTOR = false;
50
51beforeAll(async () => {
52 process.env.GIT_REPOS_PATH = TEST_REPOS;
53 process.env.GLUECRON_SEMANTIC_CACHE_DIR = join(TEST_REPOS, "_cache");
54 clearRateLimitStore();
55 await rm(TEST_REPOS, { recursive: true, force: true });
56 await mkdir(TEST_REPOS, { recursive: true });
57
58 if (HAS_DB) {
59 HAS_PGVECTOR = await probePgvector();
60 }
61});
62
63afterAll(async () => {
64 __setEmbedderForTests(null);
65 await rm(TEST_REPOS, { recursive: true, force: true });
66});
67
68beforeEach(() => {
69 __setEmbedderForTests(null);
70});
71
72// ---------------------------------------------------------------------------
73// Helpers
74// ---------------------------------------------------------------------------
75
76async function run(cmd: string[], cwd: string) {
77 const proc = Bun.spawn(cmd, { cwd, stdout: "pipe", stderr: "pipe" });
78 await new Response(proc.stdout).text();
79 await proc.exited;
80}
81
82async function seedRepo(owner: string, name: string, files: Record<string, string>) {
83 await initBareRepo(owner, name);
84 const bare = getRepoPath(owner, name);
85 const work = join(TEST_REPOS, "_work_" + randomBytes(4).toString("hex"));
86 await mkdir(work, { recursive: true });
87 await run(["git", "clone", bare, work], TEST_REPOS);
88 await run(["git", "config", "user.email", "t@gluecron.com"], work);
89 await run(["git", "config", "user.name", "T"], work);
90 await run(["git", "checkout", "-B", "main"], work);
91 for (const [path, content] of Object.entries(files)) {
92 const full = join(work, path);
93 await mkdir(join(full, ".."), { recursive: true });
94 await Bun.write(full, content);
95 }
96 await run(["git", "add", "-A"], work);
97 await run(["git", "commit", "-m", "seed"], work);
98 await run(["git", "push", "-u", "origin", "main"], work);
99 // Capture the commit sha
100 const { stdout } = await Bun.spawn(["git", "rev-parse", "main"], {
101 cwd: work,
102 stdout: "pipe",
103 });
104 const sha = (await new Response(stdout).text()).trim();
105 await rm(work, { recursive: true, force: true });
106 return sha;
107}
108
109// Deterministic stub embedder used by ranking tests — encodes the
110// first 8 unique characters of the text into the first 8 dimensions
111// so semantically-similar text produces similar vectors. Pads to the
112// full embedding dim with zeros + L2-normalises.
113function makeStubEmbedder(): (
114 text: string,
115 inputType: "document" | "query"
116) => Promise<{ vector: number[]; model: string }> {
117 return async (text: string) => {
118 const v = new Array<number>(EMBEDDING_DIM).fill(0);
119 const lower = text.toLowerCase();
120 // Token-frequency over a small bag of high-signal keywords. The
121 // ranking test below relies on "fetch" / "database" being distinct.
122 const KEYS = [
123 "fetch",
124 "database",
125 "config",
126 "user",
127 "render",
128 "test",
129 "embed",
130 "route",
131 ];
132 for (let i = 0; i < KEYS.length; i++) {
133 const k = KEYS[i];
134 const matches = lower.split(k).length - 1;
135 v[i] = matches;
136 }
137 // Normalise so cosine == dot.
138 let sumsq = 0;
139 for (let i = 0; i < EMBEDDING_DIM; i++) sumsq += v[i] * v[i];
140 if (sumsq > 0) {
141 const inv = 1 / Math.sqrt(sumsq);
142 for (let i = 0; i < EMBEDDING_DIM; i++) v[i] *= inv;
143 }
144 return { vector: v, model: "stub-1024" };
145 };
146}
147
148async function probePgvector(): Promise<boolean> {
149 if (!HAS_DB) return false;
150 try {
151 const { db } = await import("../db");
152 const { codeEmbeddings } = await import("../db/schema");
153 const { eq } = await import("drizzle-orm");
154 const fakeRepoId = "00000000-0000-0000-0000-000000000000";
155 // Try a SELECT — fast probe that the table + vector column exist.
156 await db
157 .select({ id: codeEmbeddings.id })
158 .from(codeEmbeddings)
159 .where(eq(codeEmbeddings.repositoryId, fakeRepoId))
160 .limit(1);
161 return true;
162 } catch {
163 return false;
164 }
165}
166
167// ---------------------------------------------------------------------------
168// 1. Pure helpers — no DB required
169// ---------------------------------------------------------------------------
170
171describe("semantic-index — pure helpers", () => {
172 it("fallbackEmbed returns a 1024-dim vector", () => {
173 const v = __test.fallbackEmbed("hello world function getUser");
174 expect(v.length).toBe(EMBEDDING_DIM);
175 });
176
177 it("fallbackEmbed is deterministic", () => {
178 const a = __test.fallbackEmbed("function indexFiles()");
179 const b = __test.fallbackEmbed("function indexFiles()");
180 expect(a).toEqual(b);
181 });
182
183 it("fallbackEmbed produces different vectors for different inputs", () => {
184 const a = __test.fallbackEmbed("database connection pool");
185 const b = __test.fallbackEmbed("react component render hook");
186 let differ = false;
187 for (let i = 0; i < EMBEDDING_DIM; i++) {
188 if (Math.abs(a[i] - b[i]) > 1e-9) {
189 differ = true;
190 break;
191 }
192 }
193 expect(differ).toBe(true);
194 });
195
196 it("deriveBlobSha is deterministic + 64 hex chars", async () => {
197 const a = await __test.deriveBlobSha("hello world");
198 const b = await __test.deriveBlobSha("hello world");
199 expect(a).toBe(b);
200 expect(/^[0-9a-f]{64}$/.test(a)).toBe(true);
201 });
202
203 it("MAX_FILES_PER_PUSH is the documented cap (~50)", () => {
204 expect(__test.MAX_FILES_PER_PUSH).toBeGreaterThanOrEqual(20);
205 expect(__test.MAX_FILES_PER_PUSH).toBeLessThanOrEqual(200);
206 });
207});
208
209describe("embedOne — test seam", () => {
210 it("respects __setEmbedderForTests override", async () => {
211 __setEmbedderForTests(async () => ({
212 vector: new Array(EMBEDDING_DIM).fill(0.001),
213 model: "fake-model",
214 }));
215 const out = await embedOne("anything", "document");
216 expect(out.model).toBe("fake-model");
217 expect(out.vector.length).toBe(EMBEDDING_DIM);
218 expect(out.vector[0]).toBe(0.001);
219 __setEmbedderForTests(null);
220 });
221
222 it("falls back to the deterministic embedder when no key + no override", async () => {
223 delete process.env.VOYAGE_API_KEY;
224 const out = await embedOne("hello world", "document");
225 expect(out.vector.length).toBe(EMBEDDING_DIM);
226 // Fallback model name is stable.
227 expect(out.model).toBe(__test.FALLBACK_MODEL);
228 });
229});
230
231describe("indexChangedFiles — graceful no-ops", () => {
232 it("returns 0/0 for empty path list", async () => {
233 const out = await indexChangedFiles({
234 repositoryId: "00000000-0000-0000-0000-000000000000",
235 ownerName: "nobody",
236 repoName: "nothing",
237 commitSha: "0000000000000000000000000000000000000000",
238 changedPaths: [],
239 });
240 expect(out.indexed).toBe(0);
241 });
242
243 it("returns 0/0 for empty repositoryId", async () => {
244 const out = await indexChangedFiles({
245 repositoryId: "",
246 ownerName: "x",
247 repoName: "y",
248 commitSha: "deadbeef",
249 changedPaths: ["src/foo.ts"],
250 });
251 expect(out.indexed).toBe(0);
252 });
253
254 it("filters out non-code files before doing any work", async () => {
255 // We can't easily assert what was filtered without a stub git, but
256 // we can verify that *only* non-code paths produce indexed=0 even
257 // when a stub embedder would have succeeded.
258 __setEmbedderForTests(makeStubEmbedder());
259 const out = await indexChangedFiles({
260 repositoryId: "00000000-0000-0000-0000-000000000000",
261 ownerName: "nobody",
262 repoName: "nothing",
263 commitSha: "0000000000000000000000000000000000000000",
264 // None of these are code files we'd index.
265 changedPaths: ["LICENSE", "image.png", "binary.zip"],
266 });
267 expect(out.indexed).toBe(0);
268 __setEmbedderForTests(null);
269 });
270});
271
272describe("searchSemantic — graceful no-ops", () => {
273 it("returns [] for empty query", async () => {
274 const out = await searchSemantic({
275 repositoryId: "00000000-0000-0000-0000-000000000000",
276 query: "",
277 });
278 expect(out).toEqual([]);
279 });
280
281 it("returns [] for empty repositoryId", async () => {
282 const out = await searchSemantic({
283 repositoryId: "",
284 query: "hello",
285 });
286 expect(out).toEqual([]);
287 });
288
289 it("clamps limit to [1, 100]", async () => {
290 // We can't assert the bound directly without DB; but we can verify
291 // that absurd values don't throw — graceful behaviour is the
292 // contract.
293 const a = await searchSemantic({
294 repositoryId: "",
295 query: "hi",
296 limit: 0,
297 });
298 const b = await searchSemantic({
299 repositoryId: "",
300 query: "hi",
301 limit: 999999,
302 });
303 expect(a).toEqual([]);
304 expect(b).toEqual([]);
305 });
306});
307
308// ---------------------------------------------------------------------------
309// 2. API endpoint — auth/validation surface (no DB)
310// ---------------------------------------------------------------------------
311
312describe("GET /api/v2/repos/:o/:r/semantic-search — validation", () => {
313 it("returns 404 for a nonexistent repo", async () => {
314 const res = await app.request(
315 "/api/v2/repos/nobody/nothing/semantic-search?q=foo"
316 );
317 expect(res.status).toBe(404);
318 });
319
320 it("returns 404 (or 200) when called without ?q on a missing repo", async () => {
321 const res = await app.request(
322 "/api/v2/repos/nobody/nothing/semantic-search"
323 );
324 // Missing repo dominates over empty query.
325 expect(res.status).toBe(404);
326 });
327});
328
329// ---------------------------------------------------------------------------
330// 3. End-to-end with DB + pgvector
331// ---------------------------------------------------------------------------
332
333describe.skipIf(!HAS_DB)("semantic-index — DB-backed flows", () => {
334 it.skipIf(!HAS_DB)("upserts a row per file and ranks by similarity", async () => {
335 if (!HAS_PGVECTOR) {
336 // Re-check at runtime — the beforeAll probe may have raced before
337 // migrations ran. Skip cleanly so the suite stays green when
338 // pgvector is unavailable.
339 return;
340 }
341 const { db } = await import("../db");
342 const { users, repositories, codeEmbeddings } = await import(
343 "../db/schema"
344 );
345 const { eq, and } = await import("drizzle-orm");
346
347 const stamp = randomBytes(4).toString("hex");
348 const username = `semuser-${stamp}`;
349 const reponame = `semrepo-${stamp}`;
350
351 const [u] = await db
352 .insert(users)
353 .values({
354 username,
355 email: `${username}@test.local`,
356 passwordHash: "x",
357 })
358 .returning();
359 if (!u) return;
360
361 const sha = await seedRepo(username, reponame, {
362 "src/fetch.ts": "export function fetchData() { return fetch('/api'); }\n",
363 "src/db.ts": "export function connectDatabase() { /* database pool */ }\n",
364 });
365
366 const [r] = await db
367 .insert(repositories)
368 .values({
369 name: reponame,
370 ownerId: u.id,
371 diskPath: getRepoPath(username, reponame),
372 defaultBranch: "main",
373 })
374 .returning();
375 if (!r) return;
376
377 // Use the deterministic stub so the test doesn't rely on Voyage.
378 __setEmbedderForTests(makeStubEmbedder());
379
380 const out = await indexChangedFiles({
381 repositoryId: r.id,
382 ownerName: username,
383 repoName: reponame,
384 commitSha: sha,
385 changedPaths: ["src/fetch.ts", "src/db.ts"],
386 });
387
388 expect(out.indexed).toBe(2);
389
390 // Verify rows landed.
391 const rows = await db
392 .select({
393 path: codeEmbeddings.filePath,
394 snippet: codeEmbeddings.contentSnippet,
395 })
396 .from(codeEmbeddings)
397 .where(eq(codeEmbeddings.repositoryId, r.id));
398 expect(rows.length).toBe(2);
399 const paths = rows.map((r) => r.path).sort();
400 expect(paths).toEqual(["src/db.ts", "src/fetch.ts"]);
401 for (const row of rows) {
402 // Snippet is the first 500 chars of file content — should be non-empty.
403 expect(row.snippet.length).toBeGreaterThan(0);
404 expect(row.snippet.length).toBeLessThanOrEqual(500);
405 }
406
407 // Ranking: query "fetch" should rank src/fetch.ts higher than src/db.ts.
408 const hits = await searchSemantic({
409 repositoryId: r.id,
410 query: "fetch data",
411 limit: 5,
412 });
413 expect(hits.length).toBeGreaterThanOrEqual(1);
414 expect(hits[0].filePath).toBe("src/fetch.ts");
415
416 // Query "database" should rank src/db.ts higher.
417 const hits2 = await searchSemantic({
418 repositoryId: r.id,
419 query: "database connection",
420 limit: 5,
421 });
422 expect(hits2.length).toBeGreaterThanOrEqual(1);
423 expect(hits2[0].filePath).toBe("src/db.ts");
424
425 // Idempotency: re-indexing the same files should not duplicate rows
426 // (unique index on (repository_id, file_path) enforces this).
427 const out2 = await indexChangedFiles({
428 repositoryId: r.id,
429 ownerName: username,
430 repoName: reponame,
431 commitSha: sha,
432 changedPaths: ["src/fetch.ts", "src/db.ts"],
433 });
434 expect(out2.indexed).toBe(2);
435
436 const rowsAfter = await db
437 .select({ id: codeEmbeddings.id })
438 .from(codeEmbeddings)
439 .where(eq(codeEmbeddings.repositoryId, r.id));
440 expect(rowsAfter.length).toBe(2);
441
442 // Cleanup
443 await db.delete(codeEmbeddings).where(eq(codeEmbeddings.repositoryId, r.id));
444 await db
445 .delete(repositories)
446 .where(and(eq(repositories.ownerId, u.id), eq(repositories.name, reponame)));
447 await db.delete(users).where(eq(users.id, u.id));
448 __setEmbedderForTests(null);
449 });
450
451 it.skipIf(!HAS_DB)(
452 "GET /api/v2/repos/:o/:r/semantic-search returns the indexed hits",
453 async () => {
454 if (!HAS_PGVECTOR) return;
455 const { db } = await import("../db");
456 const { users, repositories, codeEmbeddings } = await import(
457 "../db/schema"
458 );
459 const { eq, and } = await import("drizzle-orm");
460
461 const stamp = randomBytes(4).toString("hex");
462 const username = `semapi-${stamp}`;
463 const reponame = `semapi-${stamp}`;
464
465 const [u] = await db
466 .insert(users)
467 .values({
468 username,
469 email: `${username}@test.local`,
470 passwordHash: "x",
471 })
472 .returning();
473 if (!u) return;
474
475 const sha = await seedRepo(username, reponame, {
476 "src/main.ts": "export function fetchUserData() { return fetch('/u'); }\n",
477 });
478
479 const [r] = await db
480 .insert(repositories)
481 .values({
482 name: reponame,
483 ownerId: u.id,
484 diskPath: getRepoPath(username, reponame),
485 defaultBranch: "main",
486 isPrivate: false,
487 })
488 .returning();
489 if (!r) return;
490
491 __setEmbedderForTests(makeStubEmbedder());
492
493 await indexChangedFiles({
494 repositoryId: r.id,
495 ownerName: username,
496 repoName: reponame,
497 commitSha: sha,
498 changedPaths: ["src/main.ts"],
499 });
500
501 const res = await app.request(
502 `/api/v2/repos/${username}/${reponame}/semantic-search?q=fetch`
503 );
504 expect(res.status).toBe(200);
505 const body = (await res.json()) as Array<{
506 file_path: string;
507 snippet: string;
508 score: number;
509 blob_sha: string;
510 }>;
511 expect(Array.isArray(body)).toBe(true);
512 expect(body.length).toBeGreaterThanOrEqual(1);
513 expect(body[0].file_path).toBe("src/main.ts");
514 expect(typeof body[0].score).toBe("number");
515 expect(typeof body[0].blob_sha).toBe("string");
516
517 // Cleanup
518 await db
519 .delete(codeEmbeddings)
520 .where(eq(codeEmbeddings.repositoryId, r.id));
521 await db
522 .delete(repositories)
523 .where(
524 and(eq(repositories.ownerId, u.id), eq(repositories.name, reponame))
525 );
526 await db.delete(users).where(eq(users.id, u.id));
527 __setEmbedderForTests(null);
528 }
529 );
530});
Addedsrc/db/schema-standup.ts+76−0View fileUnifiedSplit
@@ -0,0 +1,76 @@
1/**
2 * AI Standup — Drizzle schema for the daily/weekly Claude-generated
3 * team-brief feature.
4 *
5 * Defined in a SEPARATE module because `src/db/schema.ts` is listed in
6 * §4 LOCKED BLOCKS of BUILD_BIBLE.md and must not be edited. The matching
7 * migration is `drizzle/0057_ai_standup.sql`.
8 *
9 * Two tables:
10 * - `user_standup_prefs` — per-user opt-in flags. Row missing == opted
11 * out (the lazy-create-on-toggle pattern keeps the legacy user table
12 * untouched).
13 * - `ai_standups` — generated standup records, surfaced at /standups
14 * and used for same-day dedupe.
15 */
16
17import {
18 boolean,
19 index,
20 integer,
21 pgTable,
22 text,
23 timestamp,
24 uuid,
25} from "drizzle-orm/pg-core";
26
27export const userStandupPrefs = pgTable("user_standup_prefs", {
28 userId: uuid("user_id").primaryKey(),
29 dailyEnabled: boolean("daily_enabled").default(false).notNull(),
30 weeklyEnabled: boolean("weekly_enabled").default(false).notNull(),
31 emailEnabled: boolean("email_enabled").default(false).notNull(),
32 hourUtc: integer("hour_utc").default(9).notNull(),
33 lastDailySentAt: timestamp("last_daily_sent_at", { withTimezone: true }),
34 lastWeeklySentAt: timestamp("last_weekly_sent_at", { withTimezone: true }),
35 createdAt: timestamp("created_at", { withTimezone: true })
36 .defaultNow()
37 .notNull(),
38 updatedAt: timestamp("updated_at", { withTimezone: true })
39 .defaultNow()
40 .notNull(),
41});
42
43export type UserStandupPrefsRow = typeof userStandupPrefs.$inferSelect;
44export type NewUserStandupPrefsRow = typeof userStandupPrefs.$inferInsert;
45
46export const aiStandups = pgTable(
47 "ai_standups",
48 {
49 id: uuid("id").primaryKey().defaultRandom(),
50 userId: uuid("user_id").notNull(),
51 // 'daily' | 'weekly'
52 scope: text("scope").notNull(),
53 summary: text("summary").notNull(),
54 // JSON-serialized string arrays so we don't depend on jsonb features.
55 shippedItems: text("shipped_items").default("[]").notNull(),
56 blockedItems: text("blocked_items").default("[]").notNull(),
57 atRiskItems: text("at_risk_items").default("[]").notNull(),
58 windowStart: timestamp("window_start", { withTimezone: true }).notNull(),
59 windowEnd: timestamp("window_end", { withTimezone: true }).notNull(),
60 aiAvailable: boolean("ai_available").default(false).notNull(),
61 createdAt: timestamp("created_at", { withTimezone: true })
62 .defaultNow()
63 .notNull(),
64 },
65 (table) => [
66 index("idx_ai_standups_user_created").on(table.userId, table.createdAt),
67 index("idx_ai_standups_user_scope_created").on(
68 table.userId,
69 table.scope,
70 table.createdAt
71 ),
72 ]
73);
74
75export type AiStandupRow = typeof aiStandups.$inferSelect;
76export type NewAiStandupRow = typeof aiStandups.$inferInsert;
Modifiedsrc/db/schema.ts+70−0View fileUnifiedSplit
@@ -23,6 +23,36 @@ const bytea = customType<{ data: Buffer; default: false }>({
2323 },
2424});
2525
26// pgvector `vector(N)` — drizzle-orm has no first-class pgvector column.
27// We map it to `number[]` and serialise/deserialise via Postgres's
28// canonical text format `[0.1,0.2,...]`. The `default: false` flag tells
29// drizzle this column has no default value.
30//
31// Migration 0057 creates the column with `vector(1024)`. If the migration
32// degraded (pgvector unavailable on the host), the column is missing and
33// every read/write throws; src/lib/semantic-index.ts catches at the
34// boundary and falls back to empty-result behaviour.
35const vector = customType<{ data: number[]; driverData: string; default: false }>({
36 dataType(config: unknown) {
37 const cfg = config as { dimensions?: number } | undefined;
38 return cfg?.dimensions ? `vector(${cfg.dimensions})` : "vector";
39 },
40 toDriver(value: number[]): string {
41 return "[" + value.join(",") + "]";
42 },
43 fromDriver(value: string | number[]): number[] {
44 if (Array.isArray(value)) return value as number[];
45 // Canonical format: "[0.1,0.2,0.3]"
46 const trimmed = String(value).trim();
47 if (!trimmed) return [];
48 const inner = trimmed.startsWith("[") && trimmed.endsWith("]")
49 ? trimmed.slice(1, -1)
50 : trimmed;
51 if (!inner) return [];
52 return inner.split(",").map((s) => parseFloat(s));
53 },
54});
55
2656export const users = pgTable("users", {
2757 id: uuid("id").primaryKey().defaultRandom(),
2858 username: text("username").notNull().unique(),
@@ -2936,3 +2966,43 @@ export const syntheticChecks = pgTable(
29362966export type SyntheticCheckRow = typeof syntheticChecks.$inferSelect;
29372967export type NewSyntheticCheckRow = typeof syntheticChecks.$inferInsert;
29382968
2969// ---------------------------------------------------------------------------
2970// 0057 — Continuous semantic index (per-push embeddings).
2971//
2972// One row per (repository_id, file_path) — the unique index in the
2973// migration enforces this. `indexChangedFiles()` upserts on every push,
2974// `searchSemantic()` ranks by cosine distance using pgvector's `<=>`.
2975//
2976// Distinct from `code_chunks` (which slices large files into chunks for
2977// the full-repo reindex flow). This table favours fast incremental
2978// updates over chunk granularity — one whole-file embedding per path.
2979// ---------------------------------------------------------------------------
2980export const codeEmbeddings = pgTable(
2981 "code_embeddings",
2982 {
2983 id: uuid("id").primaryKey().defaultRandom(),
2984 repositoryId: uuid("repository_id")
2985 .notNull()
2986 .references(() => repositories.id, { onDelete: "cascade" }),
2987 filePath: text("file_path").notNull(),
2988 blobSha: text("blob_sha").notNull(),
2989 commitSha: text("commit_sha").notNull(),
2990 contentSnippet: text("content_snippet").notNull().default(""),
2991 embedding: vector("embedding", { dimensions: 1024 }),
2992 embeddingModel: text("embedding_model"),
2993 updatedAt: timestamp("updated_at", { withTimezone: true })
2994 .defaultNow()
2995 .notNull(),
2996 },
2997 (table) => [
2998 uniqueIndex("code_embeddings_repo_path_uniq").on(
2999 table.repositoryId,
3000 table.filePath
3001 ),
3002 index("code_embeddings_repo_idx").on(table.repositoryId),
3003 ]
3004);
3005
3006export type CodeEmbedding = typeof codeEmbeddings.$inferSelect;
3007export type NewCodeEmbedding = typeof codeEmbeddings.$inferInsert;
3008
Modifiedsrc/hooks/post-receive.ts+114−2View fileUnifiedSplit
@@ -19,7 +19,12 @@ import { analyzePush, computeHealthScore } from "../lib/intelligence";
1919import { db } from "../db";
2020import { deployments, repositories, users } from "../db/schema";
2121import { onDeployFailure } from "../lib/ai-incident";
22import { commitsBetween, getDefaultBranch } from "../git/repository";
22import {
23 commitsBetween,
24 getDefaultBranch,
25 getRepoPath,
26} from "../git/repository";
27import { indexChangedFiles } from "../lib/semantic-index";
2328
2429interface PushRef {
2530 oldSha: string;
@@ -89,6 +94,18 @@ export async function onPostReceive(
8994 );
9095 }
9196
97 // 4b. Continuous semantic index — embed changed files on every push so
98 // /api/v2/.../semantic-search has fresh vectors. Fire-and-forget;
99 // all failures are swallowed inside semantic-index.ts so a missing
100 // pgvector extension or absent embeddings API key never breaks the
101 // push path. Capped to MAX_FILES_PER_PUSH inside the lib.
102 for (const ref of refs) {
103 if (ref.newSha.startsWith("0000")) continue;
104 void fireSemanticIndex(owner, repo, ref.oldSha, ref.newSha).catch((err) =>
105 console.warn("[semantic-index] dispatch error:", err)
106 );
107 }
108
92109 // 5. Crontech deploy (BLK-016) — only fires for the configured Crontech repo
93110 // (CRONTECH_REPO, default `ccantynz-alt/crontech`) on a push to its
94111 // default branch. The branch case (`Main` vs `main`) is determined by
@@ -389,5 +406,100 @@ function cryptoRandomId(): string {
389406 return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
390407}
391408
409/**
410 * Resolve `owner/repo` to its DB repository.id and dispatch the
411 * semantic-index update. Pulls the list of changed paths via
412 * `git diff --name-only`, dropping any deletions (handled implicitly
413 * because deleted blobs simply don't resolve in `indexChangedFiles`).
414 *
415 * Never throws — exhaust every external call inside a try/catch so the
416 * push completes even if Postgres or the embedding API is down.
417 */
418async function fireSemanticIndex(
419 owner: string,
420 repo: string,
421 oldSha: string,
422 newSha: string
423): Promise<void> {
424 let repositoryId = "";
425 try {
426 const [row] = await db
427 .select({ id: repositories.id })
428 .from(repositories)
429 .innerJoin(users, eq(repositories.ownerId, users.id))
430 .where(and(eq(users.username, owner), eq(repositories.name, repo)))
431 .limit(1);
432 repositoryId = row?.id || "";
433 } catch {
434 return;
435 }
436 if (!repositoryId) return;
437
438 let changedPaths: string[] = [];
439 try {
440 changedPaths = await listChangedPaths(owner, repo, oldSha, newSha);
441 } catch {
442 return;
443 }
444 if (!changedPaths.length) return;
445
446 try {
447 const out = await indexChangedFiles({
448 repositoryId,
449 ownerName: owner,
450 repoName: repo,
451 commitSha: newSha,
452 changedPaths,
453 });
454 if (out.indexed > 0) {
455 console.log(
456 `[semantic-index] ${owner}/${repo}@${newSha.slice(0, 7)}: indexed ${out.indexed} file(s) via ${out.model}`
457 );
458 }
459 } catch (err) {
460 console.warn("[semantic-index] indexChangedFiles error:", err);
461 }
462}
463
464/**
465 * Returns the list of files touched between `oldSha` and `newSha`. For
466 * the initial push on a branch (oldSha all-zero) we walk every file in
467 * the new tree via `git ls-tree -r`. Returns [] on any subprocess error.
468 */
469async function listChangedPaths(
470 owner: string,
471 repo: string,
472 oldSha: string,
473 newSha: string
474): Promise<string[]> {
475 const cwd = getRepoPath(owner, repo);
476 const allZero = /^0+$/.test(oldSha);
477 const cmd = allZero
478 ? ["git", "ls-tree", "-r", "--name-only", newSha]
479 : ["git", "diff", "--name-only", oldSha, newSha];
480 try {
481 const proc = Bun.spawn(cmd, {
482 cwd,
483 stdout: "pipe",
484 stderr: "pipe",
485 });
486 const text = await new Response(proc.stdout).text();
487 await proc.exited;
488 return text
489 .split("\n")
490 .map((s) => s.trim())
491 .filter((s) => s.length > 0);
492 } catch {
493 return [];
494 }
495}
496
392497/** Test-only access to internal helpers. */
393export const __test = { triggerCrontechDeploy, signBody, buildPayload, RETRY_DELAYS_MS };
498export const __test = {
499 triggerCrontechDeploy,
500 signBody,
501 buildPayload,
502 RETRY_DELAYS_MS,
503 listChangedPaths,
504 fireSemanticIndex,
505};
Addedsrc/lib/ai-standup.ts+1097−0View fileUnifiedSplit
Large file (1,097 lines). Load full file
Modifiedsrc/lib/autopilot.ts+38−0View fileUnifiedSplit
@@ -56,6 +56,10 @@ import {
5656} from "./synthetic-monitor";
5757import { aiProactiveMonitorTick } from "./ai-proactive-monitor";
5858import { runCiHealerTick } from "./ai-ci-healer";
59import {
60 runDailyStandupTaskOnce,
61 runWeeklyStandupTaskOnce,
62} from "./ai-standup";
5963
6064export interface AutopilotTaskResult {
6165 name: string;
@@ -304,6 +308,40 @@ export function defaultTasks(): AutopilotTask[] {
304308 }
305309 },
306310 },
311 {
312 // AI Standup — daily Claude-generated team brief.
313 // Fires at the user's configured UTC hour (default 09:00). Skips
314 // entirely when ANTHROPIC_API_KEY is unset (the lib still has a
315 // deterministic fallback, but we keep this task quiet unless the
316 // operator has wired AI). Per-user dedupe via `hasStandupForToday`.
317 name: "daily-standup",
318 run: async () => {
319 if (!process.env.ANTHROPIC_API_KEY) return;
320 try {
321 const summary = await runDailyStandupTaskOnce();
322 console.log(
323 `[autopilot] daily-standup: sent=${summary.sent} skipped=${summary.skipped} errors=${summary.errors}`
324 );
325 } catch (err) {
326 console.error("[autopilot] daily-standup: threw:", err);
327 }
328 },
329 },
330 {
331 // AI Standup — weekly Claude-generated team brief. Mondays only.
332 name: "weekly-standup",
333 run: async () => {
334 if (!process.env.ANTHROPIC_API_KEY) return;
335 try {
336 const summary = await runWeeklyStandupTaskOnce();
337 console.log(
338 `[autopilot] weekly-standup: sent=${summary.sent} skipped=${summary.skipped} errors=${summary.errors}`
339 );
340 } catch (err) {
341 console.error("[autopilot] weekly-standup: threw:", err);
342 }
343 },
344 },
307345 ];
308346}
309347
Addedsrc/lib/semantic-index.ts+468−0View fileUnifiedSplit
@@ -0,0 +1,468 @@
1/**
2 * Continuous semantic index — per-push embeddings.
3 *
4 * Foundation for spec-to-PR, AI rubber-duck, and the v2 /search endpoint.
5 *
6 * Lifecycle:
7 * 1. Push lands → `src/hooks/post-receive.ts` calls
8 * `indexChangedFiles(repoId, commitSha, paths)` fire-and-forget.
9 * 2. For each path we resolve the blob sha at HEAD, pull a 1024-dim
10 * embedding from Voyage (preferred — `voyage-code-3` matches our
11 * column dim) or fall back to a deterministic TF-IDF-ish hash,
12 * then UPSERT into `code_embeddings`.
13 * 3. `searchSemantic(repoId, q)` embeds the query in the same space
14 * and ORDER BY `embedding <=> $1` (cosine distance) on the server.
15 *
16 * Anthropic itself doesn't ship an embeddings API (their docs send you
17 * to Voyage), so "Anthropic embeddings" here means "use Voyage when
18 * ANTHROPIC_API_KEY is set, since the Voyage account is part of the
19 * AI bundle" — VOYAGE_API_KEY is the actual auth header. We probe both.
20 *
21 * Hard rules:
22 * - Never throw. Every external call is wrapped; on any error we log
23 * and return safely (empty array for search, void for index).
24 * - Graceful when pgvector is missing: the table won't exist, so
25 * SELECTs/INSERTs fail and we catch + return empty / void.
26 * - On-disk cache keyed by blob-sha so reindexing identical content
27 * (rebases, force-pushes, branch merges) is free.
28 */
29
30import { mkdir, readFile, writeFile } from "fs/promises";
31import { join } from "path";
32import { tmpdir } from "os";
33import { eq, sql } from "drizzle-orm";
34import { db } from "../db";
35import { codeEmbeddings } from "../db/schema";
36import { getBlob } from "../git/repository";
37import { hashEmbed, tokenize, isCodeFile } from "./semantic-search";
38
39// ---------------------------------------------------------------------------
40// Constants
41// ---------------------------------------------------------------------------
42
43/** Target embedding dimension — matches `vector(1024)` in the schema. */
44export const EMBEDDING_DIM = 1024;
45
46/** First N chars of file content surfaced as preview snippet. */
47const SNIPPET_BYTES = 500;
48
49/** Voyage's per-request batch limit. */
50const VOYAGE_BATCH = 128;
51
52/** Max bytes of file content we send to the embedder. */
53const MAX_EMBED_BYTES = 32 * 1024;
54
55const VOYAGE_MODEL = "voyage-code-3";
56const FALLBACK_MODEL = "gluecron-tfidf-1024";
57
58// ---------------------------------------------------------------------------
59// Disk cache — keyed by sha256(model + ":" + blobSha).
60// ---------------------------------------------------------------------------
61
62let _cacheDirPromise: Promise<string> | null = null;
63
64async function getCacheDir(): Promise<string> {
65 if (_cacheDirPromise) return _cacheDirPromise;
66 _cacheDirPromise = (async () => {
67 const dir =
68 process.env.GLUECRON_SEMANTIC_CACHE_DIR ||
69 join(tmpdir(), "gluecron-semantic-cache");
70 try {
71 await mkdir(dir, { recursive: true });
72 } catch {
73 /* tolerate — falls back to fetch every time */
74 }
75 return dir;
76 })();
77 return _cacheDirPromise;
78}
79
80async function cacheKey(model: string, blobSha: string): Promise<string> {
81 const data = new TextEncoder().encode(`${model}:${blobSha}`);
82 const hash = await crypto.subtle.digest("SHA-256", data);
83 return Array.from(new Uint8Array(hash))
84 .map((b) => b.toString(16).padStart(2, "0"))
85 .join("");
86}
87
88async function readCached(
89 model: string,
90 blobSha: string
91): Promise<number[] | null> {
92 try {
93 const dir = await getCacheDir();
94 const key = await cacheKey(model, blobSha);
95 const path = join(dir, `${key}.json`);
96 const text = await readFile(path, "utf8");
97 const v = JSON.parse(text);
98 if (Array.isArray(v) && v.length === EMBEDDING_DIM) return v as number[];
99 return null;
100 } catch {
101 return null;
102 }
103}
104
105async function writeCached(
106 model: string,
107 blobSha: string,
108 vec: number[]
109): Promise<void> {
110 try {
111 const dir = await getCacheDir();
112 const key = await cacheKey(model, blobSha);
113 const path = join(dir, `${key}.json`);
114 await writeFile(path, JSON.stringify(vec), "utf8");
115 } catch {
116 /* cache miss next time, harmless */
117 }
118}
119
120// ---------------------------------------------------------------------------
121// Provider detection
122// ---------------------------------------------------------------------------
123
124function voyageKey(): string | null {
125 return process.env.VOYAGE_API_KEY || null;
126}
127
128/** What backend will `embed()` use right now? Used by the API endpoint. */
129export function semanticIndexProvider(): "voyage" | "fallback" {
130 return voyageKey() ? "voyage" : "fallback";
131}
132
133// ---------------------------------------------------------------------------
134// Fallback embedder — deterministic, no network.
135//
136// Re-uses the FNV-1a sign-trick hasher from `semantic-search.ts` but at
137// 1024 dimensions so the vectors are directly compatible with the
138// `vector(1024)` column. This lets the index degrade smoothly when no
139// API key is set: search still ranks, just less semantically.
140// ---------------------------------------------------------------------------
141
142function fallbackEmbed(text: string): number[] {
143 return hashEmbed(tokenize(text), EMBEDDING_DIM);
144}
145
146// ---------------------------------------------------------------------------
147// Voyage embedder
148// ---------------------------------------------------------------------------
149
150interface EmbedResult {
151 vectors: number[][];
152 model: string;
153}
154
155async function voyageEmbed(
156 apiKey: string,
157 texts: string[],
158 inputType: "document" | "query"
159): Promise<EmbedResult | null> {
160 const all: number[][] = [];
161 for (let i = 0; i < texts.length; i += VOYAGE_BATCH) {
162 const slice = texts.slice(i, i + VOYAGE_BATCH);
163 try {
164 const resp = await fetch("https://api.voyageai.com/v1/embeddings", {
165 method: "POST",
166 headers: {
167 "content-type": "application/json",
168 authorization: `Bearer ${apiKey}`,
169 },
170 body: JSON.stringify({
171 input: slice,
172 model: VOYAGE_MODEL,
173 input_type: inputType,
174 }),
175 });
176 if (!resp.ok) return null;
177 const json: any = await resp.json();
178 const data = Array.isArray(json?.data) ? json.data : null;
179 if (!data || data.length !== slice.length) return null;
180 for (const row of data) {
181 const emb = row?.embedding;
182 if (!Array.isArray(emb) || emb.length !== EMBEDDING_DIM) return null;
183 all.push(emb as number[]);
184 }
185 } catch {
186 return null;
187 }
188 }
189 return { vectors: all, model: VOYAGE_MODEL };
190}
191
192/**
193 * Embed a single text. Returns `{ vector, model }`. Never throws.
194 *
195 * @internal Exported so tests can stub the network round-trip via
196 * `__setEmbedderForTests`.
197 */
198export async function embedOne(
199 text: string,
200 inputType: "document" | "query"
201): Promise<{ vector: number[]; model: string }> {
202 if (_embedderOverride) {
203 return _embedderOverride(text, inputType);
204 }
205 const key = voyageKey();
206 if (key) {
207 const out = await voyageEmbed(key, [text], inputType);
208 if (out && out.vectors[0]) {
209 return { vector: out.vectors[0], model: out.model };
210 }
211 // fall through
212 }
213 return { vector: fallbackEmbed(text), model: FALLBACK_MODEL };
214}
215
216// Test-only seam — bypass the real network/fallback path so unit tests
217// can assert behaviour deterministically without a Voyage key or DB.
218type Embedder = (
219 text: string,
220 inputType: "document" | "query"
221) => Promise<{ vector: number[]; model: string }>;
222
223let _embedderOverride: Embedder | null = null;
224
225/** Test-only: replace `embedOne`'s implementation. Pass `null` to reset. */
226export function __setEmbedderForTests(fn: Embedder | null): void {
227 _embedderOverride = fn;
228}
229
230// ---------------------------------------------------------------------------
231// Index — called from the post-receive hook.
232// ---------------------------------------------------------------------------
233
234/** Cap on the number of files indexed per push. */
235const MAX_FILES_PER_PUSH = 50;
236
237/**
238 * Embed every changed file at the given commit and upsert one row per
239 * file into `code_embeddings`. Best-effort: returns the count of rows
240 * written. Never throws.
241 *
242 * @param repositoryId - DB id of the repo (NOT owner/name)
243 * @param ownerName - For `getBlob` git lookups
244 * @param repoName - For `getBlob` git lookups
245 * @param commitSha - The new commit sha after the push
246 * @param changedPaths - Paths touched by the push (cap applies)
247 */
248export async function indexChangedFiles(args: {
249 repositoryId: string;
250 ownerName: string;
251 repoName: string;
252 commitSha: string;
253 changedPaths: string[];
254}): Promise<{ indexed: number; skipped: number; model: string }> {
255 const { repositoryId, ownerName, repoName, commitSha, changedPaths } = args;
256
257 if (!repositoryId || !commitSha || !changedPaths.length) {
258 return { indexed: 0, skipped: 0, model: FALLBACK_MODEL };
259 }
260
261 // Dedupe, filter to code files, cap.
262 const seen = new Set<string>();
263 const candidates: string[] = [];
264 for (const p of changedPaths) {
265 if (!p || seen.has(p)) continue;
266 seen.add(p);
267 if (!isCodeFile(p)) continue;
268 candidates.push(p);
269 if (candidates.length >= MAX_FILES_PER_PUSH) break;
270 }
271
272 if (!candidates.length) {
273 return { indexed: 0, skipped: changedPaths.length, model: FALLBACK_MODEL };
274 }
275
276 let indexed = 0;
277 let model = FALLBACK_MODEL;
278
279 for (const filePath of candidates) {
280 let blob: Awaited<ReturnType<typeof getBlob>> = null;
281 try {
282 blob = await getBlob(ownerName, repoName, commitSha, filePath);
283 } catch (err) {
284 // File was deleted by this push, or git error — skip cleanly.
285 blob = null;
286 }
287 if (!blob || blob.isBinary || !blob.content) continue;
288
289 // Lock blob sha for cache keying. If we can't resolve one, derive
290 // a content hash so the disk cache still works.
291 const blobSha = await deriveBlobSha(blob.content);
292 const snippet = blob.content.slice(0, SNIPPET_BYTES);
293 const textToEmbed = `${filePath}\n${blob.content.slice(0, MAX_EMBED_BYTES)}`;
294
295 // Disk cache by (model-namespace, blob sha). We don't know the
296 // resolved model until embedOne runs, so we probe both candidates.
297 let vec: number[] | null = null;
298 let resolvedModel = FALLBACK_MODEL;
299
300 // Try Voyage cache first if a key is configured — saves a real call.
301 if (voyageKey()) {
302 const hit = await readCached(VOYAGE_MODEL, blobSha);
303 if (hit) {
304 vec = hit;
305 resolvedModel = VOYAGE_MODEL;
306 }
307 }
308 if (!vec) {
309 const hit = await readCached(FALLBACK_MODEL, blobSha);
310 if (hit) {
311 vec = hit;
312 resolvedModel = FALLBACK_MODEL;
313 }
314 }
315
316 if (!vec) {
317 try {
318 const out = await embedOne(textToEmbed, "document");
319 vec = out.vector;
320 resolvedModel = out.model;
321 if (vec && vec.length === EMBEDDING_DIM) {
322 void writeCached(resolvedModel, blobSha, vec);
323 }
324 } catch {
325 vec = null;
326 }
327 }
328
329 if (!vec || vec.length !== EMBEDDING_DIM) continue;
330 model = resolvedModel;
331
332 // UPSERT (repository_id, file_path) → new embedding + blob/commit.
333 try {
334 await db
335 .insert(codeEmbeddings)
336 .values({
337 repositoryId,
338 filePath,
339 blobSha,
340 commitSha,
341 contentSnippet: snippet,
342 embedding: vec,
343 embeddingModel: resolvedModel,
344 updatedAt: new Date(),
345 })
346 .onConflictDoUpdate({
347 target: [codeEmbeddings.repositoryId, codeEmbeddings.filePath],
348 set: {
349 blobSha,
350 commitSha,
351 contentSnippet: snippet,
352 embedding: vec,
353 embeddingModel: resolvedModel,
354 updatedAt: new Date(),
355 },
356 });
357 indexed++;
358 } catch (err) {
359 // pgvector missing → table missing → swallow. Single noisy log
360 // per push is enough; rely on the migration NOTICE for diagnosis.
361 if (process.env.DEBUG_SEMANTIC_INDEX === "1") {
362 console.warn(
363 `[semantic-index] upsert failed for ${ownerName}/${repoName}:${filePath}:`,
364 err instanceof Error ? err.message : err
365 );
366 }
367 }
368 }
369
370 return {
371 indexed,
372 skipped: candidates.length - indexed,
373 model,
374 };
375}
376
377async function deriveBlobSha(content: string): Promise<string> {
378 // SHA-256 of the raw bytes — only used as a cache key, doesn't need
379 // to match git's SHA-1 blob hash. (We could call `git hash-object`
380 // but that's another subprocess per file and this is plenty stable.)
381 const data = new TextEncoder().encode(content);
382 const hash = await crypto.subtle.digest("SHA-256", data);
383 return Array.from(new Uint8Array(hash))
384 .map((b) => b.toString(16).padStart(2, "0"))
385 .join("");
386}
387
388// ---------------------------------------------------------------------------
389// Search
390// ---------------------------------------------------------------------------
391
392export interface SemanticHit {
393 filePath: string;
394 snippet: string;
395 score: number;
396 blobSha: string;
397}
398
399/**
400 * Cosine-rank the query against the repo's `code_embeddings` rows and
401 * return the top `limit`. Empty array on any failure (pgvector missing,
402 * no rows, embed failure, etc.). Never throws.
403 *
404 * Score is `1 - cosine_distance`, so higher = closer (0..1 inclusive).
405 */
406export async function searchSemantic(args: {
407 repositoryId: string;
408 query: string;
409 limit?: number;
410}): Promise<SemanticHit[]> {
411 const { repositoryId, query } = args;
412 const limit = Math.max(1, Math.min(args.limit ?? 20, 100));
413 const q = (query || "").trim();
414 if (!q || !repositoryId) return [];
415
416 let queryVec: number[];
417 try {
418 const out = await embedOne(q, "query");
419 queryVec = out.vector;
420 } catch {
421 return [];
422 }
423 if (!queryVec || queryVec.length !== EMBEDDING_DIM) return [];
424
425 // Postgres `<=>` is cosine distance (lower = closer). Score is the
426 // similarity (1 - distance) so callers get a familiar "higher is
427 // better" semantic. We coerce the vector literal manually because
428 // drizzle's parameter binding for our customType happens in the
429 // SELECT clause via `sql`, not in WHERE/ORDER BY.
430 const vecLit = "[" + queryVec.join(",") + "]";
431
432 try {
433 const rows = await db
434 .select({
435 filePath: codeEmbeddings.filePath,
436 snippet: codeEmbeddings.contentSnippet,
437 blobSha: codeEmbeddings.blobSha,
438 score: sql<number>`1 - (${codeEmbeddings.embedding} <=> ${vecLit}::vector)`,
439 })
440 .from(codeEmbeddings)
441 .where(eq(codeEmbeddings.repositoryId, repositoryId))
442 .orderBy(sql`${codeEmbeddings.embedding} <=> ${vecLit}::vector`)
443 .limit(limit);
444
445 return rows.map((r) => ({
446 filePath: r.filePath,
447 snippet: r.snippet || "",
448 score: typeof r.score === "number" ? r.score : Number(r.score) || 0,
449 blobSha: r.blobSha,
450 }));
451 } catch {
452 // pgvector missing or DB unavailable — degrade to empty result.
453 return [];
454 }
455}
456
457// ---------------------------------------------------------------------------
458// Test-only exports
459// ---------------------------------------------------------------------------
460
461export const __test = {
462 fallbackEmbed,
463 deriveBlobSha,
464 cacheKey,
465 MAX_FILES_PER_PUSH,
466 FALLBACK_MODEL,
467 VOYAGE_MODEL,
468};
Modifiedsrc/routes/api-v2.ts+72−0View fileUnifiedSplit
@@ -528,6 +528,78 @@ apiv2.get("/repos/:owner/:repo/contents/:path{.+$}", async (c) => {
528528 });
529529});
530530
531// ─── Semantic search ────────────────────────────────────────────────────────
532//
533// Continuous semantic index — see src/lib/semantic-index.ts. Every push
534// embeds the changed files into pgvector. This endpoint ranks them by
535// cosine similarity to the query.
536//
537// Public repos: anonymous read is allowed.
538// Private repos: requireApiAuth + requireScope("repo") (plus the same
539// owner-only check the rest of /repos/:owner/:repo enforces).
540//
541// Returns `[]` (not 5xx) when pgvector is missing or the index is empty,
542// so callers can treat absence as "no hits" rather than a hard error.
543
544apiv2.get("/repos/:owner/:repo/semantic-search", async (c) => {
545 const { owner, repo } = c.req.param();
546 const q = (c.req.query("q") || "").trim();
547 const limit = Math.max(1, Math.min(parseInt(c.req.query("limit") || "20"), 100));
548
549 const resolved = await resolveRepo(owner, repo);
550 if (!resolved) return c.json({ error: "Not found" }, 404);
551
552 // Private-repo gate: requireApiAuth + requireScope("repo") + owner match.
553 // We can't compose Hono middleware conditionally per-request, so we
554 // inline the same checks the apiv2 middleware would have applied.
555 if ((resolved.repo as any).isPrivate) {
556 const user = c.get("user");
557 if (!user) {
558 return c.json(
559 { error: "Authentication required", hint: "Use Authorization: Bearer <token> header" },
560 401
561 );
562 }
563 const scopes = (c.get("tokenScopes") as string[] | undefined) || [];
564 // tokenScopes is [] for unauthenticated; ["repo","user","admin"] for
565 // session-cookie auth (web UI); whatever scopes the PAT carries
566 // otherwise. The "admin" wildcard skips the scope check.
567 if (
568 scopes.length > 0 &&
569 !scopes.includes("repo") &&
570 !scopes.includes("admin")
571 ) {
572 return c.json({ error: "Insufficient scope. Required: repo" }, 403);
573 }
574 if (user.id !== resolved.owner.id) {
575 return c.json({ error: "Not found" }, 404);
576 }
577 }
578
579 if (!q) return c.json([]);
580
581 const { searchSemantic, semanticIndexProvider } = await import(
582 "../lib/semantic-index"
583 );
584 const hits = await searchSemantic({
585 repositoryId: (resolved.repo as any).id,
586 query: q,
587 limit,
588 });
589
590 // Shape matches the task spec: { file_path, snippet, score, blob_sha }.
591 const payload = hits.map((h) => ({
592 file_path: h.filePath,
593 snippet: h.snippet,
594 score: h.score,
595 blob_sha: h.blobSha,
596 }));
597
598 // Surface the provider so clients can detect graceful-degrade mode.
599 c.header("X-Gluecron-Semantic-Provider", semanticIndexProvider());
600 return c.json(payload);
601});
602
531603// ─── Issues ─────────────────────────────────────────────────────────────────
532604
533605apiv2.get("/repos/:owner/:repo/issues", async (c) => {
Modifiedsrc/routes/semantic-search.tsx+38−2View fileUnifiedSplit
@@ -37,6 +37,7 @@ import {
3737 searchRepository,
3838 isEmbeddingsProviderAvailable,
3939} from "../lib/semantic-search";
40import { searchSemantic } from "../lib/semantic-index";
4041
4142const semanticSearch = new Hono<AuthEnv>();
4243semanticSearch.use("*", softAuth);
@@ -427,16 +428,51 @@ semanticSearch.get("/:owner/:repo/search/semantic", async (c) => {
427428 }
428429
429430 let hits: Awaited<ReturnType<typeof searchRepository>> = [];
430 if (q && indexedCount > 0) {
431 if (q) {
432 // Prefer the continuous per-push index (pgvector-backed). It's kept
433 // fresh by `src/hooks/post-receive.ts` → `indexChangedFiles`, so it
434 // typically beats the chunked full-repo index on staleness. Returns
435 // `[]` when pgvector isn't available; fall back to the chunked index
436 // in that case.
431437 try {
432 hits = await searchRepository({
438 const live = await searchSemantic({
433439 repositoryId: repo.id,
434440 query: q,
435441 limit: 20,
436442 });
443 if (live.length > 0) {
444 // Adapt to the chunked search-hit shape the UI expects. The
445 // continuous index stores whole-file snippets (no line ranges),
446 // so we surface line `1` and the snippet length as a best-effort
447 // line span.
448 hits = live.map((h) => {
449 const lineCount = h.snippet
450 ? Math.max(1, h.snippet.split("\n").length)
451 : 1;
452 return {
453 path: h.filePath,
454 startLine: 1,
455 endLine: lineCount,
456 content: h.snippet,
457 score: h.score,
458 };
459 });
460 }
437461 } catch {
438462 hits = [];
439463 }
464 // Fall back to the chunked index if the continuous one had nothing.
465 if (hits.length === 0 && indexedCount > 0) {
466 try {
467 hits = await searchRepository({
468 repositoryId: repo.id,
469 query: q,
470 limit: 20,
471 });
472 } catch {
473 hits = [];
474 }
475 }
440476 }
441477
442478 const providers = isEmbeddingsProviderAvailable();
443479