Commit91d8392unknown_key
Merge pull request #77 from ccantynz-alt/claude/review-readme-docs-ulqPK
Merge pull request #77 from ccantynz-alt/claude/review-readme-docs-ulqPK Claude/review readme docs ulq pk
3 files changed+494−191d8392e3f7a526c294a954d23f6cf33d87141f9
3 changed files+494−1
Modified.env.example+8−0View fileUnifiedSplit
@@ -8,6 +8,14 @@ GATETEST_API_KEY=
88# Generate a secret with: openssl rand -hex 32
99GATETEST_CALLBACK_SECRET=
1010GATETEST_HMAC_SECRET=
11# ─── For the GateTest side, NOT Gluecron ─────────────────────────────────
12# When you point GateTest at gluecron.com to scan the live site, GateTest
13# needs the two values below in ITS environment (set them in the gatetest.ai
14# scanner config, not here). Issue the token from /admin/ops → "GateTest
15# scanner credentials" while signed in as site admin.
16# GLUECRON_BASE_URL=https://gluecron.com
17# GLUECRON_API_TOKEN=<paste from /admin/ops, shown once>
18# ─────────────────────────────────────────────────────────────────────────
1119CRONTECH_DEPLOY_URL=https://crontech.ai/api/webhooks/gluecron-push
1220# BLK-016 — only fire the Crontech deploy webhook for pushes to this
1321# `<owner>/<name>` (default `ccantynz-alt/crontech`). All other repo pushes
Addedscripts/site-crawl.ts+378−0View fileUnifiedSplit
@@ -0,0 +1,378 @@
1
2/**
3 * One-shot site crawler.
4 *
5 * Hits every known public + admin route and reports any URL that:
6 * - returned 4xx or 5xx
7 * - took > SLOW_MS to respond
8 * - bounced an admin route to /login (auth not seen)
9 *
10 * Public routes are crawled anonymously. Admin/settings/billing routes
11 * are only crawled if you pass GLUECRON_SESSION=<sid cookie value>.
12 *
13 * Usage (from the box, or anywhere with internet):
14 *
15 * GLUECRON_HOST=https://gluecron.com \
16 * GLUECRON_SESSION=$(psql "$DATABASE_URL" -tAc \
17 * "select token from sessions \
18 * where user_id=(select id from users where username='admin') \
19 * and expires_at > now() and requires_2fa = false \
20 * order by created_at desc limit 1" | tr -d ' ') \
21 * bun scripts/site-crawl.ts
22 *
23 * If you skip GLUECRON_SESSION the script still runs — it will just mark
24 * every admin route as "auth-required (skipped)".
25 *
26 * Output: a single markdown table sorted FAIL → SLOW → OK, plus a
27 * one-line summary at the bottom you can paste back.
28 */
29
30const HOST = (process.env.GLUECRON_HOST || "https://gluecron.com").replace(
31 /\/$/,
32 ""
33);
34const SESSION = process.env.GLUECRON_SESSION || "";
35const CONCURRENCY = Number(process.env.CRAWL_CONCURRENCY || 6);
36const TIMEOUT_MS = Number(process.env.CRAWL_TIMEOUT_MS || 8000);
37const SLOW_MS = Number(process.env.CRAWL_SLOW_MS || 2000);
38
39// ─── Route inventory ─────────────────────────────────────────────────
40//
41// Curated subset of the 307 routes: every static page, plus parameterised
42// pages instantiated with `ccantynz/Gluecron.com` (the canonical self-
43// hosted repo) and `admin` (the bootstrap operator). Anything that needs
44// real session state (issue numbers, gist slugs, etc.) is skipped.
45//
46// `auth` = "anon" public; "admin" needs site-admin cookie; "user" needs
47// any logged-in cookie. The crawler treats both admin+user as "skip if
48// no session cookie is set".
49
50type AuthLevel = "anon" | "user" | "admin";
51type Route = { path: string; auth: AuthLevel; method?: "GET" | "POST" };
52
53const ROUTES: Route[] = [
54 // Public landing + marketing
55 { path: "/", auth: "anon" },
56 { path: "/about", auth: "anon" },
57 { path: "/features", auth: "anon" },
58 { path: "/pricing", auth: "anon" },
59 { path: "/explore", auth: "anon" },
60 { path: "/demo", auth: "anon" },
61 { path: "/help", auth: "anon" },
62 { path: "/getting-started", auth: "anon" },
63 { path: "/install", auth: "anon" },
64 { path: "/vs-github", auth: "anon" },
65 { path: "/marketplace", auth: "anon" },
66 { path: "/shortcuts", auth: "anon" },
67 { path: "/sleep-mode", auth: "anon" },
68 { path: "/setup", auth: "anon" },
69
70 // Legal
71 { path: "/terms", auth: "anon" },
72 { path: "/privacy", auth: "anon" },
73 { path: "/acceptable-use", auth: "anon" },
74 { path: "/legal/terms", auth: "anon" },
75 { path: "/legal/privacy", auth: "anon" },
76 { path: "/legal/acceptable-use", auth: "anon" },
77 { path: "/legal/dmca", auth: "anon" },
78
79 // Auth flows (anon GET should render the form)
80 { path: "/login", auth: "anon" },
81 { path: "/register", auth: "anon" },
82 { path: "/forgot-password", auth: "anon" },
83 { path: "/reset-password", auth: "anon" },
84 { path: "/login/magic", auth: "anon" },
85 { path: "/verify-email", auth: "anon" },
86 { path: "/play", auth: "anon" },
87
88 // Health + version + manifests
89 { path: "/health", auth: "anon" },
90 { path: "/healthz", auth: "anon" },
91 { path: "/readyz", auth: "anon" },
92 { path: "/version", auth: "anon" },
93 { path: "/api/version", auth: "anon" },
94 { path: "/api/docs", auth: "anon" },
95 { path: "/status", auth: "anon" },
96 { path: "/status.svg", auth: "anon" },
97 { path: "/robots.txt", auth: "anon" },
98 { path: "/sitemap.xml", auth: "anon" },
99 { path: "/manifest.webmanifest", auth: "anon" },
100 { path: "/icon.svg", auth: "anon" },
101 { path: "/sw.js", auth: "anon" },
102 { path: "/sw-push.js", auth: "anon" },
103 { path: "/pwa/vapid-public-key", auth: "anon" },
104 { path: "/gluecron.dxt", auth: "anon" },
105
106 // Public repo browsing (canonical self-host repo)
107 { path: "/ccantynz", auth: "anon" },
108 { path: "/ccantynz/Gluecron.com", auth: "anon" },
109 { path: "/ccantynz/Gluecron.com/commits", auth: "anon" },
110 { path: "/ccantynz/Gluecron.com/branches", auth: "anon" },
111 { path: "/ccantynz/Gluecron.com/contributors", auth: "anon" },
112 { path: "/ccantynz/Gluecron.com/issues", auth: "anon" },
113 { path: "/ccantynz/Gluecron.com/pulls", auth: "anon" },
114 { path: "/ccantynz/Gluecron.com/releases", auth: "anon" },
115 { path: "/ccantynz/Gluecron.com/wiki", auth: "anon" },
116 { path: "/ccantynz/Gluecron.com/discussions", auth: "anon" },
117 { path: "/ccantynz/Gluecron.com/security", auth: "anon" },
118 { path: "/ccantynz/Gluecron.com/security/advisories", auth: "anon" },
119 { path: "/ccantynz/Gluecron.com/actions", auth: "anon" },
120 { path: "/ccantynz/Gluecron.com/deployments", auth: "anon" },
121 { path: "/ccantynz/Gluecron.com/dependencies", auth: "anon" },
122 { path: "/ccantynz/Gluecron.com/contributors", auth: "anon" },
123 { path: "/ccantynz/Gluecron.com/traffic", auth: "anon" },
124
125 // Authed user pages
126 { path: "/dashboard", auth: "user" },
127 { path: "/feed", auth: "user" },
128 { path: "/notifications", auth: "user" },
129 { path: "/me/ai-savings", auth: "user" },
130 { path: "/new", auth: "user" },
131 { path: "/import", auth: "user" },
132 { path: "/gists", auth: "user" },
133 { path: "/gists/new", auth: "user" },
134 { path: "/todos", auth: "user" },
135 { path: "/ask", auth: "user" },
136 { path: "/billing/manage", auth: "user" },
137
138 // User settings
139 { path: "/settings", auth: "user" },
140 { path: "/settings/profile", auth: "user" },
141 { path: "/settings/notifications", auth: "user" },
142 { path: "/settings/billing", auth: "user" },
143 { path: "/settings/keys", auth: "user" },
144 { path: "/settings/tokens", auth: "user" },
145 { path: "/settings/passkeys", auth: "user" },
146 { path: "/settings/2fa", auth: "user" },
147 { path: "/settings/applications", auth: "user" },
148 { path: "/settings/authorizations", auth: "user" },
149 { path: "/settings/audit", auth: "user" },
150 { path: "/settings/delete-account", auth: "user" },
151 { path: "/settings/signing-keys", auth: "user" },
152 { path: "/settings/replies", auth: "user" },
153 { path: "/settings/sponsors", auth: "user" },
154 { path: "/settings/apps", auth: "user" },
155
156 // Admin
157 { path: "/admin", auth: "admin" },
158 { path: "/admin/ops", auth: "admin" },
159 { path: "/admin/autopilot", auth: "admin" },
160 { path: "/admin/billing", auth: "admin" },
161 { path: "/admin/deploys", auth: "admin" },
162 { path: "/admin/deploys/latest.json", auth: "admin" },
163 { path: "/admin/digests", auth: "admin" },
164 { path: "/admin/flags", auth: "admin" },
165 { path: "/admin/github-oauth", auth: "admin" },
166 { path: "/admin/repos", auth: "admin" },
167 { path: "/admin/self-host", auth: "admin" },
168 { path: "/admin/sso", auth: "admin" },
169 { path: "/admin/status", auth: "admin" },
170 { path: "/admin/users", auth: "admin" },
171];
172
173// ─── Crawl machinery ─────────────────────────────────────────────────
174
175type Result = {
176 path: string;
177 auth: AuthLevel;
178 status: number;
179 ms: number;
180 verdict: "OK" | "FAIL" | "SLOW" | "SKIP" | "REDIR";
181 note: string;
182};
183
184async function check(route: Route): Promise<Result> {
185 const url = `${HOST}${route.path}`;
186 const needsAuth = route.auth !== "anon";
187
188 if (needsAuth && !SESSION) {
189 return {
190 path: route.path,
191 auth: route.auth,
192 status: 0,
193 ms: 0,
194 verdict: "SKIP",
195 note: "no session cookie",
196 };
197 }
198
199 const ctrl = new AbortController();
200 const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
201 const headers: Record<string, string> = {
202 "user-agent": "gluecron-site-crawl/1.0",
203 accept: "text/html,application/json,*/*",
204 };
205 if (needsAuth) headers["cookie"] = `session=${SESSION}`;
206
207 const t0 = performance.now();
208 let res: Response | null = null;
209 let err: unknown = null;
210 try {
211 res = await fetch(url, {
212 method: route.method || "GET",
213 headers,
214 signal: ctrl.signal,
215 redirect: "manual",
216 });
217 } catch (e) {
218 err = e;
219 } finally {
220 clearTimeout(timer);
221 }
222 const ms = Math.round(performance.now() - t0);
223
224 if (err || !res) {
225 return {
226 path: route.path,
227 auth: route.auth,
228 status: 0,
229 ms,
230 verdict: "FAIL",
231 note: (err as Error)?.message?.slice(0, 60) || "network error",
232 };
233 }
234
235 const status = res.status;
236
237 // 3xx → look at Location. If admin/user route bounced to /login, flag.
238 if (status >= 300 && status < 400) {
239 const loc = res.headers.get("location") || "";
240 if (needsAuth && /\/login(\?|$)/.test(loc)) {
241 return {
242 path: route.path,
243 auth: route.auth,
244 status,
245 ms,
246 verdict: "FAIL",
247 note: "redirected to /login (session invalid?)",
248 };
249 }
250 return {
251 path: route.path,
252 auth: route.auth,
253 status,
254 ms,
255 verdict: "REDIR",
256 note: `→ ${loc.replace(HOST, "")}`.slice(0, 60),
257 };
258 }
259
260 if (status >= 400) {
261 return {
262 path: route.path,
263 auth: route.auth,
264 status,
265 ms,
266 verdict: "FAIL",
267 note: status >= 500 ? "server error" : "client error",
268 };
269 }
270
271 if (ms > SLOW_MS) {
272 return {
273 path: route.path,
274 auth: route.auth,
275 status,
276 ms,
277 verdict: "SLOW",
278 note: `> ${SLOW_MS}ms`,
279 };
280 }
281
282 return {
283 path: route.path,
284 auth: route.auth,
285 status,
286 ms,
287 verdict: "OK",
288 note: "",
289 };
290}
291
292async function crawl(): Promise<Result[]> {
293 const results: Result[] = [];
294 const queue = [...ROUTES];
295 const workers = new Array(Math.min(CONCURRENCY, queue.length))
296 .fill(0)
297 .map(async () => {
298 while (queue.length) {
299 const route = queue.shift();
300 if (!route) break;
301 const r = await check(route);
302 results.push(r);
303 const tag =
304 r.verdict === "OK"
305 ? "·"
306 : r.verdict === "SLOW"
307 ? "~"
308 : r.verdict === "REDIR"
309 ? ">"
310 : r.verdict === "SKIP"
311 ? "?"
312 : "X";
313 process.stderr.write(`${tag} ${r.status || "---"} ${r.path}\n`);
314 }
315 });
316 await Promise.all(workers);
317 return results;
318}
319
320function pad(s: string, n: number): string {
321 return s.length >= n ? s.slice(0, n) : s + " ".repeat(n - s.length);
322}
323
324function formatTable(rows: Result[]): string {
325 const rank: Record<Result["verdict"], number> = {
326 FAIL: 0,
327 SLOW: 1,
328 REDIR: 2,
329 SKIP: 3,
330 OK: 4,
331 };
332 rows.sort((a, b) => {
333 if (rank[a.verdict] !== rank[b.verdict]) return rank[a.verdict] - rank[b.verdict];
334 return a.path.localeCompare(b.path);
335 });
336 const lines = [
337 `| verdict | status | ms | auth | path | note`,
338 `| ------- | ------ | ---- | ----- | -------------------------------------------------------- | ----`,
339 ];
340 for (const r of rows) {
341 lines.push(
342 `| ${pad(r.verdict, 7)} | ${pad(String(r.status || "---"), 6)} | ${pad(String(r.ms), 4)} | ${pad(r.auth, 5)} | ${pad(r.path, 56)} | ${r.note}`
343 );
344 }
345 return lines.join("\n");
346}
347
348async function main() {
349 console.error(`[crawl] target: ${HOST}`);
350 console.error(`[crawl] session: ${SESSION ? "yes" : "no (admin routes will skip)"}`);
351 console.error(`[crawl] routes: ${ROUTES.length}`);
352 console.error("");
353
354 const results = await crawl();
355
356 const counts = {
357 FAIL: results.filter((r) => r.verdict === "FAIL").length,
358 SLOW: results.filter((r) => r.verdict === "SLOW").length,
359 REDIR: results.filter((r) => r.verdict === "REDIR").length,
360 SKIP: results.filter((r) => r.verdict === "SKIP").length,
361 OK: results.filter((r) => r.verdict === "OK").length,
362 };
363
364 console.log("");
365 console.log(formatTable(results));
366 console.log("");
367 console.log(
368 `[crawl] ${counts.OK} ok · ${counts.FAIL} fail · ${counts.SLOW} slow · ${counts.REDIR} redir · ${counts.SKIP} skip (no cookie)`
369 );
370
371 if (counts.FAIL > 0) process.exit(1);
372 process.exit(0);
373}
374
375main().catch((err) => {
376 console.error("[crawl] crashed:", err);
377 process.exit(2);
378});
Modifiedsrc/routes/admin-ops.tsx+108−1View fileUnifiedSplit
@@ -34,13 +34,14 @@
3434import { Hono } from "hono";
3535import { and, desc, eq, sql } from "drizzle-orm";
3636import { db } from "../db";
37import { branchProtection, repositories, users } from "../db/schema";
37import { apiTokens, branchProtection, repositories, users } from "../db/schema";
3838import { platformDeploys } from "../db/schema-deploys";
3939import { Layout } from "../views/layout";
4040import { softAuth } from "../middleware/auth";
4141import type { AuthEnv } from "../middleware/auth";
4242import { isSiteAdmin } from "../lib/admin";
4343import { audit as realAudit } from "../lib/notify";
44import { config } from "../lib/config";
4445import {
4546 runEnableAutoMerge as realRunEnableAutoMerge,
4647 type DbLike,
@@ -463,6 +464,48 @@ ops.get("/admin/ops", async (c) => {
463464 </div>
464465 </CardShell>
465466
467 {/* ---- GateTest scanner credentials card ---- */}
468 <CardShell title="GateTest scanner credentials">
469 <p style="font-size:13px;color:var(--text-muted);margin:0 0 12px 0">
470 Two values to paste into GateTest's environment so it can scan
471 this site. Token is admin-scoped — revoke at{" "}
472 <a href="/settings/tokens">/settings/tokens</a> when scanning is done.
473 </p>
474 <div style="display:grid;grid-template-columns:160px 1fr;gap:6px 10px;font-size:13px;margin-bottom:14px">
475 <code class="meta-mono">GLUECRON_BASE_URL</code>
476 <code class="meta-mono" style="word-break:break-all">
477 {config.appBaseUrl}
478 </code>
479 <code class="meta-mono">GLUECRON_API_TOKEN</code>
480 <code
481 class="meta-mono"
482 style={
483 c.req.query("gatetest_token")
484 ? "word-break:break-all;color:var(--accent)"
485 : "color:var(--text-muted)"
486 }
487 >
488 {c.req.query("gatetest_token") ||
489 "— click below to issue (shown once) —"}
490 </code>
491 </div>
492 {c.req.query("gatetest_token") && (
493 <p style="font-size:12px;color:#f59e0b;margin:0 0 12px 0">
494 Copy the token now. It is hashed in the DB and will not be shown
495 again.
496 </p>
497 )}
498 <form
499 method="post"
500 action="/admin/ops/gatetest-token"
501 style="margin:0"
502 >
503 <button type="submit" class="btn btn-sm btn-primary">
504 Issue scanner token
505 </button>
506 </form>
507 </CardShell>
508
466509 {/* ---- Rollback card ---- */}
467510 <CardShell title="Rollback">
468511 <div style="margin-bottom:12px;font-size:13px">
@@ -693,6 +736,70 @@ ops.post("/admin/ops/rollback", async (c) => {
693736 );
694737});
695738
739// ---------------------------------------------------------------------------
740// POST /admin/ops/gatetest-token
741// ---------------------------------------------------------------------------
742//
743// Mint a fresh admin-scoped API token for the GateTest scanner and surface
744// it once via the redirect query string. The DB stores only the SHA-256
745// hash (same shape tokens.tsx uses) so the plaintext is unrecoverable after
746// this redirect — operator must copy it immediately into GateTest's
747// environment.
748
749function generateGateTestToken(): string {
750 const bytes = crypto.getRandomValues(new Uint8Array(32));
751 return (
752 "glc_" +
753 Array.from(bytes)
754 .map((b) => b.toString(16).padStart(2, "0"))
755 .join("")
756 );
757}
758
759async function sha256Hex(input: string): Promise<string> {
760 const data = new TextEncoder().encode(input);
761 const hash = await crypto.subtle.digest("SHA-256", data);
762 return Array.from(new Uint8Array(hash))
763 .map((b) => b.toString(16).padStart(2, "0"))
764 .join("");
765}
766
767ops.post("/admin/ops/gatetest-token", async (c) => {
768 const g = await gate(c);
769 if (g instanceof Response) return g;
770 const { user } = g;
771
772 const token = generateGateTestToken();
773 const tokenH = await sha256Hex(token);
774 const stamp = new Date().toISOString().slice(0, 10);
775
776 try {
777 await db.insert(apiTokens).values({
778 userId: user.id,
779 name: `GateTest scanner (${stamp})`,
780 tokenHash: tokenH,
781 tokenPrefix: token.slice(0, 12),
782 scopes: "admin",
783 });
784 } catch (err) {
785 const message = err instanceof Error ? err.message : String(err);
786 return redirectWith(c, "error", `Token insert failed: ${message}`);
787 }
788
789 try {
790 await _deps.audit({
791 userId: user.id,
792 action: "admin.ops.gatetest_token_issued",
793 targetType: "api_token",
794 metadata: { scope: "admin", prefix: token.slice(0, 12) },
795 });
796 } catch {
797 /* non-fatal */
798 }
799
800 return c.redirect(`/admin/ops?gatetest_token=${encodeURIComponent(token)}`);
801});
802
696803export const __test = {
697804 readAutoMergeState,
698805 readLatestDeploy,
699806