CodeIssuesDiscussionsWikiPull RequestsProjectsCommitsActionsReleasesContributorsPulse● GatesSecuritySettingsDeploymentsPipelineInsightsAgents✨ Explain✨ Ask AI✨ Workspace✨ Spec✨ Tests▓ Debt Map✨ NL Search🏛 Archaeology
Blame · Line-by-line history
site-crawl.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| bbac5c5 | 1 | #!/usr/bin/env bun |
| 2 | /** | |
| 3 | * One-shot site crawler. | |
| 4 | * | |
| 5 | * Hits every known public + admin route and reports any URL that: | |
| 6 | * - returned 4xx or 5xx | |
| 7 | * - took > SLOW_MS to respond | |
| 8 | * - bounced an admin route to /login (auth not seen) | |
| 9 | * | |
| 10 | * Public routes are crawled anonymously. Admin/settings/billing routes | |
| 11 | * are only crawled if you pass GLUECRON_SESSION=<sid cookie value>. | |
| 12 | * | |
| 13 | * Usage (from the box, or anywhere with internet): | |
| 14 | * | |
| 15 | * GLUECRON_HOST=https://gluecron.com \ | |
| 16 | * GLUECRON_SESSION=$(psql "$DATABASE_URL" -tAc \ | |
| 17 | * "select token from sessions \ | |
| 18 | * where user_id=(select id from users where username='admin') \ | |
| 19 | * and expires_at > now() and requires_2fa = false \ | |
| 20 | * order by created_at desc limit 1" | tr -d ' ') \ | |
| 21 | * bun scripts/site-crawl.ts | |
| 22 | * | |
| 23 | * If you skip GLUECRON_SESSION the script still runs — it will just mark | |
| 24 | * every admin route as "auth-required (skipped)". | |
| 25 | * | |
| 26 | * Output: a single markdown table sorted FAIL → SLOW → OK, plus a | |
| 27 | * one-line summary at the bottom you can paste back. | |
| 28 | */ | |
| 29 | ||
| 30 | const HOST = (process.env.GLUECRON_HOST || "https://gluecron.com").replace( | |
| 31 | /\/$/, | |
| 32 | "" | |
| 33 | ); | |
| 34 | const SESSION = process.env.GLUECRON_SESSION || ""; | |
| 35 | const CONCURRENCY = Number(process.env.CRAWL_CONCURRENCY || 6); | |
| 36 | const TIMEOUT_MS = Number(process.env.CRAWL_TIMEOUT_MS || 8000); | |
| 37 | const SLOW_MS = Number(process.env.CRAWL_SLOW_MS || 2000); | |
| 38 | ||
| 39 | // ─── Route inventory ───────────────────────────────────────────────── | |
| 40 | // | |
| 41 | // Curated subset of the 307 routes: every static page, plus parameterised | |
| 42 | // pages instantiated with `ccantynz/Gluecron.com` (the canonical self- | |
| 43 | // hosted repo) and `admin` (the bootstrap operator). Anything that needs | |
| 44 | // real session state (issue numbers, gist slugs, etc.) is skipped. | |
| 45 | // | |
| 46 | // `auth` = "anon" public; "admin" needs site-admin cookie; "user" needs | |
| 47 | // any logged-in cookie. The crawler treats both admin+user as "skip if | |
| 48 | // no session cookie is set". | |
| 49 | ||
| 50 | type AuthLevel = "anon" | "user" | "admin"; | |
| 51 | type Route = { path: string; auth: AuthLevel; method?: "GET" | "POST" }; | |
| 52 | ||
| 53 | const ROUTES: Route[] = [ | |
| 54 | // Public landing + marketing | |
| 55 | { path: "/", auth: "anon" }, | |
| 56 | { path: "/about", auth: "anon" }, | |
| 57 | { path: "/features", auth: "anon" }, | |
| 58 | { path: "/pricing", auth: "anon" }, | |
| 59 | { path: "/explore", auth: "anon" }, | |
| 60 | { path: "/demo", auth: "anon" }, | |
| 61 | { path: "/help", auth: "anon" }, | |
| 62 | { path: "/getting-started", auth: "anon" }, | |
| 63 | { path: "/install", auth: "anon" }, | |
| 64 | { path: "/vs-github", auth: "anon" }, | |
| 65 | { path: "/marketplace", auth: "anon" }, | |
| 66 | { path: "/shortcuts", auth: "anon" }, | |
| 67 | { path: "/sleep-mode", auth: "anon" }, | |
| 68 | { path: "/setup", auth: "anon" }, | |
| 69 | ||
| 70 | // Legal | |
| 71 | { path: "/terms", auth: "anon" }, | |
| 72 | { path: "/privacy", auth: "anon" }, | |
| 73 | { path: "/acceptable-use", auth: "anon" }, | |
| 74 | { path: "/legal/terms", auth: "anon" }, | |
| 75 | { path: "/legal/privacy", auth: "anon" }, | |
| 76 | { path: "/legal/acceptable-use", auth: "anon" }, | |
| 77 | { path: "/legal/dmca", auth: "anon" }, | |
| 78 | ||
| 79 | // Auth flows (anon GET should render the form) | |
| 80 | { path: "/login", auth: "anon" }, | |
| 81 | { path: "/register", auth: "anon" }, | |
| 82 | { path: "/forgot-password", auth: "anon" }, | |
| 83 | { path: "/reset-password", auth: "anon" }, | |
| 84 | { path: "/login/magic", auth: "anon" }, | |
| 85 | { path: "/verify-email", auth: "anon" }, | |
| 86 | { path: "/play", auth: "anon" }, | |
| 87 | ||
| 88 | // Health + version + manifests | |
| 89 | { path: "/health", auth: "anon" }, | |
| 90 | { path: "/healthz", auth: "anon" }, | |
| 91 | { path: "/readyz", auth: "anon" }, | |
| 92 | { path: "/version", auth: "anon" }, | |
| 93 | { path: "/api/version", auth: "anon" }, | |
| 94 | { path: "/api/docs", auth: "anon" }, | |
| 95 | { path: "/status", auth: "anon" }, | |
| 96 | { path: "/status.svg", auth: "anon" }, | |
| 97 | { path: "/robots.txt", auth: "anon" }, | |
| 98 | { path: "/sitemap.xml", auth: "anon" }, | |
| 99 | { path: "/manifest.webmanifest", auth: "anon" }, | |
| 100 | { path: "/icon.svg", auth: "anon" }, | |
| 101 | { path: "/sw.js", auth: "anon" }, | |
| 102 | { path: "/sw-push.js", auth: "anon" }, | |
| 103 | { path: "/pwa/vapid-public-key", auth: "anon" }, | |
| 104 | { path: "/gluecron.dxt", auth: "anon" }, | |
| 105 | ||
| 106 | // Public repo browsing (canonical self-host repo) | |
| 107 | { path: "/ccantynz", auth: "anon" }, | |
| 108 | { path: "/ccantynz/Gluecron.com", auth: "anon" }, | |
| 109 | { path: "/ccantynz/Gluecron.com/commits", auth: "anon" }, | |
| 110 | { path: "/ccantynz/Gluecron.com/branches", auth: "anon" }, | |
| 111 | { path: "/ccantynz/Gluecron.com/contributors", auth: "anon" }, | |
| 112 | { path: "/ccantynz/Gluecron.com/issues", auth: "anon" }, | |
| 113 | { path: "/ccantynz/Gluecron.com/pulls", auth: "anon" }, | |
| 114 | { path: "/ccantynz/Gluecron.com/releases", auth: "anon" }, | |
| 115 | { path: "/ccantynz/Gluecron.com/wiki", auth: "anon" }, | |
| 116 | { path: "/ccantynz/Gluecron.com/discussions", auth: "anon" }, | |
| 117 | { path: "/ccantynz/Gluecron.com/security", auth: "anon" }, | |
| 118 | { path: "/ccantynz/Gluecron.com/security/advisories", auth: "anon" }, | |
| 119 | { path: "/ccantynz/Gluecron.com/actions", auth: "anon" }, | |
| 120 | { path: "/ccantynz/Gluecron.com/deployments", auth: "anon" }, | |
| 121 | { path: "/ccantynz/Gluecron.com/dependencies", auth: "anon" }, | |
| 122 | { path: "/ccantynz/Gluecron.com/contributors", auth: "anon" }, | |
| 123 | { path: "/ccantynz/Gluecron.com/traffic", auth: "anon" }, | |
| 124 | ||
| 125 | // Authed user pages | |
| 126 | { path: "/dashboard", auth: "user" }, | |
| 127 | { path: "/feed", auth: "user" }, | |
| 128 | { path: "/notifications", auth: "user" }, | |
| 129 | { path: "/me/ai-savings", auth: "user" }, | |
| 130 | { path: "/new", auth: "user" }, | |
| 131 | { path: "/import", auth: "user" }, | |
| 132 | { path: "/gists", auth: "user" }, | |
| 133 | { path: "/gists/new", auth: "user" }, | |
| 134 | { path: "/todos", auth: "user" }, | |
| 135 | { path: "/ask", auth: "user" }, | |
| 136 | { path: "/billing/manage", auth: "user" }, | |
| 137 | ||
| 138 | // User settings | |
| 139 | { path: "/settings", auth: "user" }, | |
| 140 | { path: "/settings/profile", auth: "user" }, | |
| 141 | { path: "/settings/notifications", auth: "user" }, | |
| 142 | { path: "/settings/billing", auth: "user" }, | |
| 143 | { path: "/settings/keys", auth: "user" }, | |
| 144 | { path: "/settings/tokens", auth: "user" }, | |
| 145 | { path: "/settings/passkeys", auth: "user" }, | |
| 146 | { path: "/settings/2fa", auth: "user" }, | |
| 147 | { path: "/settings/applications", auth: "user" }, | |
| 148 | { path: "/settings/authorizations", auth: "user" }, | |
| 149 | { path: "/settings/audit", auth: "user" }, | |
| 150 | { path: "/settings/delete-account", auth: "user" }, | |
| 151 | { path: "/settings/signing-keys", auth: "user" }, | |
| 152 | { path: "/settings/replies", auth: "user" }, | |
| 153 | { path: "/settings/sponsors", auth: "user" }, | |
| 154 | { path: "/settings/apps", auth: "user" }, | |
| 155 | ||
| 156 | // Admin | |
| 157 | { path: "/admin", auth: "admin" }, | |
| 158 | { path: "/admin/ops", auth: "admin" }, | |
| 159 | { path: "/admin/autopilot", auth: "admin" }, | |
| 160 | { path: "/admin/billing", auth: "admin" }, | |
| 161 | { path: "/admin/deploys", auth: "admin" }, | |
| 162 | { path: "/admin/deploys/latest.json", auth: "admin" }, | |
| 163 | { path: "/admin/digests", auth: "admin" }, | |
| 164 | { path: "/admin/flags", auth: "admin" }, | |
| 165 | { path: "/admin/github-oauth", auth: "admin" }, | |
| 166 | { path: "/admin/repos", auth: "admin" }, | |
| 167 | { path: "/admin/self-host", auth: "admin" }, | |
| 168 | { path: "/admin/sso", auth: "admin" }, | |
| 169 | { path: "/admin/status", auth: "admin" }, | |
| 170 | { path: "/admin/users", auth: "admin" }, | |
| 171 | ]; | |
| 172 | ||
| 173 | // ─── Crawl machinery ───────────────────────────────────────────────── | |
| 174 | ||
| 175 | type Result = { | |
| 176 | path: string; | |
| 177 | auth: AuthLevel; | |
| 178 | status: number; | |
| 179 | ms: number; | |
| 180 | verdict: "OK" | "FAIL" | "SLOW" | "SKIP" | "REDIR"; | |
| 181 | note: string; | |
| 182 | }; | |
| 183 | ||
| 184 | async function check(route: Route): Promise<Result> { | |
| 185 | const url = `${HOST}${route.path}`; | |
| 186 | const needsAuth = route.auth !== "anon"; | |
| 187 | ||
| 188 | if (needsAuth && !SESSION) { | |
| 189 | return { | |
| 190 | path: route.path, | |
| 191 | auth: route.auth, | |
| 192 | status: 0, | |
| 193 | ms: 0, | |
| 194 | verdict: "SKIP", | |
| 195 | note: "no session cookie", | |
| 196 | }; | |
| 197 | } | |
| 198 | ||
| 199 | const ctrl = new AbortController(); | |
| 200 | const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS); | |
| 201 | const headers: Record<string, string> = { | |
| 202 | "user-agent": "gluecron-site-crawl/1.0", | |
| 203 | accept: "text/html,application/json,*/*", | |
| 204 | }; | |
| 205 | if (needsAuth) headers["cookie"] = `session=${SESSION}`; | |
| 206 | ||
| 207 | const t0 = performance.now(); | |
| 208 | let res: Response | null = null; | |
| 209 | let err: unknown = null; | |
| 210 | try { | |
| 211 | res = await fetch(url, { | |
| 212 | method: route.method || "GET", | |
| 213 | headers, | |
| 214 | signal: ctrl.signal, | |
| 215 | redirect: "manual", | |
| 216 | }); | |
| 217 | } catch (e) { | |
| 218 | err = e; | |
| 219 | } finally { | |
| 220 | clearTimeout(timer); | |
| 221 | } | |
| 222 | const ms = Math.round(performance.now() - t0); | |
| 223 | ||
| 224 | if (err || !res) { | |
| 225 | return { | |
| 226 | path: route.path, | |
| 227 | auth: route.auth, | |
| 228 | status: 0, | |
| 229 | ms, | |
| 230 | verdict: "FAIL", | |
| 231 | note: (err as Error)?.message?.slice(0, 60) || "network error", | |
| 232 | }; | |
| 233 | } | |
| 234 | ||
| 235 | const status = res.status; | |
| 236 | ||
| 237 | // 3xx → look at Location. If admin/user route bounced to /login, flag. | |
| 238 | if (status >= 300 && status < 400) { | |
| 239 | const loc = res.headers.get("location") || ""; | |
| 240 | if (needsAuth && /\/login(\?|$)/.test(loc)) { | |
| 241 | return { | |
| 242 | path: route.path, | |
| 243 | auth: route.auth, | |
| 244 | status, | |
| 245 | ms, | |
| 246 | verdict: "FAIL", | |
| 247 | note: "redirected to /login (session invalid?)", | |
| 248 | }; | |
| 249 | } | |
| 250 | return { | |
| 251 | path: route.path, | |
| 252 | auth: route.auth, | |
| 253 | status, | |
| 254 | ms, | |
| 255 | verdict: "REDIR", | |
| 256 | note: `→ ${loc.replace(HOST, "")}`.slice(0, 60), | |
| 257 | }; | |
| 258 | } | |
| 259 | ||
| 260 | if (status >= 400) { | |
| 261 | return { | |
| 262 | path: route.path, | |
| 263 | auth: route.auth, | |
| 264 | status, | |
| 265 | ms, | |
| 266 | verdict: "FAIL", | |
| 267 | note: status >= 500 ? "server error" : "client error", | |
| 268 | }; | |
| 269 | } | |
| 270 | ||
| 271 | if (ms > SLOW_MS) { | |
| 272 | return { | |
| 273 | path: route.path, | |
| 274 | auth: route.auth, | |
| 275 | status, | |
| 276 | ms, | |
| 277 | verdict: "SLOW", | |
| 278 | note: `> ${SLOW_MS}ms`, | |
| 279 | }; | |
| 280 | } | |
| 281 | ||
| 282 | return { | |
| 283 | path: route.path, | |
| 284 | auth: route.auth, | |
| 285 | status, | |
| 286 | ms, | |
| 287 | verdict: "OK", | |
| 288 | note: "", | |
| 289 | }; | |
| 290 | } | |
| 291 | ||
| 292 | async function crawl(): Promise<Result[]> { | |
| 293 | const results: Result[] = []; | |
| 294 | const queue = [...ROUTES]; | |
| 295 | const workers = new Array(Math.min(CONCURRENCY, queue.length)) | |
| 296 | .fill(0) | |
| 297 | .map(async () => { | |
| 298 | while (queue.length) { | |
| 299 | const route = queue.shift(); | |
| 300 | if (!route) break; | |
| 301 | const r = await check(route); | |
| 302 | results.push(r); | |
| 303 | const tag = | |
| 304 | r.verdict === "OK" | |
| 305 | ? "·" | |
| 306 | : r.verdict === "SLOW" | |
| 307 | ? "~" | |
| 308 | : r.verdict === "REDIR" | |
| 309 | ? ">" | |
| 310 | : r.verdict === "SKIP" | |
| 311 | ? "?" | |
| 312 | : "X"; | |
| 313 | process.stderr.write(`${tag} ${r.status || "---"} ${r.path}\n`); | |
| 314 | } | |
| 315 | }); | |
| 316 | await Promise.all(workers); | |
| 317 | return results; | |
| 318 | } | |
| 319 | ||
| 320 | function pad(s: string, n: number): string { | |
| 321 | return s.length >= n ? s.slice(0, n) : s + " ".repeat(n - s.length); | |
| 322 | } | |
| 323 | ||
| 324 | function formatTable(rows: Result[]): string { | |
| 325 | const rank: Record<Result["verdict"], number> = { | |
| 326 | FAIL: 0, | |
| 327 | SLOW: 1, | |
| 328 | REDIR: 2, | |
| 329 | SKIP: 3, | |
| 330 | OK: 4, | |
| 331 | }; | |
| 332 | rows.sort((a, b) => { | |
| 333 | if (rank[a.verdict] !== rank[b.verdict]) return rank[a.verdict] - rank[b.verdict]; | |
| 334 | return a.path.localeCompare(b.path); | |
| 335 | }); | |
| 336 | const lines = [ | |
| 337 | `| verdict | status | ms | auth | path | note`, | |
| 338 | `| ------- | ------ | ---- | ----- | -------------------------------------------------------- | ----`, | |
| 339 | ]; | |
| 340 | for (const r of rows) { | |
| 341 | lines.push( | |
| 342 | `| ${pad(r.verdict, 7)} | ${pad(String(r.status || "---"), 6)} | ${pad(String(r.ms), 4)} | ${pad(r.auth, 5)} | ${pad(r.path, 56)} | ${r.note}` | |
| 343 | ); | |
| 344 | } | |
| 345 | return lines.join("\n"); | |
| 346 | } | |
| 347 | ||
| 348 | async function main() { | |
| 349 | console.error(`[crawl] target: ${HOST}`); | |
| 350 | console.error(`[crawl] session: ${SESSION ? "yes" : "no (admin routes will skip)"}`); | |
| 351 | console.error(`[crawl] routes: ${ROUTES.length}`); | |
| 352 | console.error(""); | |
| 353 | ||
| 354 | const results = await crawl(); | |
| 355 | ||
| 356 | const counts = { | |
| 357 | FAIL: results.filter((r) => r.verdict === "FAIL").length, | |
| 358 | SLOW: results.filter((r) => r.verdict === "SLOW").length, | |
| 359 | REDIR: results.filter((r) => r.verdict === "REDIR").length, | |
| 360 | SKIP: results.filter((r) => r.verdict === "SKIP").length, | |
| 361 | OK: results.filter((r) => r.verdict === "OK").length, | |
| 362 | }; | |
| 363 | ||
| 364 | console.log(""); | |
| 365 | console.log(formatTable(results)); | |
| 366 | console.log(""); | |
| 367 | console.log( | |
| 368 | `[crawl] ${counts.OK} ok · ${counts.FAIL} fail · ${counts.SLOW} slow · ${counts.REDIR} redir · ${counts.SKIP} skip (no cookie)` | |
| 369 | ); | |
| 370 | ||
| 371 | if (counts.FAIL > 0) process.exit(1); | |
| 372 | process.exit(0); | |
| 373 | } | |
| 374 | ||
| 375 | main().catch((err) => { | |
| 376 | console.error("[crawl] crashed:", err); | |
| 377 | process.exit(2); | |
| 378 | }); |