Blame · Line-by-line history
language-stats.ts
Each line is annotated with the commit that last touched it. Click any SHA to jump to that commit and see the surrounding change.
| 07efa08 | 1 | /** |
| 2 | * Block J30 — Repository language breakdown. | |
| 3 | * | |
| 4 | * Pure helpers that map file paths to a language (via extension + a handful | |
| 5 | * of filename-only special cases like `Dockerfile`, `Makefile`), compute | |
| 6 | * per-language byte totals + percentages, and can fold low-share languages | |
| 7 | * into an "Other" bucket. | |
| 8 | * | |
| 9 | * We also ship a compact `LANGUAGE_COLORS` map so the route can render a | |
| 10 | * stacked bar matching common colour conventions (GitHub-ish). | |
| 11 | * | |
| 12 | * This deliberately lives outside any linguist-style heuristic soup — | |
| 13 | * we don't auto-detect by content, we trust extensions + filenames. It's | |
| 14 | * lightweight and good enough for the breakdown UI. | |
| 15 | */ | |
| 16 | ||
| 17 | /** | |
| 18 | * Extension → language. Keys are without leading dot, lowercased. | |
| 19 | * A deliberately opinionated subset — no one-off exotic languages. | |
| 20 | */ | |
| 21 | export const EXTENSION_MAP: ReadonlyMap<string, string> = new Map( | |
| 22 | Object.entries({ | |
| 23 | ts: "TypeScript", | |
| 24 | tsx: "TypeScript", | |
| 25 | js: "JavaScript", | |
| 26 | jsx: "JavaScript", | |
| 27 | mjs: "JavaScript", | |
| 28 | cjs: "JavaScript", | |
| 29 | py: "Python", | |
| 30 | rb: "Ruby", | |
| 31 | go: "Go", | |
| 32 | rs: "Rust", | |
| 33 | java: "Java", | |
| 34 | kt: "Kotlin", | |
| 35 | kts: "Kotlin", | |
| 36 | swift: "Swift", | |
| 37 | c: "C", | |
| 38 | h: "C", | |
| 39 | cpp: "C++", | |
| 40 | cxx: "C++", | |
| 41 | cc: "C++", | |
| 42 | hpp: "C++", | |
| 43 | hh: "C++", | |
| 44 | cs: "C#", | |
| 45 | php: "PHP", | |
| 46 | pl: "Perl", | |
| 47 | pm: "Perl", | |
| 48 | r: "R", | |
| 49 | scala: "Scala", | |
| 50 | clj: "Clojure", | |
| 51 | ex: "Elixir", | |
| 52 | exs: "Elixir", | |
| 53 | erl: "Erlang", | |
| 54 | hs: "Haskell", | |
| 55 | lua: "Lua", | |
| 56 | ml: "OCaml", | |
| 57 | mli: "OCaml", | |
| 58 | dart: "Dart", | |
| 59 | sh: "Shell", | |
| 60 | bash: "Shell", | |
| 61 | zsh: "Shell", | |
| 62 | fish: "Shell", | |
| 63 | ps1: "PowerShell", | |
| 64 | sql: "SQL", | |
| 65 | html: "HTML", | |
| 66 | htm: "HTML", | |
| 67 | css: "CSS", | |
| 68 | scss: "SCSS", | |
| 69 | sass: "Sass", | |
| 70 | less: "Less", | |
| 71 | vue: "Vue", | |
| 72 | svelte: "Svelte", | |
| 73 | astro: "Astro", | |
| 74 | md: "Markdown", | |
| 75 | mdx: "MDX", | |
| 76 | rst: "reStructuredText", | |
| 77 | tex: "TeX", | |
| 78 | yaml: "YAML", | |
| 79 | yml: "YAML", | |
| 80 | json: "JSON", | |
| 81 | json5: "JSON", | |
| 82 | toml: "TOML", | |
| 83 | xml: "XML", | |
| 84 | ini: "INI", | |
| 85 | conf: "INI", | |
| 86 | proto: "Protocol Buffers", | |
| 87 | graphql: "GraphQL", | |
| 88 | gql: "GraphQL", | |
| 89 | zig: "Zig", | |
| 90 | nim: "Nim", | |
| 91 | vim: "Vim script", | |
| 92 | vb: "Visual Basic", | |
| 93 | asm: "Assembly", | |
| 94 | s: "Assembly", | |
| 95 | dockerfile: "Dockerfile", | |
| 96 | }) | |
| 97 | ); | |
| 98 | ||
| 99 | /** Filename → language for files that traditionally have no extension. */ | |
| 100 | export const FILENAME_MAP: ReadonlyMap<string, string> = new Map( | |
| 101 | Object.entries({ | |
| 102 | dockerfile: "Dockerfile", | |
| 103 | makefile: "Makefile", | |
| 104 | gnumakefile: "Makefile", | |
| 105 | rakefile: "Ruby", | |
| 106 | gemfile: "Ruby", | |
| 107 | procfile: "Procfile", | |
| 108 | jenkinsfile: "Groovy", | |
| 109 | "cmakelists.txt": "CMake", | |
| 110 | "meson.build": "Meson", | |
| 111 | "build.gradle": "Groovy", | |
| 112 | "build.gradle.kts": "Kotlin", | |
| 113 | "pom.xml": "XML", | |
| 114 | "package.json": "JSON", | |
| 115 | "tsconfig.json": "JSON", | |
| 116 | }) | |
| 117 | ); | |
| 118 | ||
| 119 | /** | |
| 120 | * GitHub-ish language colours. Omitted entries render with a neutral grey. | |
| 121 | * Only widely-used languages get a brand colour here. | |
| 122 | */ | |
| 123 | export const LANGUAGE_COLORS: ReadonlyMap<string, string> = new Map( | |
| 124 | Object.entries({ | |
| 125 | TypeScript: "#3178c6", | |
| 126 | JavaScript: "#f1e05a", | |
| 127 | Python: "#3572A5", | |
| 128 | Ruby: "#701516", | |
| 129 | Go: "#00ADD8", | |
| 130 | Rust: "#dea584", | |
| 131 | Java: "#b07219", | |
| 132 | Kotlin: "#A97BFF", | |
| 133 | Swift: "#F05138", | |
| 134 | C: "#555555", | |
| 135 | "C++": "#f34b7d", | |
| 136 | "C#": "#178600", | |
| 137 | PHP: "#4F5D95", | |
| 138 | Scala: "#c22d40", | |
| 139 | Clojure: "#db5855", | |
| 140 | Elixir: "#6e4a7e", | |
| 141 | Haskell: "#5e5086", | |
| 142 | Lua: "#000080", | |
| 143 | Shell: "#89e051", | |
| 144 | PowerShell: "#012456", | |
| 145 | HTML: "#e34c26", | |
| 146 | CSS: "#563d7c", | |
| 147 | SCSS: "#c6538c", | |
| 148 | Vue: "#41b883", | |
| 149 | Svelte: "#ff3e00", | |
| 150 | Astro: "#ff5d01", | |
| 151 | Markdown: "#083fa1", | |
| 152 | YAML: "#cb171e", | |
| 153 | JSON: "#292929", | |
| 154 | TOML: "#9c4221", | |
| 155 | GraphQL: "#e10098", | |
| 156 | Dockerfile: "#384d54", | |
| 157 | Makefile: "#427819", | |
| 158 | SQL: "#e38c00", | |
| 159 | Dart: "#00B4AB", | |
| 160 | Zig: "#ec915c", | |
| 161 | Other: "#888888", | |
| 162 | }) | |
| 163 | ); | |
| 164 | ||
| 165 | /** Fallback colour when a language has no LANGUAGE_COLORS entry. */ | |
| 166 | export const DEFAULT_LANGUAGE_COLOR = "#888888"; | |
| 167 | ||
| 168 | /** | |
| 169 | * Paths we never want in the breakdown: vendored + generated + lock files. | |
| 170 | * Callers can opt out by passing `ignoreVendored: false`. | |
| 171 | */ | |
| 172 | export const VENDORED_PREFIXES: readonly string[] = [ | |
| 173 | "node_modules/", | |
| 174 | "vendor/", | |
| 175 | "dist/", | |
| 176 | "build/", | |
| 177 | ".next/", | |
| 178 | ".nuxt/", | |
| 179 | "coverage/", | |
| 180 | ".git/", | |
| 181 | "target/", | |
| 182 | "bin/", | |
| 183 | ]; | |
| 184 | ||
| 185 | export const GENERATED_SUFFIXES: readonly string[] = [ | |
| 186 | "package-lock.json", | |
| 187 | "yarn.lock", | |
| 188 | "pnpm-lock.yaml", | |
| 189 | "bun.lockb", | |
| 190 | "Cargo.lock", | |
| 191 | "composer.lock", | |
| 192 | "Gemfile.lock", | |
| 193 | "poetry.lock", | |
| 194 | ]; | |
| 195 | ||
| 196 | export interface LanguageFileEntry { | |
| 197 | path: string; | |
| 198 | size: number; | |
| 199 | } | |
| 200 | ||
| 201 | export interface LanguageBucket { | |
| 202 | language: string; | |
| 203 | bytes: number; | |
| 204 | fileCount: number; | |
| 205 | percent: number; | |
| 206 | color: string; | |
| 207 | } | |
| 208 | ||
| 209 | export interface LanguageReport { | |
| 210 | totalBytes: number; | |
| 211 | totalFiles: number; | |
| 212 | countedFiles: number; | |
| 213 | /** Non-"Other" per-language rollup, sorted by bytes desc. */ | |
| 214 | buckets: LanguageBucket[]; | |
| 215 | primary: LanguageBucket | null; | |
| 216 | } | |
| 217 | ||
| 218 | function lastSegment(p: string): string { | |
| 219 | const i = p.lastIndexOf("/"); | |
| 220 | return i >= 0 ? p.slice(i + 1) : p; | |
| 221 | } | |
| 222 | ||
| 223 | function extensionOf(basename: string): string | null { | |
| 224 | const i = basename.lastIndexOf("."); | |
| 225 | if (i <= 0) return null; | |
| 226 | return basename.slice(i + 1).toLowerCase(); | |
| 227 | } | |
| 228 | ||
| 229 | /** | |
| 230 | * Return the language for a path, or null if it can't be classified. | |
| 231 | * Tries `FILENAME_MAP` on the basename first (for extensionless files | |
| 232 | * like `Dockerfile`), then `EXTENSION_MAP`. | |
| 233 | */ | |
| 234 | export function detectLanguage(path: string): string | null { | |
| 235 | if (typeof path !== "string" || path.length === 0) return null; | |
| 236 | const base = lastSegment(path).toLowerCase(); | |
| 237 | const byName = FILENAME_MAP.get(base); | |
| 238 | if (byName) return byName; | |
| 239 | const ext = extensionOf(base); | |
| 240 | if (!ext) return null; | |
| 241 | return EXTENSION_MAP.get(ext) ?? null; | |
| 242 | } | |
| 243 | ||
| 244 | export function isVendoredOrGenerated(path: string): boolean { | |
| 245 | if (typeof path !== "string") return false; | |
| 246 | const normal = path.startsWith("/") ? path.slice(1) : path; | |
| 247 | for (const pref of VENDORED_PREFIXES) { | |
| 248 | if (normal.startsWith(pref) || normal.includes("/" + pref)) return true; | |
| 249 | } | |
| 250 | const base = lastSegment(normal); | |
| 251 | for (const sfx of GENERATED_SUFFIXES) { | |
| 252 | if (base === sfx) return true; | |
| 253 | } | |
| 254 | return false; | |
| 255 | } | |
| 256 | ||
| 257 | export interface StatsOptions { | |
| 258 | /** Exclude `node_modules/`, `dist/`, lock files, etc. Default true. */ | |
| 259 | ignoreVendored?: boolean; | |
| 260 | /** Cap the per-file size we'll count (prevents a 500MB blob biasing the pie). */ | |
| 261 | maxFileSize?: number; | |
| 262 | /** Minimum bytes for a language to avoid being folded into "Other". Default 0 (= keep all). */ | |
| 263 | minLanguageBytes?: number; | |
| 264 | } | |
| 265 | ||
| 266 | function colorFor(lang: string): string { | |
| 267 | return LANGUAGE_COLORS.get(lang) ?? DEFAULT_LANGUAGE_COLOR; | |
| 268 | } | |
| 269 | ||
| 270 | /** | |
| 271 | * Walk the entries, bucket by detected language, compute percentages. | |
| 272 | * Paths that can't be classified are ignored (they don't count against | |
| 273 | * the total). Returns a sorted `buckets` array. | |
| 274 | */ | |
| 275 | export function computeLanguageStats( | |
| 276 | entries: readonly LanguageFileEntry[], | |
| 277 | opts: StatsOptions = {} | |
| 278 | ): LanguageReport { | |
| 279 | const ignoreVendored = opts.ignoreVendored !== false; | |
| 280 | const maxFileSize = | |
| 281 | opts.maxFileSize !== undefined && opts.maxFileSize > 0 | |
| 282 | ? opts.maxFileSize | |
| 283 | : Number.POSITIVE_INFINITY; | |
| 284 | const minLanguageBytes = opts.minLanguageBytes ?? 0; | |
| 285 | ||
| 286 | let totalFiles = 0; | |
| 287 | let countedFiles = 0; | |
| 288 | const byLang = new Map<string, { bytes: number; fileCount: number }>(); | |
| 289 | ||
| 290 | for (const e of entries) { | |
| 291 | totalFiles++; | |
| 292 | if (!e || typeof e.path !== "string") continue; | |
| 293 | if (ignoreVendored && isVendoredOrGenerated(e.path)) continue; | |
| 294 | const lang = detectLanguage(e.path); | |
| 295 | if (!lang) continue; | |
| 296 | const sz = Math.min( | |
| 297 | Math.max(0, Number.isFinite(e.size) ? e.size : 0), | |
| 298 | maxFileSize | |
| 299 | ); | |
| 300 | if (sz === 0) continue; | |
| 301 | countedFiles++; | |
| 302 | const agg = byLang.get(lang) ?? { bytes: 0, fileCount: 0 }; | |
| 303 | agg.bytes += sz; | |
| 304 | agg.fileCount++; | |
| 305 | byLang.set(lang, agg); | |
| 306 | } | |
| 307 | ||
| 308 | const totalBytes = Array.from(byLang.values()).reduce( | |
| 309 | (acc, v) => acc + v.bytes, | |
| 310 | 0 | |
| 311 | ); | |
| 312 | ||
| 313 | const buckets: LanguageBucket[] = []; | |
| 314 | let otherBytes = 0; | |
| 315 | let otherFiles = 0; | |
| 316 | for (const [language, { bytes, fileCount }] of byLang) { | |
| 317 | if (bytes < minLanguageBytes) { | |
| 318 | otherBytes += bytes; | |
| 319 | otherFiles += fileCount; | |
| 320 | continue; | |
| 321 | } | |
| 322 | buckets.push({ | |
| 323 | language, | |
| 324 | bytes, | |
| 325 | fileCount, | |
| 326 | percent: totalBytes === 0 ? 0 : (bytes / totalBytes) * 100, | |
| 327 | color: colorFor(language), | |
| 328 | }); | |
| 329 | } | |
| 330 | if (otherBytes > 0) { | |
| 331 | buckets.push({ | |
| 332 | language: "Other", | |
| 333 | bytes: otherBytes, | |
| 334 | fileCount: otherFiles, | |
| 335 | percent: totalBytes === 0 ? 0 : (otherBytes / totalBytes) * 100, | |
| 336 | color: colorFor("Other"), | |
| 337 | }); | |
| 338 | } | |
| 339 | ||
| 340 | buckets.sort((a, b) => { | |
| 341 | if (a.language === "Other" && b.language !== "Other") return 1; | |
| 342 | if (b.language === "Other" && a.language !== "Other") return -1; | |
| 343 | if (a.bytes !== b.bytes) return b.bytes - a.bytes; | |
| 344 | return a.language.localeCompare(b.language); | |
| 345 | }); | |
| 346 | ||
| 347 | const primary = buckets.find((b) => b.language !== "Other") ?? null; | |
| 348 | return { | |
| 349 | totalBytes, | |
| 350 | totalFiles, | |
| 351 | countedFiles, | |
| 352 | buckets, | |
| 353 | primary, | |
| 354 | }; | |
| 355 | } | |
| 356 | ||
| 357 | /** | |
| 358 | * Fold languages under `thresholdPercent` into an "Other" bucket. | |
| 359 | * Only rebucketing — re-running on the output is idempotent. | |
| 360 | */ | |
| 361 | export function foldIntoOther( | |
| 362 | report: LanguageReport, | |
| 363 | thresholdPercent: number | |
| 364 | ): LanguageReport { | |
| 365 | if (thresholdPercent <= 0) return report; | |
| 366 | const keep: LanguageBucket[] = []; | |
| 367 | let otherBytes = 0; | |
| 368 | let otherFiles = 0; | |
| 369 | for (const b of report.buckets) { | |
| 370 | if (b.language === "Other" || b.percent < thresholdPercent) { | |
| 371 | otherBytes += b.bytes; | |
| 372 | otherFiles += b.fileCount; | |
| 373 | } else { | |
| 374 | keep.push(b); | |
| 375 | } | |
| 376 | } | |
| 377 | if (otherBytes > 0) { | |
| 378 | keep.push({ | |
| 379 | language: "Other", | |
| 380 | bytes: otherBytes, | |
| 381 | fileCount: otherFiles, | |
| 382 | percent: | |
| 383 | report.totalBytes === 0 ? 0 : (otherBytes / report.totalBytes) * 100, | |
| 384 | color: colorFor("Other"), | |
| 385 | }); | |
| 386 | } | |
| 387 | keep.sort((a, b) => { | |
| 388 | if (a.language === "Other" && b.language !== "Other") return 1; | |
| 389 | if (b.language === "Other" && a.language !== "Other") return -1; | |
| 390 | if (a.bytes !== b.bytes) return b.bytes - a.bytes; | |
| 391 | return a.language.localeCompare(b.language); | |
| 392 | }); | |
| 393 | return { | |
| 394 | ...report, | |
| 395 | buckets: keep, | |
| 396 | primary: keep.find((b) => b.language !== "Other") ?? null, | |
| 397 | }; | |
| 398 | } | |
| 399 | ||
| 400 | const SIZE_UNITS = ["B", "KB", "MB", "GB", "TB"] as const; | |
| 401 | ||
| 402 | export function formatBytes(n: number): string { | |
| 403 | if (!Number.isFinite(n) || n < 0) return "0 B"; | |
| 404 | if (n < 1024) return `${n} B`; | |
| 405 | let v = n; | |
| 406 | let u = 0; | |
| 407 | while (v >= 1024 && u < SIZE_UNITS.length - 1) { | |
| 408 | v /= 1024; | |
| 409 | u++; | |
| 410 | } | |
| 411 | return `${v.toFixed(v >= 10 || u === 0 ? 0 : 1)} ${SIZE_UNITS[u]}`; | |
| 412 | } | |
| 413 | ||
| 414 | export function formatPercent(p: number, digits = 1): string { | |
| 415 | if (!Number.isFinite(p)) return "0%"; | |
| 416 | const v = Math.max(0, Math.min(100, p)); | |
| 417 | return `${v.toFixed(digits)}%`; | |
| 418 | } | |
| 419 | ||
| 420 | export interface BuildReportOptions extends StatsOptions { | |
| 421 | entries: readonly LanguageFileEntry[]; | |
| 422 | /** After rollup, fold any language under this percentage into Other. */ | |
| 423 | foldUnderPercent?: number; | |
| 424 | } | |
| 425 | ||
| 426 | export function buildLanguageReport(opts: BuildReportOptions): LanguageReport { | |
| 427 | const base = computeLanguageStats(opts.entries, opts); | |
| 428 | if (opts.foldUnderPercent && opts.foldUnderPercent > 0) { | |
| 429 | return foldIntoOther(base, opts.foldUnderPercent); | |
| 430 | } | |
| 431 | return base; | |
| 432 | } | |
| 433 | ||
| 434 | export const __internal = { | |
| 435 | EXTENSION_MAP, | |
| 436 | FILENAME_MAP, | |
| 437 | LANGUAGE_COLORS, | |
| 438 | DEFAULT_LANGUAGE_COLOR, | |
| 439 | VENDORED_PREFIXES, | |
| 440 | GENERATED_SUFFIXES, | |
| 441 | detectLanguage, | |
| 442 | isVendoredOrGenerated, | |
| 443 | computeLanguageStats, | |
| 444 | foldIntoOther, | |
| 445 | formatBytes, | |
| 446 | formatPercent, | |
| 447 | buildLanguageReport, | |
| 448 | colorFor, | |
| 449 | }; |