#!/usr/bin/env bash
# code-stats -- Codebase statistics across the workspace.
#
# Walks every nested .git repo under the workspace root and runs cloc to
# produce: total lines (code/comment/blank/files), per-repo breakdown, and
# per-language breakdown. Respects .gitignore by default.
#
# Usage:
# code-stats # workspace-wide (default)
# code-stats --repo PATH # single repo
# code-stats --root PATH # workspace rooted at PATH
# code-stats --by-language # only show language table
# code-stats --by-repo # only show repo table
# code-stats --all-files # ignore .gitignore (count vendored too)
# code-stats --top N # show top N rows in tables (default: 25)
# code-stats --json # emit machine-readable JSON
# code-stats --no-umbrella # skip WORKSPACE-VM (workspace-root repo)
#
# Exit codes:
# 0 ok
# 1 cloc not found / invalid args
# 2 no repos found
set -euo pipefail
_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=../lib/ci.sh
_ci_lib="$_SCRIPT_DIR/../lib/ci.sh"
if ! source "$_ci_lib"; then
echo "ERROR: failed to source $_ci_lib" >&2
exit 2
fi
# ---------------------------------------------------------------------------
# Locate cloc: prefer CI's own .boot-linux (CI owns the cloc bootstrap), then
# the workspace root's boot dir, then .boot-macos, then fall back to PATH.
# ---------------------------------------------------------------------------
_find_cloc() {
local _cands=(
"${CI_PROJECT_ROOT:-}/.boot-linux/bin/cloc"
"$CI_WORKSPACE_ROOT/.boot-linux/bin/cloc"
"$CI_WORKSPACE_ROOT/.boot-macos/bin/cloc"
)
for _c in "${_cands[@]}"; do
[[ -x "$_c" ]] && { echo "$_c"; return 0; }
done
local _cloc_path=""
if _cloc_path="$(command -v cloc 2>&1)"; then
echo "$_cloc_path"
return 0
fi
return 1
}
# Resolve cloc; empty string if not found. _find_cloc returns 0/1 only
# (writes path to stdout on success, no output on failure); no stderr
# normally. Command substitution's exit status IS the inner command's,
# so `if CLOC="$(...)"` is rc-capture: enters the if-branch on rc=0.
CLOC=""
if CLOC="$(_find_cloc)"; then
:
else
CLOC=""
fi
if [[ -z "$CLOC" ]]; then
ci_fail "cloc not found. Install via: bash scripts/bootstrap-cloc (or: make install-cloc)"
exit 1
fi
# ---------------------------------------------------------------------------
# Arg parsing
# ---------------------------------------------------------------------------
_mode="all" # all | by-language | by-repo
_root="$CI_WORKSPACE_ROOT"
_single_repo=""
_respect_gitignore=1
_top_n=25
_json=0
_skip_umbrella=0
while [[ $# -gt 0 ]]; do
case "$1" in
--repo) _single_repo="$2"; shift 2 ;;
--root) _root="$2"; shift 2 ;;
--by-language) _mode="by-language"; shift ;;
--by-repo) _mode="by-repo"; shift ;;
--all-files) _respect_gitignore=0; shift ;;
--top) _top_n="$2"; shift 2 ;;
--json) _json=1; shift ;;
--no-umbrella) _skip_umbrella=1; shift ;;
-h|--help) sed -n '2,20p' "$0" | sed 's/^# \?//'; exit 0 ;;
*) ci_fail "unknown arg: $1"; exit 1 ;;
esac
done
_root="$(cd "$_root" && pwd)"
_tmpdir="$(mktemp -d)"
trap 'rm -rf "$_tmpdir"' EXIT
# Skip patterns for nested repo discovery (mirrors audit-workspace).
_SKIP_DIRS="tmp|node_modules|\.venv|__pycache__|research|python-ta-reference|\.boot-linux|\.boot-macos|dist|build|target|\.cache|\.local"
# ---------------------------------------------------------------------------
# Repo discovery
# ---------------------------------------------------------------------------
_repos=()
if [[ -n "$_single_repo" ]]; then
_single_repo="$(cd "$_single_repo" && pwd)"
_repos+=("$_single_repo")
else
if [[ "$_skip_umbrella" -eq 0 && -d "$_root/.git" ]]; then
_repos+=("$_root")
fi
_gitdirs=()
ci_capture_pipe _gitdirs 'find "$1" -name ".git" -type d | sort' "$_root"
for _gitdir in "${_gitdirs[@]}"; do
_repo="$(dirname "$_gitdir")"
[[ "$_repo" == "$_root" ]] && continue
# Prod wiki stages README/Makefile under projects/WORKSPACE-VM; not a real repo.
[[ "$_repo" == "$_root/projects/WORKSPACE-VM" ]] && continue
echo "$_repo" | grep -qE "$_SKIP_DIRS" && continue
_repos+=("$_repo")
done
fi
if [[ ${#_repos[@]} -eq 0 ]]; then
ci_warn "No git repos found under $_root"
exit 2
fi
# ---------------------------------------------------------------------------
# Run cloc per repo, collect results
# ---------------------------------------------------------------------------
# Per-repo aggregate file: tab-separated columns
# rel_path files blank comment code
_per_repo="$_tmpdir/per_repo.tsv"
# Per-(repo, language) detail file: tab-separated columns
# rel_path language files blank comment code
_per_repo_lang="$_tmpdir/per_repo_lang.tsv"
# Track failures to distinguish "0 lines" from "skipped".
_failed_repos=()
if [[ $_json -eq 1 ]]; then
printf '[code-stats] scanning %s repo(s) under %s...\n' \
"${#_repos[@]}" "${_root#$HOME/~}" >&2
else
ci_info "Scanning ${#_repos[@]} repo(s) under ${_root#$HOME/~}..."
fi
for _repo in "${_repos[@]}"; do
_is_umbrella=0
if [[ "$_repo" == "$_root" ]]; then
_is_umbrella=1
_rel="WORKSPACE-VM"
else
_rel="${_repo#"$_root"/}"
[[ "$_rel" == projects/* ]] && _rel="${_rel#projects/}"
fi
if [[ $_json -eq 1 ]]; then
printf '[code-stats] cloc: %s (running; large repos may take several minutes)...\n' \
"$_rel" >&2
fi
# No --quiet: cloc progress goes to stderr (visible during long repos).
_cloc_args=(--json --not-match-f='package-lock\.json')
# Always use --exclude-dir instead of --vcs=git because the workspace
# guard blocks ``git -c safe.directory=* ls-files`` (the command cloc
# invokes internally under --vcs=git). The expanded exclude list covers
# the same heavy/vendored directories that .gitignore would filter.
if [[ $_respect_gitignore -eq 1 ]]; then
_cloc_args+=(
--exclude-dir=node_modules,.venv,__pycache__,target,dist,build,.cache,.local,.boot-linux,.boot-macos,.git,.next,.turbo,coverage,res
)
else
_cloc_args+=(
--exclude-dir=node_modules,.venv,__pycache__,target,dist,build,.cache,.local,.boot-linux,.boot-macos,.git,.next,.turbo,coverage
)
fi
_out="$_tmpdir/$(echo "$_rel" | tr '/' '_').json"
_co_rc=0
_co_out="$(mktemp)"; _co_err="$(mktemp)"
if [[ $_is_umbrella -eq 1 ]]; then
# Umbrella tracks only root-level files; nested projects/ repos are separate .git trees.
_filelist="$_tmpdir/umbrella_files.txt"
git -C "$_repo" ls-files >"$_filelist" 2>"$_co_err" || _co_rc=$?
if [[ $_co_rc -eq 0 && -s "$_filelist" ]]; then
( cd "$_repo" && "$CLOC" "${_cloc_args[@]}" --list-file="$_filelist" --report-file="$_out" . ) \
>"$_co_out" 2>"$_co_err" || _co_rc=$?
elif [[ $_co_rc -eq 0 ]]; then
_co_rc=1
printf 'no tracked files\n' >"$_co_err"
fi
else
( cd "$_repo" && "$CLOC" "${_cloc_args[@]}" --report-file="$_out" . ) \
>"$_co_out" 2>"$_co_err" || _co_rc=$?
fi
if [[ $_co_rc -ne 0 ]]; then
if [[ -s "$_co_err" ]]; then
printf '[code-stats] cloc failed on %s (rc=%d): %s\n' \
"$_rel" "$_co_rc" "$(head -1 "$_co_err")" >&2
else
printf '[code-stats] cloc failed on %s (rc=%d, no stderr)\n' \
"$_rel" "$_co_rc" >&2
fi
rm -f "$_co_out" "$_co_err"
_failed_repos+=("$_rel")
continue
fi
rm -f "$_co_out" "$_co_err"
[[ ! -s "$_out" ]] && continue
# Parse JSON output. cloc's JSON shape:
# { "header": {...}, "<Lang>": {nFiles, blank, comment, code}, ..., "SUM": {...} }
# Use a small Python helper for robustness (jq isn't always present).
_py_rc=0
uv run python - "$_out" "$_rel" "$_per_repo" "$_per_repo_lang" <<'PY' || _py_rc=$?
import json, sys, os
src, rel, per_repo, per_lang = sys.argv[1:]
with open(src) as f:
data = json.load(f)
total = data.pop("SUM", None)
data.pop("header", None)
if total:
with open(per_repo, "a") as fh:
fh.write(f"{rel}\t{total['nFiles']}\t{total['blank']}\t{total['comment']}\t{total['code']}\n")
with open(per_lang, "a") as fh:
for lang, vals in data.items():
if not isinstance(vals, dict) or "code" not in vals:
continue
fh.write(f"{rel}\t{lang}\t{vals['nFiles']}\t{vals['blank']}\t{vals['comment']}\t{vals['code']}\n")
PY
done
if [[ ${#_failed_repos[@]} -gt 0 && $_json -eq 0 ]]; then
ci_warn "cloc failed on ${#_failed_repos[@]} repo(s): ${_failed_repos[*]}"
fi
# ---------------------------------------------------------------------------
# Render
# ---------------------------------------------------------------------------
# Use Python for table formatting: awk's number formatting + sort across
# multiple keys gets ugly fast and we already depend on python above.
uv run python - "$_per_repo" "$_per_repo_lang" "$_mode" "$_top_n" "$_json" "$_root" <<'PY'
import sys, json, os
per_repo_path, per_lang_path, mode, top_n, as_json, root = sys.argv[1:]
top_n = int(top_n)
as_json = as_json == "1"
def read_tsv(path, cols):
rows = []
if not os.path.exists(path):
return rows
with open(path) as f:
for line in f:
line = line.rstrip("\n")
if not line:
continue
parts = line.split("\t")
if len(parts) != len(cols):
continue
row = {}
for c, v in zip(cols, parts):
row[c] = int(v) if c in {"files","blank","comment","code"} else v
rows.append(row)
return rows
repos = read_tsv(per_repo_path, ["repo","files","blank","comment","code"])
langs_raw = read_tsv(per_lang_path, ["repo","language","files","blank","comment","code"])
# Aggregate languages across all repos.
agg = {}
for r in langs_raw:
a = agg.setdefault(r["language"], {"files":0,"blank":0,"comment":0,"code":0,"repos":set()})
a["files"] += r["files"]; a["blank"] += r["blank"]
a["comment"] += r["comment"]; a["code"] += r["code"]
a["repos"].add(r["repo"])
totals = {
"repos": len(repos),
"files": sum(r["files"] for r in repos),
"blank": sum(r["blank"] for r in repos),
"comment": sum(r["comment"] for r in repos),
"code": sum(r["code"] for r in repos),
}
totals["lines"] = totals["blank"] + totals["comment"] + totals["code"]
if as_json:
# Per-repo language breakdown for wiki card badges.
repo_langs = []
for r in langs_raw:
repo_langs.append({
"repo": r["repo"],
"language": r["language"],
"files": r["files"],
"blank": r["blank"],
"comment": r["comment"],
"code": r["code"],
})
out = {
"root": root,
"totals": totals,
"repos": sorted(repos, key=lambda x: -x["code"]),
"languages": sorted(
[{"language":k, **{kk:vv for kk,vv in v.items() if kk!="repos"},
"repos": len(v["repos"])} for k,v in agg.items()],
key=lambda x: -x["code"],
),
"repo_languages": repo_langs,
}
print(json.dumps(out, indent=2))
sys.exit(0)
def fmt(n): return f"{n:,}"
BOLD = "\033[1m"; CYAN = "\033[0;36m"; GREEN = "\033[0;32m"
DIM = "\033[2m"; NC = "\033[0m"
if not sys.stdout.isatty():
BOLD = CYAN = GREEN = DIM = NC = ""
def hr(ch="="): print(ch * 78)
# -- Summary --
hr("=")
print(f"{BOLD}CODE STATISTICS{NC} root: {root}")
hr("=")
print(f" Repositories scanned : {fmt(totals['repos'])}")
print(f" Files : {fmt(totals['files'])}")
print(f" Code lines : {GREEN}{fmt(totals['code'])}{NC}")
print(f" Comment lines : {fmt(totals['comment'])}")
print(f" Blank lines : {fmt(totals['blank'])}")
print(f" Total lines : {BOLD}{fmt(totals['lines'])}{NC}")
print()
def print_repo_table():
if not repos: return
print(f"{BOLD}Per-repository (top {top_n} by code lines){NC}")
rows = sorted(repos, key=lambda x: -x["code"])[:top_n]
name_w = max(len(r["repo"]) for r in rows)
name_w = min(max(name_w, 24), 60)
print(f" {'REPO':<{name_w}} {'FILES':>7} {'CODE':>12} {'COMMENT':>10} {'BLANK':>10}")
print(f" {'-'*name_w} {'-'*7} {'-'*12} {'-'*10} {'-'*10}")
for r in rows:
nm = r["repo"]
if len(nm) > name_w: nm = "..." + nm[-(name_w-3):]
print(f" {nm:<{name_w}} {fmt(r['files']):>7} {fmt(r['code']):>12} {fmt(r['comment']):>10} {fmt(r['blank']):>10}")
if len(repos) > top_n:
print(f" {DIM}... {len(repos)-top_n} more repo(s) omitted (use --top to show more){NC}")
print()
def print_lang_table():
if not agg: return
print(f"{BOLD}Per-language (top {top_n} by code lines){NC}")
rows = sorted(agg.items(), key=lambda kv: -kv[1]["code"])[:top_n]
name_w = max(len(k) for k,_ in rows)
name_w = max(name_w, 16)
print(f" {'LANGUAGE':<{name_w}} {'REPOS':>5} {'FILES':>7} {'CODE':>12} {'COMMENT':>10} {'BLANK':>10}")
print(f" {'-'*name_w} {'-'*5} {'-'*7} {'-'*12} {'-'*10} {'-'*10}")
for lang, v in rows:
print(f" {lang:<{name_w}} {len(v['repos']):>5} {fmt(v['files']):>7} {fmt(v['code']):>12} {fmt(v['comment']):>10} {fmt(v['blank']):>10}")
if len(agg) > top_n:
print(f" {DIM}... {len(agg)-top_n} more language(s) omitted{NC}")
print()
if mode in ("all","by-repo"):
print_repo_table()
if mode in ("all","by-language"):
print_lang_table()
PY