| 1 | #!/usr/bin/env python3 |
| 2 | """Enforce one-way ceilings on Codewhale's source ownership structure. |
| 3 | |
| 4 | The budget permits deletion and line-neutral ownership moves freely. Adding a |
| 5 | workspace package or binary, creating a new thousand-line production Rust |
| 6 | module, increasing the largest owned file, or growing aggregate owned Rust |
| 7 | source requires an explicit budget update and review. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import argparse |
| 13 | import json |
| 14 | import os |
| 15 | import stat |
| 16 | import subprocess |
| 17 | import sys |
| 18 | import tempfile |
| 19 | from dataclasses import dataclass |
| 20 | from pathlib import Path |
| 21 | from typing import Any, Sequence |
| 22 | |
| 23 | REPO_ROOT = Path(__file__).resolve().parent.parent |
| 24 | BUDGET_PATH = REPO_ROOT / "scripts" / "source-structure-budget.json" |
| 25 | DOCUMENT_KIND = "codewhale.source_structure_budget" |
| 26 | SCHEMA_VERSION = 1 |
| 27 | LARGE_MODULE_THRESHOLD = 1_000 |
| 28 | |
| 29 | |
| 30 | class StructureBudgetError(ValueError): |
| 31 | """The budget or measured source structure is invalid.""" |
| 32 | |
| 33 | |
| 34 | @dataclass(frozen=True) |
| 35 | class StructureSnapshot: |
| 36 | workspace_packages: tuple[str, ...] |
| 37 | binary_targets: tuple[str, ...] |
| 38 | module_lines: dict[str, int] |
| 39 | |
| 40 | |
| 41 | @dataclass(frozen=True) |
| 42 | class StructureBudget: |
| 43 | workspace_packages: tuple[str, ...] |
| 44 | binary_targets: tuple[str, ...] |
| 45 | allowed_large_modules: tuple[str, ...] |
| 46 | max_large_module_count: int |
| 47 | max_module_lines: int |
| 48 | max_total_owned_rust_lines: int |
| 49 | |
| 50 | |
| 51 | def production_rust_files(root: Path) -> list[Path]: |
| 52 | files = [] |
| 53 | for path in (root / "crates").glob("*/src/**/*.rs"): |
| 54 | relative = path.relative_to(root) |
| 55 | if path.name == "tests.rs" or "tests" in relative.parts: |
| 56 | continue |
| 57 | files.append(path) |
| 58 | return sorted(files) |
| 59 | |
| 60 | |
| 61 | def source_line_count(path: Path) -> int: |
| 62 | return len(path.read_bytes().splitlines()) |
| 63 | |
| 64 | |
| 65 | def cargo_metadata(root: Path) -> dict[str, Any]: |
| 66 | environment = os.environ.copy() |
| 67 | environment["CARGO_NET_OFFLINE"] = "true" |
| 68 | process = subprocess.run( |
| 69 | [ |
| 70 | "cargo", |
| 71 | "metadata", |
| 72 | "--offline", |
| 73 | "--locked", |
| 74 | "--no-deps", |
| 75 | "--format-version", |
| 76 | "1", |
| 77 | ], |
| 78 | cwd=root, |
| 79 | env=environment, |
| 80 | capture_output=True, |
| 81 | text=True, |
| 82 | check=False, |
| 83 | ) |
| 84 | if process.returncode != 0: |
| 85 | sys.stderr.write(process.stderr) |
| 86 | raise StructureBudgetError( |
| 87 | f"cargo metadata failed with exit code {process.returncode}" |
| 88 | ) |
| 89 | try: |
| 90 | document = json.loads(process.stdout) |
| 91 | except json.JSONDecodeError as error: |
| 92 | raise StructureBudgetError(f"cargo metadata emitted invalid JSON: {error}") from error |
| 93 | if not isinstance(document, dict) or not isinstance(document.get("packages"), list): |
| 94 | raise StructureBudgetError("cargo metadata did not contain a package list") |
| 95 | return document |
| 96 | |
| 97 | |
| 98 | def measure(root: Path = REPO_ROOT) -> StructureSnapshot: |
| 99 | metadata = cargo_metadata(root) |
| 100 | packages = tuple(sorted(package["name"] for package in metadata["packages"])) |
| 101 | binaries = tuple( |
| 102 | sorted( |
| 103 | f"{package['name']}:{target['name']}" |
| 104 | for package in metadata["packages"] |
| 105 | for target in package.get("targets", []) |
| 106 | if "bin" in target.get("kind", []) |
| 107 | ) |
| 108 | ) |
| 109 | module_lines = { |
| 110 | path.relative_to(root).as_posix(): source_line_count(path) |
| 111 | for path in production_rust_files(root) |
| 112 | } |
| 113 | return StructureSnapshot(packages, binaries, module_lines) |
| 114 | |
| 115 | |
| 116 | def require_string_list(document: dict[str, Any], field: str) -> tuple[str, ...]: |
| 117 | value = document.get(field) |
| 118 | if ( |
| 119 | not isinstance(value, list) |
| 120 | or any(not isinstance(item, str) or not item for item in value) |
| 121 | or value != sorted(set(value)) |
| 122 | ): |
| 123 | raise StructureBudgetError(f"`{field}` must be sorted unique non-empty strings") |
| 124 | return tuple(value) |
| 125 | |
| 126 | |
| 127 | def require_non_negative_int(document: dict[str, Any], field: str) -> int: |
| 128 | value = document.get(field) |
| 129 | if isinstance(value, bool) or not isinstance(value, int) or value < 0: |
| 130 | raise StructureBudgetError(f"`{field}` must be a non-negative integer") |
| 131 | return value |
| 132 | |
| 133 | |
| 134 | def validate_budget(document: dict[str, Any]) -> StructureBudget: |
| 135 | if document.get("document_kind") != DOCUMENT_KIND: |
| 136 | raise StructureBudgetError(f"document_kind must be `{DOCUMENT_KIND}`") |
| 137 | schema_version = document.get("schema_version") |
| 138 | if isinstance(schema_version, bool) or schema_version != SCHEMA_VERSION: |
| 139 | raise StructureBudgetError(f"schema_version must be {SCHEMA_VERSION}") |
| 140 | threshold = document.get("large_module_threshold_lines") |
| 141 | if isinstance(threshold, bool) or threshold != LARGE_MODULE_THRESHOLD: |
| 142 | raise StructureBudgetError( |
| 143 | f"large_module_threshold_lines must be {LARGE_MODULE_THRESHOLD}" |
| 144 | ) |
| 145 | allowed_large_modules = require_string_list(document, "allowed_large_modules") |
| 146 | for path in allowed_large_modules: |
| 147 | if ( |
| 148 | not path.startswith("crates/") |
| 149 | or Path(path).is_absolute() |
| 150 | or ".." in Path(path).parts |
| 151 | ): |
| 152 | raise StructureBudgetError(f"invalid large-module path: {path!r}") |
| 153 | return StructureBudget( |
| 154 | require_string_list(document, "workspace_packages"), |
| 155 | require_string_list(document, "binary_targets"), |
| 156 | allowed_large_modules, |
| 157 | require_non_negative_int(document, "max_large_module_count"), |
| 158 | require_non_negative_int(document, "max_module_lines"), |
| 159 | require_non_negative_int(document, "max_total_owned_rust_lines"), |
| 160 | ) |
| 161 | |
| 162 | |
| 163 | def load_budget(path: Path) -> tuple[dict[str, Any], StructureBudget]: |
| 164 | try: |
| 165 | document = json.loads(path.read_text(encoding="utf-8")) |
| 166 | except FileNotFoundError as error: |
| 167 | raise StructureBudgetError(f"missing source-structure budget: {path}") from error |
| 168 | except (OSError, json.JSONDecodeError) as error: |
| 169 | raise StructureBudgetError(f"invalid source-structure budget {path}: {error}") from error |
| 170 | if not isinstance(document, dict): |
| 171 | raise StructureBudgetError("source-structure budget must be a JSON object") |
| 172 | return document, validate_budget(document) |
| 173 | |
| 174 | |
| 175 | def compare(current: StructureSnapshot, budget: StructureBudget) -> tuple[list[str], list[str]]: |
| 176 | failures: list[str] = [] |
| 177 | improvements: list[str] = [] |
| 178 | added_packages = sorted(set(current.workspace_packages) - set(budget.workspace_packages)) |
| 179 | if added_packages: |
| 180 | failures.append(f"workspace packages added: {', '.join(added_packages)}") |
| 181 | removed_packages = sorted(set(budget.workspace_packages) - set(current.workspace_packages)) |
| 182 | if removed_packages: |
| 183 | improvements.append(f"workspace packages removed: {', '.join(removed_packages)}") |
| 184 | |
| 185 | added_binaries = sorted(set(current.binary_targets) - set(budget.binary_targets)) |
| 186 | if added_binaries: |
| 187 | failures.append(f"binary targets added: {', '.join(added_binaries)}") |
| 188 | removed_binaries = sorted(set(budget.binary_targets) - set(current.binary_targets)) |
| 189 | if removed_binaries: |
| 190 | improvements.append(f"binary targets removed: {', '.join(removed_binaries)}") |
| 191 | |
| 192 | current_large = { |
| 193 | path for path, lines in current.module_lines.items() if lines >= LARGE_MODULE_THRESHOLD |
| 194 | } |
| 195 | allowed_large = set(budget.allowed_large_modules) |
| 196 | for path in sorted(current_large - allowed_large): |
| 197 | failures.append( |
| 198 | f"new thousand-line production module: {path} has " |
| 199 | f"{current.module_lines[path]} lines" |
| 200 | ) |
| 201 | removed_large = sorted(allowed_large - current_large) |
| 202 | if removed_large: |
| 203 | improvements.append( |
| 204 | "large modules removed or split below " |
| 205 | f"{LARGE_MODULE_THRESHOLD}: {', '.join(removed_large)}" |
| 206 | ) |
| 207 | |
| 208 | large_count = len(current_large) |
| 209 | if large_count > budget.max_large_module_count: |
| 210 | failures.append( |
| 211 | f"large-module count grew: {large_count} > {budget.max_large_module_count}" |
| 212 | ) |
| 213 | elif large_count < budget.max_large_module_count: |
| 214 | improvements.append( |
| 215 | f"large-module count shrank: {large_count} < {budget.max_large_module_count}" |
| 216 | ) |
| 217 | |
| 218 | largest = max(current.module_lines.values(), default=0) |
| 219 | if largest > budget.max_module_lines: |
| 220 | failures.append(f"largest module grew: {largest} > {budget.max_module_lines} lines") |
| 221 | elif largest < budget.max_module_lines: |
| 222 | improvements.append(f"largest module shrank: {largest} < {budget.max_module_lines} lines") |
| 223 | |
| 224 | total = sum(current.module_lines.values()) |
| 225 | if total > budget.max_total_owned_rust_lines: |
| 226 | failures.append( |
| 227 | "aggregate owned Rust source grew: " |
| 228 | f"{total} > {budget.max_total_owned_rust_lines} lines" |
| 229 | ) |
| 230 | elif total < budget.max_total_owned_rust_lines: |
| 231 | improvements.append( |
| 232 | "aggregate owned Rust source shrank: " |
| 233 | f"{total} < {budget.max_total_owned_rust_lines} lines" |
| 234 | ) |
| 235 | return failures, improvements |
| 236 | |
| 237 | |
| 238 | def budget_document(snapshot: StructureSnapshot) -> dict[str, Any]: |
| 239 | large_modules = sorted( |
| 240 | path for path, lines in snapshot.module_lines.items() if lines >= LARGE_MODULE_THRESHOLD |
| 241 | ) |
| 242 | return { |
| 243 | "_comment": ( |
| 244 | "One-way source ownership ceilings. Deletion and line-neutral ownership moves " |
| 245 | "pass; new packages, binaries, 1000-line module paths, a larger maximum module, " |
| 246 | "or aggregate owned Rust growth require an explicit reviewed update." |
| 247 | ), |
| 248 | "allowed_large_modules": large_modules, |
| 249 | "binary_targets": list(snapshot.binary_targets), |
| 250 | "document_kind": DOCUMENT_KIND, |
| 251 | "large_module_threshold_lines": LARGE_MODULE_THRESHOLD, |
| 252 | "max_large_module_count": len(large_modules), |
| 253 | "max_module_lines": max(snapshot.module_lines.values(), default=0), |
| 254 | "max_total_owned_rust_lines": sum(snapshot.module_lines.values()), |
| 255 | "schema_version": SCHEMA_VERSION, |
| 256 | "workspace_packages": list(snapshot.workspace_packages), |
| 257 | } |
| 258 | |
| 259 | |
| 260 | def write_json_atomic(path: Path, document: dict[str, Any]) -> None: |
| 261 | mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else 0o644 |
| 262 | path.parent.mkdir(parents=True, exist_ok=True) |
| 263 | descriptor, temporary_name = tempfile.mkstemp( |
| 264 | prefix=f".{path.name}.", suffix=".tmp", dir=path.parent |
| 265 | ) |
| 266 | temporary = Path(temporary_name) |
| 267 | try: |
| 268 | with os.fdopen(descriptor, "w", encoding="utf-8") as handle: |
| 269 | json.dump(document, handle, indent=2, sort_keys=True) |
| 270 | handle.write("\n") |
| 271 | handle.flush() |
| 272 | os.fsync(handle.fileno()) |
| 273 | os.chmod(temporary, mode) |
| 274 | os.replace(temporary, path) |
| 275 | except BaseException: |
| 276 | temporary.unlink(missing_ok=True) |
| 277 | raise |
| 278 | |
| 279 | |
| 280 | def main(argv: Sequence[str] | None = None) -> int: |
| 281 | parser = argparse.ArgumentParser(description=__doc__) |
| 282 | parser.add_argument("--budget", type=Path, default=BUDGET_PATH, help=argparse.SUPPRESS) |
| 283 | parser.add_argument("--update", action="store_true", help="lock in current improvements") |
| 284 | args = parser.parse_args(argv) |
| 285 | |
| 286 | try: |
| 287 | current = measure() |
| 288 | if args.update and not args.budget.exists(): |
| 289 | write_json_atomic(args.budget, budget_document(current)) |
| 290 | print(f"[source-structure-budget] initialized {args.budget}") |
| 291 | return 0 |
| 292 | _document, budget = load_budget(args.budget) |
| 293 | failures, improvements = compare(current, budget) |
| 294 | except (OSError, StructureBudgetError) as error: |
| 295 | print(f"[source-structure-budget] ERROR: {error}", file=sys.stderr) |
| 296 | return 2 |
| 297 | |
| 298 | if failures: |
| 299 | print("[source-structure-budget] FAIL:", file=sys.stderr) |
| 300 | for failure in failures: |
| 301 | print(f" {failure}", file=sys.stderr) |
| 302 | return 1 |
| 303 | if args.update: |
| 304 | try: |
| 305 | write_json_atomic(args.budget, budget_document(current)) |
| 306 | except OSError as error: |
| 307 | print(f"[source-structure-budget] ERROR: {error}", file=sys.stderr) |
| 308 | return 2 |
| 309 | print( |
| 310 | f"[source-structure-budget] tightened {args.budget}: " |
| 311 | f"{len(improvements)} improvement(s) locked in" |
| 312 | ) |
| 313 | return 0 |
| 314 | print( |
| 315 | "[source-structure-budget] PASS: " |
| 316 | f"{len(current.workspace_packages)} packages, " |
| 317 | f"{len(current.binary_targets)} binaries, " |
| 318 | f"{sum(lines >= LARGE_MODULE_THRESHOLD for lines in current.module_lines.values())} " |
| 319 | "large owned modules, " |
| 320 | f"{sum(current.module_lines.values())} owned Rust lines" |
| 321 | ) |
| 322 | for improvement in improvements: |
| 323 | print(f" can tighten: {improvement}") |
| 324 | return 0 |
| 325 | |
| 326 | |
| 327 | if __name__ == "__main__": |
| 328 | raise SystemExit(main()) |
| 329 |