| 1 | #!/usr/bin/env python3 |
| 2 | """Offline tests for scripts/catalog_models_dev.py (#4117).""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import json |
| 7 | import os |
| 8 | import subprocess |
| 9 | import sys |
| 10 | import tempfile |
| 11 | import unittest |
| 12 | from pathlib import Path |
| 13 | |
| 14 | ROOT = Path(__file__).resolve().parents[1] |
| 15 | SCRIPT = ROOT / "scripts" / "catalog_models_dev.py" |
| 16 | SEED = ROOT / "crates" / "config" / "assets" / "models_dev.bundled.json" |
| 17 | |
| 18 | |
| 19 | class CatalogModelsDevScriptTests(unittest.TestCase): |
| 20 | def test_snapshot_check_validates_offline_seed(self) -> None: |
| 21 | proc = subprocess.run( |
| 22 | [sys.executable, str(SCRIPT), "snapshot", "--check", str(SEED)], |
| 23 | cwd=ROOT, |
| 24 | capture_output=True, |
| 25 | text=True, |
| 26 | check=False, |
| 27 | ) |
| 28 | self.assertEqual(proc.returncode, 0, proc.stderr) |
| 29 | self.assertIn("ok:", proc.stdout) |
| 30 | self.assertIn("providers=", proc.stdout) |
| 31 | |
| 32 | def test_scrub_drops_api_key_fields(self) -> None: |
| 33 | # Import helpers without network. |
| 34 | sys.path.insert(0, str(ROOT / "scripts")) |
| 35 | import catalog_models_dev as mod # type: ignore |
| 36 | |
| 37 | dirty = { |
| 38 | "models": {}, |
| 39 | "providers": { |
| 40 | "deepseek": { |
| 41 | "api_key": "sk-should-never-persist", |
| 42 | "models": {"deepseek-v4-pro": {"id": "deepseek-v4-pro"}}, |
| 43 | } |
| 44 | }, |
| 45 | "token": "nope", |
| 46 | } |
| 47 | clean = mod.scrub_secrets(dirty) |
| 48 | self.assertNotIn("token", clean) |
| 49 | self.assertNotIn("api_key", clean["providers"]["deepseek"]) |
| 50 | self.assertIn("models", clean["providers"]["deepseek"]) |
| 51 | |
| 52 | def test_ensure_shape_rejects_empty_object(self) -> None: |
| 53 | sys.path.insert(0, str(ROOT / "scripts")) |
| 54 | import catalog_models_dev as mod # type: ignore |
| 55 | |
| 56 | with self.assertRaises(SystemExit): |
| 57 | mod.ensure_models_dev_shape({}, "test") |
| 58 | |
| 59 | def test_public_document_drops_api_key(self) -> None: |
| 60 | sys.path.insert(0, str(ROOT / "scripts")) |
| 61 | import catalog_models_dev as mod # type: ignore |
| 62 | |
| 63 | dirty = { |
| 64 | "models": {}, |
| 65 | "providers": {"deepseek": {"api_key": "sk-x", "models": {}}}, |
| 66 | "token": "nope", |
| 67 | } |
| 68 | clean = mod.public_models_dev_document(dirty) |
| 69 | self.assertNotIn("token", clean) |
| 70 | self.assertNotIn("api_key", clean["providers"]["deepseek"]) |
| 71 | |
| 72 | def test_refresh_write_cache_is_rejected_without_writing(self) -> None: |
| 73 | with tempfile.TemporaryDirectory() as td: |
| 74 | source = Path(td) / "catalog.json" |
| 75 | target = Path(td) / "cache.json" |
| 76 | source.write_text( |
| 77 | json.dumps({"models": {}, "providers": {}, "api_key": "sk-nope"}), |
| 78 | encoding="utf-8", |
| 79 | ) |
| 80 | env = os.environ.copy() |
| 81 | env["CODEWHALE_MODELS_DEV_PATH"] = str(source) |
| 82 | |
| 83 | proc = subprocess.run( |
| 84 | [ |
| 85 | sys.executable, |
| 86 | str(SCRIPT), |
| 87 | "refresh", |
| 88 | "--write-cache", |
| 89 | str(target), |
| 90 | ], |
| 91 | cwd=ROOT, |
| 92 | capture_output=True, |
| 93 | text=True, |
| 94 | check=False, |
| 95 | env=env, |
| 96 | ) |
| 97 | |
| 98 | self.assertNotEqual(proc.returncode, 0) |
| 99 | self.assertIn("disk writes are intentionally unsupported", proc.stderr) |
| 100 | self.assertFalse(target.exists(), "refresh must remain dry-run only") |
| 101 | |
| 102 | |
| 103 | if __name__ == "__main__": |
| 104 | unittest.main() |
| 105 |