| 1 | #!/usr/bin/env python3 |
| 2 | """Offline tests for scripts/catalog_models_dev.py (#4117).""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import json |
| 7 | import os |
| 8 | import subprocess |
| 9 | import sys |
| 10 | import tempfile |
| 11 | import unittest |
| 12 | from pathlib import Path |
| 13 | |
| 14 | ROOT = Path(__file__).resolve().parents[1] |
| 15 | SCRIPT = ROOT / "scripts" / "catalog_models_dev.py" |
| 16 | SEED = ROOT / "crates" / "config" / "assets" / "models_dev.bundled.json" |
| 17 | |
| 18 | |
| 19 | class CatalogModelsDevScriptTests(unittest.TestCase): |
| 20 | def test_snapshot_check_validates_offline_seed(self) -> None: |
| 21 | proc = subprocess.run( |
| 22 | [sys.executable, str(SCRIPT), "snapshot", "--check", str(SEED)], |
| 23 | cwd=ROOT, |
| 24 | capture_output=True, |
| 25 | text=True, |
| 26 | check=False, |
| 27 | ) |
| 28 | self.assertEqual(proc.returncode, 0, proc.stderr) |
| 29 | self.assertIn("ok:", proc.stdout) |
| 30 | self.assertIn("providers=", proc.stdout) |
| 31 | |
| 32 | def test_scrub_drops_api_key_fields(self) -> None: |
| 33 | # Import helpers without network. |
| 34 | sys.path.insert(0, str(ROOT / "scripts")) |
| 35 | import catalog_models_dev as mod # type: ignore |
| 36 | |
| 37 | dirty = { |
| 38 | "models": {}, |
| 39 | "providers": { |
| 40 | "deepseek": { |
| 41 | "api_key": "sk-should-never-persist", |
| 42 | "models": {"deepseek-v4-pro": {"id": "deepseek-v4-pro"}}, |
| 43 | } |
| 44 | }, |
| 45 | "token": "nope", |
| 46 | } |
| 47 | clean = mod.scrub_secrets(dirty) |
| 48 | self.assertNotIn("token", clean) |
| 49 | self.assertNotIn("api_key", clean["providers"]["deepseek"]) |
| 50 | self.assertIn("models", clean["providers"]["deepseek"]) |
| 51 | |
| 52 | def test_ensure_shape_rejects_empty_object(self) -> None: |
| 53 | sys.path.insert(0, str(ROOT / "scripts")) |
| 54 | import catalog_models_dev as mod # type: ignore |
| 55 | |
| 56 | with self.assertRaises(SystemExit): |
| 57 | mod.ensure_models_dev_shape({}, "test") |
| 58 | |
| 59 | def test_public_document_drops_api_key(self) -> None: |
| 60 | sys.path.insert(0, str(ROOT / "scripts")) |
| 61 | import catalog_models_dev as mod # type: ignore |
| 62 | |
| 63 | dirty = { |
| 64 | "models": {}, |
| 65 | "providers": {"deepseek": {"api_key": "sk-x", "models": {}}}, |
| 66 | "token": "nope", |
| 67 | } |
| 68 | clean = mod.public_models_dev_document(dirty) |
| 69 | self.assertNotIn("token", clean) |
| 70 | self.assertNotIn("api_key", clean["providers"]["deepseek"]) |
| 71 | |
| 72 | def test_refresh_write_cache_is_rejected_without_writing(self) -> None: |
| 73 | with tempfile.TemporaryDirectory() as td: |
| 74 | source = Path(td) / "catalog.json" |
| 75 | target = Path(td) / "cache.json" |
| 76 | source.write_text( |
| 77 | json.dumps({"models": {}, "providers": {}, "api_key": "sk-nope"}), |
| 78 | encoding="utf-8", |
| 79 | ) |
| 80 | env = os.environ.copy() |
| 81 | env["CODEWHALE_MODELS_DEV_PATH"] = str(source) |
| 82 | |
| 83 | proc = subprocess.run( |
| 84 | [ |
| 85 | sys.executable, |
| 86 | str(SCRIPT), |
| 87 | "refresh", |
| 88 | "--write-cache", |
| 89 | str(target), |
| 90 | ], |
| 91 | cwd=ROOT, |
| 92 | capture_output=True, |
| 93 | text=True, |
| 94 | check=False, |
| 95 | env=env, |
| 96 | ) |
| 97 | |
| 98 | self.assertNotEqual(proc.returncode, 0) |
| 99 | self.assertIn("disk writes are intentionally unsupported", proc.stderr) |
| 100 | self.assertFalse(target.exists(), "refresh must remain dry-run only") |
| 101 | |
| 102 | def test_public_limit_value_never_echoes_tokens(self) -> None: |
| 103 | sys.path.insert(0, str(ROOT / "scripts")) |
| 104 | import catalog_models_dev as mod # type: ignore |
| 105 | |
| 106 | self.assertEqual(mod.public_limit_value(128000), "128000") |
| 107 | self.assertEqual(mod.public_limit_value(None), "null") |
| 108 | self.assertEqual(mod.public_limit_value("sk-this-is-a-token"), "redacted") |
| 109 | self.assertEqual(mod.public_limit_value({"authorization": "Bearer secret"}), "redacted") |
| 110 | self.assertEqual(mod.public_limit_value(True), "redacted") |
| 111 | |
| 112 | def test_public_source_label_strips_query_string(self) -> None: |
| 113 | sys.path.insert(0, str(ROOT / "scripts")) |
| 114 | import catalog_models_dev as mod # type: ignore |
| 115 | |
| 116 | self.assertEqual( |
| 117 | mod.public_source_label("url:https://models.dev/catalog.json?token=sk-leak"), |
| 118 | "url:https://models.dev/catalog.json", |
| 119 | ) |
| 120 | self.assertEqual(mod.public_source_label("file:/tmp/catalog.json"), "file:/tmp/catalog.json") |
| 121 | |
| 122 | def test_drift_does_not_print_token_shaped_upstream_limits(self) -> None: |
| 123 | with tempfile.TemporaryDirectory() as td: |
| 124 | seed = Path(td) / "seed.json" |
| 125 | upstream = Path(td) / "upstream.json" |
| 126 | seed.write_text( |
| 127 | json.dumps( |
| 128 | { |
| 129 | "models": { |
| 130 | "demo": {"limit": {"context": 1000, "output": 100}}, |
| 131 | }, |
| 132 | "providers": {}, |
| 133 | } |
| 134 | ), |
| 135 | encoding="utf-8", |
| 136 | ) |
| 137 | upstream.write_text( |
| 138 | json.dumps( |
| 139 | { |
| 140 | "models": { |
| 141 | "demo": { |
| 142 | "limit": { |
| 143 | "context": 1000, |
| 144 | "output": "sk-this-is-a-token", |
| 145 | } |
| 146 | }, |
| 147 | }, |
| 148 | "providers": {}, |
| 149 | "token": "sk-header-token", |
| 150 | } |
| 151 | ), |
| 152 | encoding="utf-8", |
| 153 | ) |
| 154 | env = os.environ.copy() |
| 155 | env["CODEWHALE_MODELS_DEV_PATH"] = str(upstream) |
| 156 | proc = subprocess.run( |
| 157 | [sys.executable, str(SCRIPT), "drift", "--seed", str(seed)], |
| 158 | cwd=ROOT, |
| 159 | capture_output=True, |
| 160 | text=True, |
| 161 | check=False, |
| 162 | env=env, |
| 163 | ) |
| 164 | combined = f"{proc.stdout}\n{proc.stderr}" |
| 165 | self.assertNotEqual(proc.returncode, 0) |
| 166 | self.assertNotIn("sk-this-is-a-token", combined) |
| 167 | self.assertNotIn("sk-header-token", combined) |
| 168 | self.assertIn("redacted", combined) |
| 169 | self.assertIn("limit.output", combined) |
| 170 | |
| 171 | |
| 172 | if __name__ == "__main__": |
| 173 | unittest.main() |
| 174 |