| 1 | """Characterization tests freezing the legacy --diagnose / --preflight JSON shapes. |
| 2 | |
| 3 | U4 of the doctor plan (R2): `--diagnose` and `--preflight` become frozen-shape |
| 4 | aliases once `doctor` lands. These snapshots freeze the CURRENT shapes so the |
| 5 | doctor work can prove it changed neither. Additions to the legacy shapes are |
| 6 | prohibited — new data appears only in `doctor --json` — so the key-set |
| 7 | assertions below use exact equality, not superset checks. |
| 8 | |
| 9 | The two real consumers, each with an explicit test: |
| 10 | |
| 11 | 1. The Go MCP `preflight` tool (mcp/internal/tools/preflight.go) invokes |
| 12 | `--preflight --preflight-report-on-save-dir <dir> --emit=json` and passes |
| 13 | engine stdout through VERBATIM (`mcplib.NewToolResultText(res.Stdout)`). |
| 14 | The engine side of that contract is asserted here: the exact invocation, |
| 15 | the frozen top-level key set, the nested shapes, the conditional-writes |
| 16 | entry for the report-on-save dir, and the byte format |
| 17 | (json.dumps(..., indent=2, sort_keys=True) + newline). |
| 18 | |
| 19 | 2. SKILL.md reads `--diagnose`'s `available_sources` array (the engine's |
| 20 | authoritative source list). Asserted: the key exists, is a list of |
| 21 | source-name strings. |
| 22 | |
| 23 | There is no SessionStart hook; last-run.json is not consumed at session start. |
| 24 | |
| 25 | NOTE: snapshots re-recorded against the committed v3.10.0 baseline |
| 26 | (origin/main a5b3ca1, post-v3.9.x source wave: arxiv/techmeme/stocktwits/ |
| 27 | trustpilot + the x_pending_browser_auth diag flag). The SHAPES here must |
| 28 | otherwise stay frozen; re-record only when a committed baseline legitimately |
| 29 | changes them. |
| 30 | """ |
| 31 | |
| 32 | import contextlib |
| 33 | import io |
| 34 | import json |
| 35 | import sys |
| 36 | import tempfile |
| 37 | import unittest |
| 38 | from contextlib import redirect_stderr, redirect_stdout |
| 39 | from pathlib import Path |
| 40 | from unittest import mock |
| 41 | |
| 42 | import last30days as cli |
| 43 | |
| 44 | # Source names the engine can emit in available_sources today (v3.10.0). |
| 45 | KNOWN_SOURCE_NAMES = { |
| 46 | "reddit", "x", "youtube", "tiktok", "instagram", "hackernews", "bluesky", |
| 47 | "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", |
| 48 | "perplexity", "threads", "pinterest", "digg", "jobs", "linkedin", |
| 49 | "arxiv", "techmeme", "stocktwits", "trustpilot", "dripstack", |
| 50 | } |
| 51 | |
| 52 | # --------------------------------------------------------------------------- |
| 53 | # Frozen shapes (exact key sets — additions to legacy JSON are prohibited). |
| 54 | # --------------------------------------------------------------------------- |
| 55 | |
| 56 | DIAGNOSE_TOP_KEYS = { |
| 57 | "providers", |
| 58 | "local_mode", |
| 59 | "reasoning_provider", |
| 60 | "x_backend", |
| 61 | "bird_installed", |
| 62 | "bird_authenticated", |
| 63 | "bird_username", |
| 64 | "xquik_available", |
| 65 | "xquik_working", |
| 66 | "xquik_status", |
| 67 | "native_web_backend", |
| 68 | "native_search", |
| 69 | "has_scrapecreators", |
| 70 | "has_github", |
| 71 | "brightdata_installed", |
| 72 | "brightdata_authenticated", |
| 73 | "x_pending_browser_auth", |
| 74 | "available_sources", |
| 75 | "safe", |
| 76 | "config_source", |
| 77 | "ignored_project_config", |
| 78 | "ignored_project_config_keys", |
| 79 | "ignored_endpoint_overrides", |
| 80 | "browser_cookies", |
| 81 | "external_commands", |
| 82 | "credential_destinations", |
| 83 | "local_writes", |
| 84 | "permission_preflight", |
| 85 | } |
| 86 | |
| 87 | DIAGNOSE_PROVIDERS_KEYS = {"google", "openai", "xai", "openrouter", "perplexity"} |
| 88 | DIAGNOSE_BROWSER_COOKIES_KEYS = {"mode", "browsers", "reads_values"} |
| 89 | DIAGNOSE_EXTERNAL_COMMANDS_KEYS = { |
| 90 | "yt-dlp", "digg-pp-cli", "arxiv-pp-cli", "techmeme-pp-cli", "trustpilot-pp-cli", |
| 91 | "brightdata", "gh", |
| 92 | } |
| 93 | DIAGNOSE_CREDENTIAL_DESTINATIONS_KEYS = {"global_env"} |
| 94 | |
| 95 | PREFLIGHT_TOP_KEYS = { |
| 96 | "status", |
| 97 | "safe", |
| 98 | "local_reads", |
| 99 | "local_writes", |
| 100 | "conditional_writes", |
| 101 | "external_commands", |
| 102 | "credentials", |
| 103 | "network", |
| 104 | "action_items", |
| 105 | } |
| 106 | PREFLIGHT_LOCAL_READS_KEYS = {"config_source", "project_config", "browser_cookies"} |
| 107 | PREFLIGHT_PROJECT_CONFIG_KEYS = {"status", "trusted", "ignored_path", "ignored_keys"} |
| 108 | PREFLIGHT_BROWSER_COOKIES_KEYS = {"status", "mode", "browsers", "reads_values"} |
| 109 | PREFLIGHT_CREDENTIALS_KEYS = { |
| 110 | "google", "openai", "xai", "openrouter", "perplexity", "scrapecreators", "github", |
| 111 | # X API bearer (X_BEARER_TOKEN): computed inside permission_preflight from |
| 112 | # config, never surfaced through diagnose.providers (whose key set above |
| 113 | # stays frozen). |
| 114 | "x_bearer", |
| 115 | } |
| 116 | PREFLIGHT_NETWORK_KEYS = { |
| 117 | "available_sources", "native_search", "endpoint_overrides", "ignored_endpoint_overrides", |
| 118 | } |
| 119 | |
| 120 | |
| 121 | FAKE_KEYLESS_CONFIG: dict = {} |
| 122 | |
| 123 | # Obvious dummies only (repo security hygiene): used to prove key-presence |
| 124 | # booleans stay booleans and no credential value ever reaches legacy JSON. |
| 125 | FAKE_KEYED_CONFIG = { |
| 126 | "SCRAPECREATORS_API_KEY": "dummy-sc-key-not-real-000", |
| 127 | "XAI_API_KEY": "dummy-xai-key-not-real-000", |
| 128 | "BRAVE_API_KEY": "dummy-brave-key-not-real-000", |
| 129 | } |
| 130 | |
| 131 | |
| 132 | _BIRD_STATUS_KEYLESS = { |
| 133 | "installed": False, |
| 134 | "authenticated": False, |
| 135 | "username": None, |
| 136 | "can_install": True, |
| 137 | } |
| 138 | |
| 139 | |
| 140 | def _run_cli( |
| 141 | argv: list[str], |
| 142 | config: dict, |
| 143 | *, |
| 144 | extra_patches: tuple = (), |
| 145 | patch_has_stored_auth: bool = True, |
| 146 | ) -> tuple[int, str]: |
| 147 | """Run cli.main() in-process with a controlled config; return (rc, stdout). |
| 148 | |
| 149 | `extra_patches` are additional `mock.patch(...)` context managers layered |
| 150 | on top of the common safe-path mock stack (e.g. to fake xurl's on-disk |
| 151 | token store instead of stubbing `has_stored_auth` directly). |
| 152 | `patch_has_stored_auth=False` omits the default `has_stored_auth` stub so |
| 153 | a caller-supplied patch (or the real function) can take its place. |
| 154 | """ |
| 155 | with contextlib.ExitStack() as stack: |
| 156 | stack.enter_context(mock.patch.object(cli.env, "get_config", return_value=dict(config))) |
| 157 | stack.enter_context(mock.patch("lib.bird_x.get_bird_status", return_value=_BIRD_STATUS_KEYLESS)) |
| 158 | stack.enter_context(mock.patch("lib.bird_x.is_bird_installed", return_value=False)) |
| 159 | stack.enter_context(mock.patch("lib.bird_x.set_credentials", lambda *a, **k: None)) |
| 160 | stack.enter_context(mock.patch( |
| 161 | "lib.xurl_x.is_available", |
| 162 | side_effect=AssertionError( |
| 163 | "--diagnose/--preflight are safe paths and must not run the " |
| 164 | "live `xurl whoami` network check" |
| 165 | ), |
| 166 | )) |
| 167 | if patch_has_stored_auth: |
| 168 | stack.enter_context(mock.patch("lib.xurl_x.has_stored_auth", return_value=False)) |
| 169 | for patch in extra_patches: |
| 170 | stack.enter_context(patch) |
| 171 | stack.enter_context(mock.patch.object(sys, "argv", ["last30days.py"] + argv)) |
| 172 | stdout = io.StringIO() |
| 173 | stderr = io.StringIO() |
| 174 | with redirect_stdout(stdout), redirect_stderr(stderr): |
| 175 | rc = cli.main() |
| 176 | return rc, stdout.getvalue() |
| 177 | |
| 178 | |
| 179 | class DiagnoseShapeCompat(unittest.TestCase): |
| 180 | """Freeze the --diagnose JSON shape (snapshot: pre-v3.9.0 baseline).""" |
| 181 | |
| 182 | def _diagnose(self, config: dict) -> dict: |
| 183 | rc, out = _run_cli(["--diagnose"], config) |
| 184 | self.assertEqual(0, rc) |
| 185 | return json.loads(out) |
| 186 | |
| 187 | def test_top_level_key_set_is_frozen(self): |
| 188 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 189 | self.assertEqual(DIAGNOSE_TOP_KEYS, set(payload.keys())) |
| 190 | |
| 191 | def test_top_level_key_set_is_frozen_with_keys_configured(self): |
| 192 | payload = self._diagnose(FAKE_KEYED_CONFIG) |
| 193 | self.assertEqual(DIAGNOSE_TOP_KEYS, set(payload.keys())) |
| 194 | |
| 195 | def test_nested_shapes_are_frozen(self): |
| 196 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 197 | self.assertEqual(DIAGNOSE_PROVIDERS_KEYS, set(payload["providers"].keys())) |
| 198 | self.assertEqual( |
| 199 | DIAGNOSE_BROWSER_COOKIES_KEYS, set(payload["browser_cookies"].keys()) |
| 200 | ) |
| 201 | self.assertEqual( |
| 202 | DIAGNOSE_EXTERNAL_COMMANDS_KEYS, set(payload["external_commands"].keys()) |
| 203 | ) |
| 204 | self.assertEqual( |
| 205 | DIAGNOSE_CREDENTIAL_DESTINATIONS_KEYS, |
| 206 | set(payload["credential_destinations"].keys()), |
| 207 | ) |
| 208 | # The embedded permission preflight carries the same frozen shape the |
| 209 | # --preflight alias emits. |
| 210 | self.assertEqual( |
| 211 | PREFLIGHT_TOP_KEYS, set(payload["permission_preflight"].keys()) |
| 212 | ) |
| 213 | |
| 214 | def test_key_presence_fields_are_booleans_never_values(self): |
| 215 | payload = self._diagnose(FAKE_KEYED_CONFIG) |
| 216 | self.assertIs(True, payload["has_scrapecreators"]) |
| 217 | for value in payload["providers"].values(): |
| 218 | self.assertIsInstance(value, bool) |
| 219 | raw = json.dumps(payload) |
| 220 | for secret in FAKE_KEYED_CONFIG.values(): |
| 221 | self.assertNotIn(secret, raw) |
| 222 | |
| 223 | def test_diagnose_runs_safe_mode(self): |
| 224 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 225 | self.assertIs(True, payload["safe"]) |
| 226 | |
| 227 | def test_skill_md_consumer_available_sources_array(self): |
| 228 | """Consumer (b): SKILL.md reads `available_sources` as the engine's |
| 229 | authoritative list of source names.""" |
| 230 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 231 | self.assertIn("available_sources", payload) |
| 232 | sources = payload["available_sources"] |
| 233 | self.assertIsInstance(sources, list) |
| 234 | self.assertTrue(sources, "available_sources must never be empty (reddit/hn are free)") |
| 235 | for name in sources: |
| 236 | self.assertIsInstance(name, str) |
| 237 | self.assertIn(name, KNOWN_SOURCE_NAMES) |
| 238 | # Free sources are always present even in a keyless environment. |
| 239 | for free in ("reddit", "hackernews", "polymarket", "github"): |
| 240 | self.assertIn(free, sources) |
| 241 | |
| 242 | |
| 243 | class DiagnoseXurlAuthWiring(unittest.TestCase): |
| 244 | """Regression for #978 / PR #1027: prove `--diagnose`'s `x_backend` and |
| 245 | `available_sources` reflect xurl_x.stored_auth_status()'s corrected |
| 246 | directory-layout detection end-to-end through the real CLI entrypoint, |
| 247 | not just at the `has_stored_auth()` unit level (test_xurl_x.py) or a |
| 248 | mocked wiring level (test_env_v3.py). Neither of those proves the two |
| 249 | compose correctly through `pipeline.diagnose()` into the exact |
| 250 | `available_sources` array SKILL.md reads. |
| 251 | |
| 252 | Scope: this covers the AUTH_OK / AUTH_MISSING distinction only. |
| 253 | `has_stored_auth()` collapses AUTH_ERROR (permission-denied store) to |
| 254 | the same `False` as AUTH_MISSING, so a permission-denied store is not |
| 255 | separately observable through `--diagnose`/`available_sources` -- only |
| 256 | `doctor`'s `backends._probe_xurl` surfaces the typed AUTH_ERROR. Not |
| 257 | covered here; that distinction is doctor-only by the current design.""" |
| 258 | |
| 259 | def setUp(self): |
| 260 | self._tmp = tempfile.TemporaryDirectory() |
| 261 | self.addCleanup(self._tmp.cleanup) |
| 262 | self.fake_home = Path(self._tmp.name) |
| 263 | self.store = self.fake_home / ".xurl" |
| 264 | boom = mock.patch( |
| 265 | "subprocess.run", |
| 266 | side_effect=AssertionError( |
| 267 | "--diagnose is a safe path and must not spawn any subprocess" |
| 268 | ), |
| 269 | ) |
| 270 | boom.start() |
| 271 | self.addCleanup(boom.stop) |
| 272 | |
| 273 | def _diagnose_with_store(self, auth_yml_content: str | None) -> dict: |
| 274 | """Run `--diagnose` with `Path.home()` pointed at a fake home dir |
| 275 | holding a real `~/.xurl/auth.yml` (current directory layout); |
| 276 | `auth_yml_content=None` leaves no store at all. `Path.home()` -- not |
| 277 | `token_store_path()` -- is what's faked, so this exercises |
| 278 | `token_store_path()`'s own directory-layout logic instead of |
| 279 | bypassing it; a pre-fix `token_store_path()` returning the bare |
| 280 | `~/.xurl` directory would make this fail exactly as it did for #978. |
| 281 | |
| 282 | `Path.home` is a shared class attribute, so this patch also redirects |
| 283 | any other lib module's `Path.home()` call for the duration of the |
| 284 | CLI invocation (e.g. `brightdata.gate_status`). That's inert today: |
| 285 | the paired `shutil.which` patch below makes `brightdata.is_installed()` |
| 286 | return False, short-circuiting `has_credentials()` before it would |
| 287 | reach `Path.home()`. If that short-circuit is ever removed, re-check |
| 288 | whether another module's home-relative lookup needs isolating too.""" |
| 289 | if auth_yml_content is not None: |
| 290 | self.store.mkdir(exist_ok=True) |
| 291 | (self.store / "auth.yml").write_text(auth_yml_content, encoding="utf-8") |
| 292 | rc, out = _run_cli( |
| 293 | ["--diagnose"], |
| 294 | FAKE_KEYLESS_CONFIG, |
| 295 | patch_has_stored_auth=False, |
| 296 | extra_patches=( |
| 297 | mock.patch("lib.xurl_x.Path.home", return_value=self.fake_home), |
| 298 | mock.patch( |
| 299 | "lib.xurl_x.shutil.which", |
| 300 | side_effect=lambda name: "/usr/local/bin/xurl" if name == "xurl" else None, |
| 301 | ), |
| 302 | ), |
| 303 | ) |
| 304 | self.assertEqual(0, rc) |
| 305 | return json.loads(out) |
| 306 | |
| 307 | def test_new_layout_auth_yml_makes_xurl_the_reported_backend(self): |
| 308 | payload = self._diagnose_with_store("access_token: dummy-not-real\n") |
| 309 | self.assertEqual("xurl", payload["x_backend"]) |
| 310 | self.assertIn("x", payload["available_sources"]) |
| 311 | |
| 312 | def test_no_store_leaves_xurl_unreported(self): |
| 313 | payload = self._diagnose_with_store(None) |
| 314 | self.assertNotEqual("xurl", payload["x_backend"]) |
| 315 | self.assertFalse( |
| 316 | payload["x_pending_browser_auth"], |
| 317 | "no browser-cookie config in this test, so pending-auth must be False", |
| 318 | ) |
| 319 | self.assertNotIn("x", payload["available_sources"]) |
| 320 | |
| 321 | |
| 322 | class PreflightShapeCompat(unittest.TestCase): |
| 323 | """Freeze the --preflight JSON shape (snapshot: pre-v3.9.0 baseline).""" |
| 324 | |
| 325 | MCP_SAVE_DIR = "/tmp/last30days-mcp-save-dir" |
| 326 | |
| 327 | def _preflight_mcp_invocation(self, config: dict) -> tuple[str, dict]: |
| 328 | # Consumer (a): mcp/internal/tools/preflight.go builds exactly |
| 329 | # ["--preflight", "--preflight-report-on-save-dir", mcpSaveDir()] |
| 330 | # plus "--emit=json" for format=json, and passes stdout through |
| 331 | # verbatim. Mirror that invocation exactly. |
| 332 | rc, out = _run_cli( |
| 333 | [ |
| 334 | "--preflight", |
| 335 | "--preflight-report-on-save-dir", |
| 336 | self.MCP_SAVE_DIR, |
| 337 | "--emit=json", |
| 338 | ], |
| 339 | config, |
| 340 | ) |
| 341 | self.assertEqual(0, rc) |
| 342 | return out, json.loads(out) |
| 343 | |
| 344 | def test_mcp_passthrough_top_level_key_set_is_frozen(self): |
| 345 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 346 | self.assertEqual(PREFLIGHT_TOP_KEYS, set(payload.keys())) |
| 347 | |
| 348 | def test_mcp_passthrough_top_level_key_set_is_frozen_with_keys(self): |
| 349 | _, payload = self._preflight_mcp_invocation(FAKE_KEYED_CONFIG) |
| 350 | self.assertEqual(PREFLIGHT_TOP_KEYS, set(payload.keys())) |
| 351 | |
| 352 | def test_mcp_passthrough_nested_shapes_are_frozen(self): |
| 353 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 354 | self.assertEqual(PREFLIGHT_LOCAL_READS_KEYS, set(payload["local_reads"].keys())) |
| 355 | self.assertEqual( |
| 356 | PREFLIGHT_PROJECT_CONFIG_KEYS, |
| 357 | set(payload["local_reads"]["project_config"].keys()), |
| 358 | ) |
| 359 | self.assertEqual( |
| 360 | PREFLIGHT_BROWSER_COOKIES_KEYS, |
| 361 | set(payload["local_reads"]["browser_cookies"].keys()), |
| 362 | ) |
| 363 | self.assertEqual(PREFLIGHT_CREDENTIALS_KEYS, set(payload["credentials"].keys())) |
| 364 | for entry in payload["credentials"].values(): |
| 365 | self.assertEqual({"present", "label"}, set(entry.keys())) |
| 366 | self.assertIsInstance(entry["present"], bool) |
| 367 | self.assertEqual(PREFLIGHT_NETWORK_KEYS, set(payload["network"].keys())) |
| 368 | self.assertIsInstance(payload["network"]["available_sources"], list) |
| 369 | |
| 370 | def test_mcp_passthrough_reports_conditional_write_for_save_dir(self): |
| 371 | # With --preflight-report-on-save-dir and no --save-dir, the report |
| 372 | # dir appears as a conditional write — the MCP tool relies on this to |
| 373 | # explain what a save WOULD touch. |
| 374 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 375 | self.assertIn( |
| 376 | {"kind": "report_on_save", "path": self.MCP_SAVE_DIR}, |
| 377 | payload["conditional_writes"], |
| 378 | ) |
| 379 | |
| 380 | def test_mcp_passthrough_byte_format_is_stable(self): |
| 381 | # The Go tool surfaces stdout verbatim, so the serialization format |
| 382 | # (indent=2, sort_keys=True, trailing newline) is part of the contract. |
| 383 | out, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 384 | self.assertEqual(json.dumps(payload, indent=2, sort_keys=True) + "\n", out) |
| 385 | |
| 386 | def test_no_secret_values_in_preflight_output(self): |
| 387 | out, _ = self._preflight_mcp_invocation(FAKE_KEYED_CONFIG) |
| 388 | for secret in FAKE_KEYED_CONFIG.values(): |
| 389 | self.assertNotIn(secret, out) |
| 390 | |
| 391 | |
| 392 | if __name__ == "__main__": |
| 393 | unittest.main() |
| 394 |