| 1 | """Characterization tests freezing the legacy --diagnose / --preflight JSON shapes. |
| 2 | |
| 3 | U4 of the doctor plan (R2): `--diagnose` and `--preflight` become frozen-shape |
| 4 | aliases once `doctor` lands. These snapshots freeze the CURRENT shapes so the |
| 5 | doctor work can prove it changed neither. Additions to the legacy shapes are |
| 6 | prohibited — new data appears only in `doctor --json` — so the key-set |
| 7 | assertions below use exact equality, not superset checks. |
| 8 | |
| 9 | The two real consumers, each with an explicit test: |
| 10 | |
| 11 | 1. The Go MCP `preflight` tool (mcp/internal/tools/preflight.go) invokes |
| 12 | `--preflight --preflight-report-on-save-dir <dir> --emit=json` and passes |
| 13 | engine stdout through VERBATIM (`mcplib.NewToolResultText(res.Stdout)`). |
| 14 | The engine side of that contract is asserted here: the exact invocation, |
| 15 | the frozen top-level key set, the nested shapes, the conditional-writes |
| 16 | entry for the report-on-save dir, and the byte format |
| 17 | (json.dumps(..., indent=2, sort_keys=True) + newline). |
| 18 | |
| 19 | 2. SKILL.md reads `--diagnose`'s `available_sources` array (the engine's |
| 20 | authoritative source list). Asserted: the key exists, is a list of |
| 21 | source-name strings. |
| 22 | |
| 23 | hooks/scripts/check-config.sh is NOT a JSON consumer (it reads env files and |
| 24 | last-run.json, never engine JSON) — deliberately no compat test for it. |
| 25 | |
| 26 | NOTE: snapshots re-recorded against the committed v3.10.0 baseline |
| 27 | (origin/main a5b3ca1, post-v3.9.x source wave: arxiv/techmeme/stocktwits/ |
| 28 | trustpilot + the x_pending_browser_auth diag flag). The SHAPES here must |
| 29 | otherwise stay frozen; re-record only when a committed baseline legitimately |
| 30 | changes them. |
| 31 | """ |
| 32 | |
| 33 | import io |
| 34 | import json |
| 35 | import sys |
| 36 | import unittest |
| 37 | from contextlib import redirect_stderr, redirect_stdout |
| 38 | from unittest import mock |
| 39 | |
| 40 | import last30days as cli |
| 41 | |
| 42 | # Source names the engine can emit in available_sources today (v3.10.0). |
| 43 | KNOWN_SOURCE_NAMES = { |
| 44 | "reddit", "x", "youtube", "tiktok", "instagram", "hackernews", "bluesky", |
| 45 | "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", |
| 46 | "perplexity", "threads", "pinterest", "digg", "jobs", "linkedin", |
| 47 | "arxiv", "techmeme", "stocktwits", "trustpilot", "dripstack", |
| 48 | } |
| 49 | |
| 50 | # --------------------------------------------------------------------------- |
| 51 | # Frozen shapes (exact key sets — additions to legacy JSON are prohibited). |
| 52 | # --------------------------------------------------------------------------- |
| 53 | |
| 54 | DIAGNOSE_TOP_KEYS = { |
| 55 | "providers", |
| 56 | "local_mode", |
| 57 | "reasoning_provider", |
| 58 | "x_backend", |
| 59 | "bird_installed", |
| 60 | "bird_authenticated", |
| 61 | "bird_username", |
| 62 | "xquik_available", |
| 63 | "xquik_working", |
| 64 | "xquik_status", |
| 65 | "native_web_backend", |
| 66 | "native_search", |
| 67 | "has_scrapecreators", |
| 68 | "has_github", |
| 69 | "x_pending_browser_auth", |
| 70 | "available_sources", |
| 71 | "safe", |
| 72 | "config_source", |
| 73 | "ignored_project_config", |
| 74 | "ignored_project_config_keys", |
| 75 | "ignored_endpoint_overrides", |
| 76 | "browser_cookies", |
| 77 | "external_commands", |
| 78 | "credential_destinations", |
| 79 | "local_writes", |
| 80 | "permission_preflight", |
| 81 | } |
| 82 | |
| 83 | DIAGNOSE_PROVIDERS_KEYS = {"google", "openai", "xai", "openrouter", "perplexity"} |
| 84 | DIAGNOSE_BROWSER_COOKIES_KEYS = {"mode", "browsers", "reads_values"} |
| 85 | DIAGNOSE_EXTERNAL_COMMANDS_KEYS = {"yt-dlp", "digg-pp-cli", "arxiv-pp-cli", "techmeme-pp-cli", "trustpilot-pp-cli", "gh"} |
| 86 | DIAGNOSE_CREDENTIAL_DESTINATIONS_KEYS = {"global_env"} |
| 87 | |
| 88 | PREFLIGHT_TOP_KEYS = { |
| 89 | "status", |
| 90 | "safe", |
| 91 | "local_reads", |
| 92 | "local_writes", |
| 93 | "conditional_writes", |
| 94 | "external_commands", |
| 95 | "credentials", |
| 96 | "network", |
| 97 | "action_items", |
| 98 | } |
| 99 | PREFLIGHT_LOCAL_READS_KEYS = {"config_source", "project_config", "browser_cookies"} |
| 100 | PREFLIGHT_PROJECT_CONFIG_KEYS = {"status", "trusted", "ignored_path", "ignored_keys"} |
| 101 | PREFLIGHT_BROWSER_COOKIES_KEYS = {"status", "mode", "browsers", "reads_values"} |
| 102 | PREFLIGHT_CREDENTIALS_KEYS = { |
| 103 | "google", "openai", "xai", "openrouter", "perplexity", "scrapecreators", "github", |
| 104 | } |
| 105 | PREFLIGHT_NETWORK_KEYS = { |
| 106 | "available_sources", "native_search", "endpoint_overrides", "ignored_endpoint_overrides", |
| 107 | } |
| 108 | |
| 109 | |
| 110 | FAKE_KEYLESS_CONFIG: dict = {} |
| 111 | |
| 112 | # Obvious dummies only (repo security hygiene): used to prove key-presence |
| 113 | # booleans stay booleans and no credential value ever reaches legacy JSON. |
| 114 | FAKE_KEYED_CONFIG = { |
| 115 | "SCRAPECREATORS_API_KEY": "dummy-sc-key-not-real-000", |
| 116 | "XAI_API_KEY": "dummy-xai-key-not-real-000", |
| 117 | "BRAVE_API_KEY": "dummy-brave-key-not-real-000", |
| 118 | } |
| 119 | |
| 120 | |
| 121 | def _run_cli(argv: list[str], config: dict) -> tuple[int, str]: |
| 122 | """Run cli.main() in-process with a controlled config; return (rc, stdout).""" |
| 123 | bird_status = { |
| 124 | "installed": False, |
| 125 | "authenticated": False, |
| 126 | "username": None, |
| 127 | "can_install": True, |
| 128 | } |
| 129 | with mock.patch.object(cli.env, "get_config", return_value=dict(config)), \ |
| 130 | mock.patch("lib.bird_x.get_bird_status", return_value=bird_status), \ |
| 131 | mock.patch("lib.bird_x.is_bird_installed", return_value=False), \ |
| 132 | mock.patch("lib.bird_x.set_credentials", lambda *a, **k: None), \ |
| 133 | mock.patch( |
| 134 | "lib.xurl_x.is_available", |
| 135 | side_effect=AssertionError( |
| 136 | "--diagnose/--preflight are safe paths and must not run the " |
| 137 | "live `xurl whoami` network check" |
| 138 | ), |
| 139 | ), \ |
| 140 | mock.patch("lib.xurl_x.has_stored_auth", return_value=False), \ |
| 141 | mock.patch.object(sys, "argv", ["last30days.py"] + argv): |
| 142 | stdout = io.StringIO() |
| 143 | stderr = io.StringIO() |
| 144 | with redirect_stdout(stdout), redirect_stderr(stderr): |
| 145 | rc = cli.main() |
| 146 | return rc, stdout.getvalue() |
| 147 | |
| 148 | |
| 149 | class DiagnoseShapeCompat(unittest.TestCase): |
| 150 | """Freeze the --diagnose JSON shape (snapshot: pre-v3.9.0 baseline).""" |
| 151 | |
| 152 | def _diagnose(self, config: dict) -> dict: |
| 153 | rc, out = _run_cli(["--diagnose"], config) |
| 154 | self.assertEqual(0, rc) |
| 155 | return json.loads(out) |
| 156 | |
| 157 | def test_top_level_key_set_is_frozen(self): |
| 158 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 159 | self.assertEqual(DIAGNOSE_TOP_KEYS, set(payload.keys())) |
| 160 | |
| 161 | def test_top_level_key_set_is_frozen_with_keys_configured(self): |
| 162 | payload = self._diagnose(FAKE_KEYED_CONFIG) |
| 163 | self.assertEqual(DIAGNOSE_TOP_KEYS, set(payload.keys())) |
| 164 | |
| 165 | def test_nested_shapes_are_frozen(self): |
| 166 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 167 | self.assertEqual(DIAGNOSE_PROVIDERS_KEYS, set(payload["providers"].keys())) |
| 168 | self.assertEqual( |
| 169 | DIAGNOSE_BROWSER_COOKIES_KEYS, set(payload["browser_cookies"].keys()) |
| 170 | ) |
| 171 | self.assertEqual( |
| 172 | DIAGNOSE_EXTERNAL_COMMANDS_KEYS, set(payload["external_commands"].keys()) |
| 173 | ) |
| 174 | self.assertEqual( |
| 175 | DIAGNOSE_CREDENTIAL_DESTINATIONS_KEYS, |
| 176 | set(payload["credential_destinations"].keys()), |
| 177 | ) |
| 178 | # The embedded permission preflight carries the same frozen shape the |
| 179 | # --preflight alias emits. |
| 180 | self.assertEqual( |
| 181 | PREFLIGHT_TOP_KEYS, set(payload["permission_preflight"].keys()) |
| 182 | ) |
| 183 | |
| 184 | def test_key_presence_fields_are_booleans_never_values(self): |
| 185 | payload = self._diagnose(FAKE_KEYED_CONFIG) |
| 186 | self.assertIs(True, payload["has_scrapecreators"]) |
| 187 | for value in payload["providers"].values(): |
| 188 | self.assertIsInstance(value, bool) |
| 189 | raw = json.dumps(payload) |
| 190 | for secret in FAKE_KEYED_CONFIG.values(): |
| 191 | self.assertNotIn(secret, raw) |
| 192 | |
| 193 | def test_diagnose_runs_safe_mode(self): |
| 194 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 195 | self.assertIs(True, payload["safe"]) |
| 196 | |
| 197 | def test_skill_md_consumer_available_sources_array(self): |
| 198 | """Consumer (b): SKILL.md reads `available_sources` as the engine's |
| 199 | authoritative list of source names.""" |
| 200 | payload = self._diagnose(FAKE_KEYLESS_CONFIG) |
| 201 | self.assertIn("available_sources", payload) |
| 202 | sources = payload["available_sources"] |
| 203 | self.assertIsInstance(sources, list) |
| 204 | self.assertTrue(sources, "available_sources must never be empty (reddit/hn are free)") |
| 205 | for name in sources: |
| 206 | self.assertIsInstance(name, str) |
| 207 | self.assertIn(name, KNOWN_SOURCE_NAMES) |
| 208 | # Free sources are always present even in a keyless environment. |
| 209 | for free in ("reddit", "hackernews", "polymarket", "github"): |
| 210 | self.assertIn(free, sources) |
| 211 | |
| 212 | |
| 213 | class PreflightShapeCompat(unittest.TestCase): |
| 214 | """Freeze the --preflight JSON shape (snapshot: pre-v3.9.0 baseline).""" |
| 215 | |
| 216 | MCP_SAVE_DIR = "/tmp/last30days-mcp-save-dir" |
| 217 | |
| 218 | def _preflight_mcp_invocation(self, config: dict) -> tuple[str, dict]: |
| 219 | # Consumer (a): mcp/internal/tools/preflight.go builds exactly |
| 220 | # ["--preflight", "--preflight-report-on-save-dir", mcpSaveDir()] |
| 221 | # plus "--emit=json" for format=json, and passes stdout through |
| 222 | # verbatim. Mirror that invocation exactly. |
| 223 | rc, out = _run_cli( |
| 224 | [ |
| 225 | "--preflight", |
| 226 | "--preflight-report-on-save-dir", |
| 227 | self.MCP_SAVE_DIR, |
| 228 | "--emit=json", |
| 229 | ], |
| 230 | config, |
| 231 | ) |
| 232 | self.assertEqual(0, rc) |
| 233 | return out, json.loads(out) |
| 234 | |
| 235 | def test_mcp_passthrough_top_level_key_set_is_frozen(self): |
| 236 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 237 | self.assertEqual(PREFLIGHT_TOP_KEYS, set(payload.keys())) |
| 238 | |
| 239 | def test_mcp_passthrough_top_level_key_set_is_frozen_with_keys(self): |
| 240 | _, payload = self._preflight_mcp_invocation(FAKE_KEYED_CONFIG) |
| 241 | self.assertEqual(PREFLIGHT_TOP_KEYS, set(payload.keys())) |
| 242 | |
| 243 | def test_mcp_passthrough_nested_shapes_are_frozen(self): |
| 244 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 245 | self.assertEqual(PREFLIGHT_LOCAL_READS_KEYS, set(payload["local_reads"].keys())) |
| 246 | self.assertEqual( |
| 247 | PREFLIGHT_PROJECT_CONFIG_KEYS, |
| 248 | set(payload["local_reads"]["project_config"].keys()), |
| 249 | ) |
| 250 | self.assertEqual( |
| 251 | PREFLIGHT_BROWSER_COOKIES_KEYS, |
| 252 | set(payload["local_reads"]["browser_cookies"].keys()), |
| 253 | ) |
| 254 | self.assertEqual(PREFLIGHT_CREDENTIALS_KEYS, set(payload["credentials"].keys())) |
| 255 | for entry in payload["credentials"].values(): |
| 256 | self.assertEqual({"present", "label"}, set(entry.keys())) |
| 257 | self.assertIsInstance(entry["present"], bool) |
| 258 | self.assertEqual(PREFLIGHT_NETWORK_KEYS, set(payload["network"].keys())) |
| 259 | self.assertIsInstance(payload["network"]["available_sources"], list) |
| 260 | |
| 261 | def test_mcp_passthrough_reports_conditional_write_for_save_dir(self): |
| 262 | # With --preflight-report-on-save-dir and no --save-dir, the report |
| 263 | # dir appears as a conditional write — the MCP tool relies on this to |
| 264 | # explain what a save WOULD touch. |
| 265 | _, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 266 | self.assertIn( |
| 267 | {"kind": "report_on_save", "path": self.MCP_SAVE_DIR}, |
| 268 | payload["conditional_writes"], |
| 269 | ) |
| 270 | |
| 271 | def test_mcp_passthrough_byte_format_is_stable(self): |
| 272 | # The Go tool surfaces stdout verbatim, so the serialization format |
| 273 | # (indent=2, sort_keys=True, trailing newline) is part of the contract. |
| 274 | out, payload = self._preflight_mcp_invocation(FAKE_KEYLESS_CONFIG) |
| 275 | self.assertEqual(json.dumps(payload, indent=2, sort_keys=True) + "\n", out) |
| 276 | |
| 277 | def test_no_secret_values_in_preflight_output(self): |
| 278 | out, _ = self._preflight_mcp_invocation(FAKE_KEYED_CONFIG) |
| 279 | for secret in FAKE_KEYED_CONFIG.values(): |
| 280 | self.assertNotIn(secret, out) |
| 281 | |
| 282 | |
| 283 | if __name__ == "__main__": |
| 284 | unittest.main() |
| 285 |