| 1 | """Tests for the Bright Data CLI adapter (U1). |
| 2 | |
| 3 | Covers the activation gate (PATH + presence-only credential probe), the |
| 4 | never-raise error envelope across every failure mode, and the verbatim |
| 5 | passthrough of the CLI's own auth/credit warnings. |
| 6 | |
| 7 | No test in this file spawns a real subprocess or touches the network. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import json |
| 13 | import sys |
| 14 | from pathlib import Path |
| 15 | |
| 16 | import pytest |
| 17 | |
| 18 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts")) |
| 19 | |
| 20 | from lib import brightdata, subproc # noqa: E402 |
| 21 | |
| 22 | |
| 23 | class _Result: |
| 24 | """Stand-in for subproc.SubprocResult.""" |
| 25 | |
| 26 | def __init__(self, returncode=0, stdout="", stderr=""): |
| 27 | self.returncode = returncode |
| 28 | self.stdout = stdout |
| 29 | self.stderr = stderr |
| 30 | |
| 31 | |
| 32 | # One real search record, trimmed. Field names verified against a live |
| 33 | # amazon_product_search payload on 2026-08-13. |
| 34 | SEARCH_RECORD = { |
| 35 | "asin": "B0DWD47BW7", |
| 36 | "url": "https://www.amazon.com/dp/B0DWD47BW7", |
| 37 | "name": "Chill Max Leak-Proof XL Bento-Style Lunch Box | Ice Pack Included", |
| 38 | "brand": "Bentgo", |
| 39 | "sponsored": "false", |
| 40 | "rating": 4.4, |
| 41 | "num_ratings": 459, |
| 42 | "final_price": 39.99, |
| 43 | "currency": "USD", |
| 44 | } |
| 45 | |
| 46 | |
| 47 | # ---------------------------------------------------------------- gate |
| 48 | |
| 49 | |
| 50 | def test_gate_false_when_binary_missing(monkeypatch): |
| 51 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: None) |
| 52 | assert brightdata.is_installed() is False |
| 53 | assert brightdata.is_available({}) is False |
| 54 | |
| 55 | |
| 56 | def test_gate_reports_unauthenticated_when_no_credential_signal(monkeypatch, tmp_path): |
| 57 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: "/usr/local/bin/brightdata") |
| 58 | monkeypatch.setattr(brightdata, "_config_dir", lambda: tmp_path / "absent") |
| 59 | status = brightdata.gate_status({}) |
| 60 | assert status["brightdata_installed"] is True |
| 61 | assert status["brightdata_authenticated"] is False |
| 62 | assert brightdata.is_available({}) is False |
| 63 | |
| 64 | |
| 65 | def test_api_key_alone_satisfies_the_credential_probe(monkeypatch, tmp_path): |
| 66 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: "/usr/local/bin/brightdata") |
| 67 | monkeypatch.setattr(brightdata, "_config_dir", lambda: tmp_path / "absent") |
| 68 | assert brightdata.is_available({"BRIGHTDATA_API_KEY": "dummy-key-not-real"}) is True |
| 69 | |
| 70 | |
| 71 | def test_credentials_file_satisfies_the_probe_without_being_read(monkeypatch, tmp_path): |
| 72 | cfg = tmp_path / "brightdata-cli" |
| 73 | cfg.mkdir() |
| 74 | creds = cfg / "credentials.json" |
| 75 | creds.write_text('{"token": "dummy-not-real"}') |
| 76 | creds.chmod(0o600) |
| 77 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: "/usr/local/bin/brightdata") |
| 78 | monkeypatch.setattr(brightdata, "_config_dir", lambda: cfg) |
| 79 | |
| 80 | def _explode(*a, **k): # pragma: no cover - must never run |
| 81 | raise AssertionError("credential contents must never be read") |
| 82 | |
| 83 | monkeypatch.setattr(Path, "read_text", _explode) |
| 84 | assert brightdata.has_credentials({}) is True |
| 85 | |
| 86 | |
| 87 | def test_gate_does_not_spawn_a_subprocess(monkeypatch, tmp_path): |
| 88 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: None) |
| 89 | |
| 90 | def _explode(*a, **k): # pragma: no cover - must never run |
| 91 | raise AssertionError("gate must stay offline") |
| 92 | |
| 93 | monkeypatch.setattr(subproc, "run_with_timeout", _explode) |
| 94 | brightdata.gate_status({}) |
| 95 | assert brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30)["records"] == [] |
| 96 | |
| 97 | |
| 98 | # ------------------------------------------------------------ envelope |
| 99 | |
| 100 | |
| 101 | def _installed(monkeypatch): |
| 102 | monkeypatch.setattr(brightdata.shutil, "which", lambda _: "/usr/local/bin/brightdata") |
| 103 | |
| 104 | |
| 105 | def test_happy_path_parses_a_bare_json_array(monkeypatch): |
| 106 | """Live payloads are a top-level list, not a {"results": ...} envelope.""" |
| 107 | _installed(monkeypatch) |
| 108 | monkeypatch.setattr( |
| 109 | subproc, "run_with_timeout", |
| 110 | lambda *a, **k: _Result(stdout=json.dumps([SEARCH_RECORD, SEARCH_RECORD])), |
| 111 | ) |
| 112 | out = brightdata.run_pipeline("amazon_product_search", ["bentgo", "https://www.amazon.com"], timeout=60) |
| 113 | assert "error" not in out |
| 114 | assert len(out["records"]) == 2 |
| 115 | assert out["records"][0]["asin"] == "B0DWD47BW7" |
| 116 | |
| 117 | |
| 118 | @pytest.mark.parametrize("wrapper", ["records", "results", "data"]) |
| 119 | def test_dict_envelopes_are_tolerated_for_cli_churn(monkeypatch, wrapper): |
| 120 | _installed(monkeypatch) |
| 121 | monkeypatch.setattr( |
| 122 | subproc, "run_with_timeout", |
| 123 | lambda *a, **k: _Result(stdout=json.dumps({wrapper: [SEARCH_RECORD]})), |
| 124 | ) |
| 125 | assert len(brightdata.run_pipeline("amazon_product", ["u"], timeout=30)["records"]) == 1 |
| 126 | |
| 127 | |
| 128 | def test_auth_401_surfaces_the_cli_error_without_raising(monkeypatch): |
| 129 | _installed(monkeypatch) |
| 130 | stderr = "Triggering pipeline collection...\nError: 401 Unauthorized - run `brightdata login`" |
| 131 | monkeypatch.setattr( |
| 132 | subproc, "run_with_timeout", lambda *a, **k: _Result(returncode=1, stderr=stderr) |
| 133 | ) |
| 134 | out = brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 135 | assert out["records"] == [] |
| 136 | assert "401" in out["error"] |
| 137 | |
| 138 | |
| 139 | def test_error_uses_the_last_stderr_line_not_the_polling_narration(monkeypatch): |
| 140 | """The CLI narrates polling on stderr; the failure is the final line.""" |
| 141 | _installed(monkeypatch) |
| 142 | stderr = ( |
| 143 | "Triggering pipeline collection for amazon_product_reviews...\n" |
| 144 | "Status: running - polling again (attempt 1/600)\n" |
| 145 | "Error: snapshot failed" |
| 146 | ) |
| 147 | monkeypatch.setattr( |
| 148 | subproc, "run_with_timeout", lambda *a, **k: _Result(returncode=1, stderr=stderr) |
| 149 | ) |
| 150 | assert brightdata.run_pipeline("amazon_product_reviews", ["u", "50"], timeout=30)["error"] == "Error: snapshot failed" |
| 151 | |
| 152 | |
| 153 | def test_subprocess_timeout_returns_empty_records_and_error(monkeypatch): |
| 154 | _installed(monkeypatch) |
| 155 | |
| 156 | def _timeout(*a, **k): |
| 157 | raise subproc.SubprocTimeout("Command brightdata timed out after 180s") |
| 158 | |
| 159 | monkeypatch.setattr(subproc, "run_with_timeout", _timeout) |
| 160 | out = brightdata.run_pipeline("amazon_product_reviews", ["u", "50"], timeout=180) |
| 161 | assert out["records"] == [] |
| 162 | assert "timed out" in out["error"] |
| 163 | |
| 164 | |
| 165 | def test_malformed_json_returns_empty_records_and_error(monkeypatch): |
| 166 | _installed(monkeypatch) |
| 167 | monkeypatch.setattr(subproc, "run_with_timeout", lambda *a, **k: _Result(stdout="not json{")) |
| 168 | out = brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 169 | assert out["records"] == [] |
| 170 | assert "json decode" in out["error"] |
| 171 | |
| 172 | |
| 173 | def test_empty_stdout_is_not_an_error(monkeypatch): |
| 174 | _installed(monkeypatch) |
| 175 | monkeypatch.setattr(subproc, "run_with_timeout", lambda *a, **k: _Result(stdout=" \n")) |
| 176 | out = brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 177 | assert out["records"] == [] |
| 178 | assert "error" not in out |
| 179 | |
| 180 | |
| 181 | def test_spawn_failures_never_raise(monkeypatch): |
| 182 | _installed(monkeypatch) |
| 183 | for exc in (FileNotFoundError("no binary"), OSError("spawn failed")): |
| 184 | def _raise(*a, __e=exc, **k): |
| 185 | raise __e |
| 186 | |
| 187 | monkeypatch.setattr(subproc, "run_with_timeout", _raise) |
| 188 | out = brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 189 | assert out["records"] == [] and out["error"] |
| 190 | |
| 191 | |
| 192 | # ------------------------------------------------------------ arguments |
| 193 | |
| 194 | |
| 195 | def test_cli_timeout_sits_below_the_subprocess_timeout(monkeypatch): |
| 196 | """The CLI must fail cleanly on its own before we SIGTERM it.""" |
| 197 | _installed(monkeypatch) |
| 198 | seen = {} |
| 199 | monkeypatch.setattr( |
| 200 | subproc, "run_with_timeout", |
| 201 | lambda cmd, **k: seen.update(cmd=cmd, timeout=k["timeout"]) or _Result(stdout="[]"), |
| 202 | ) |
| 203 | brightdata.run_pipeline("amazon_product_reviews", ["u", "50"], timeout=180) |
| 204 | cli_timeout = int(seen["cmd"][seen["cmd"].index("--timeout") + 1]) |
| 205 | assert cli_timeout < seen["timeout"] == 180 |
| 206 | |
| 207 | |
| 208 | def test_params_are_passed_positionally_in_order(monkeypatch): |
| 209 | _installed(monkeypatch) |
| 210 | seen = {} |
| 211 | monkeypatch.setattr( |
| 212 | subproc, "run_with_timeout", |
| 213 | lambda cmd, **k: seen.update(cmd=cmd) or _Result(stdout="[]"), |
| 214 | ) |
| 215 | brightdata.run_pipeline("amazon_product_reviews", ["https://a/dp/X", 50], timeout=60) |
| 216 | cmd = seen["cmd"] |
| 217 | assert cmd[:3] == ["brightdata", "pipelines", "amazon_product_reviews"] |
| 218 | # Positionals sit after the option fence, in the order supplied. |
| 219 | assert cmd[cmd.index("--") + 1:] == ["https://a/dp/X", "50"] |
| 220 | assert "--json" in cmd |
| 221 | |
| 222 | |
| 223 | def test_api_key_travels_in_the_environment_never_in_argv(monkeypatch): |
| 224 | """argv is not a secret channel: /proc/<pid>/cmdline is world-readable.""" |
| 225 | _installed(monkeypatch) |
| 226 | seen = {} |
| 227 | monkeypatch.setattr( |
| 228 | subproc, "run_with_timeout", |
| 229 | lambda cmd, **k: seen.update(cmd=cmd, env=k.get("env")) or _Result(stdout="[]"), |
| 230 | ) |
| 231 | monkeypatch.delenv("BRIGHTDATA_API_KEY", raising=False) |
| 232 | brightdata.run_pipeline( |
| 233 | "amazon_product", ["u"], timeout=30, config={"BRIGHTDATA_API_KEY": "dummy-key"} |
| 234 | ) |
| 235 | assert "-k" not in seen["cmd"] |
| 236 | assert "dummy-key" not in " ".join(seen["cmd"]) |
| 237 | assert seen["env"]["BRIGHTDATA_API_KEY"] == "dummy-key" |
| 238 | |
| 239 | |
| 240 | def test_no_key_means_the_child_simply_inherits_the_parent_env(monkeypatch): |
| 241 | _installed(monkeypatch) |
| 242 | seen = {} |
| 243 | monkeypatch.setattr( |
| 244 | subproc, "run_with_timeout", |
| 245 | lambda cmd, **k: seen.update(env=k.get("env")) or _Result(stdout="[]"), |
| 246 | ) |
| 247 | brightdata.run_pipeline("amazon_product", ["u"], timeout=30, config={}) |
| 248 | assert seen["env"] is None |
| 249 | |
| 250 | |
| 251 | def test_positional_params_are_fenced_behind_a_double_dash(monkeypatch): |
| 252 | """A keyword beginning with '-' must not be parsed as an option.""" |
| 253 | _installed(monkeypatch) |
| 254 | seen = {} |
| 255 | monkeypatch.setattr( |
| 256 | subproc, "run_with_timeout", |
| 257 | lambda cmd, **k: seen.update(cmd=cmd) or _Result(stdout="[]"), |
| 258 | ) |
| 259 | brightdata.run_pipeline("amazon_product_search", ["--help", "https://a"], timeout=30) |
| 260 | cmd = seen["cmd"] |
| 261 | assert "--" in cmd |
| 262 | assert cmd.index("--") < cmd.index("--help") |
| 263 | |
| 264 | |
| 265 | def test_the_key_is_scrubbed_out_of_surfaced_stderr(monkeypatch): |
| 266 | """Auth failures are exactly where a CLI echoes the rejected key back.""" |
| 267 | _installed(monkeypatch) |
| 268 | logged = [] |
| 269 | monkeypatch.setattr(brightdata.log, "source_log", lambda n, m, **k: logged.append(m)) |
| 270 | monkeypatch.setattr( |
| 271 | subproc, "run_with_timeout", |
| 272 | lambda *a, **k: _Result(returncode=1, stderr="Error: 401 for key dummy-key"), |
| 273 | ) |
| 274 | out = brightdata.run_pipeline( |
| 275 | "amazon_product", ["u"], timeout=30, config={"BRIGHTDATA_API_KEY": "dummy-key"} |
| 276 | ) |
| 277 | assert "dummy-key" not in out["error"] |
| 278 | assert not any("dummy-key" in m for m in logged) |
| 279 | |
| 280 | |
| 281 | # ------------------------------------------------------------- warnings |
| 282 | |
| 283 | |
| 284 | def test_credit_and_auth_warnings_pass_through_verbatim(monkeypatch): |
| 285 | _installed(monkeypatch) |
| 286 | warned = [] |
| 287 | monkeypatch.setattr(brightdata.log, "source_log", lambda name, msg, **k: warned.append(msg)) |
| 288 | stderr = ( |
| 289 | "Triggering pipeline collection for amazon_product_search...\n" |
| 290 | "Status: running - polling again (attempt 1/600)\n" |
| 291 | "Warning: only 12 credits remaining on your free tier\n" |
| 292 | ) |
| 293 | monkeypatch.setattr( |
| 294 | subproc, "run_with_timeout", lambda *a, **k: _Result(stdout="[]", stderr=stderr) |
| 295 | ) |
| 296 | brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 297 | assert "Warning: only 12 credits remaining on your free tier" in warned |
| 298 | # Routine polling narration stays out of the log. |
| 299 | assert not any("polling again" in m for m in warned) |
| 300 | |
| 301 | |
| 302 | def test_source_log_is_called_with_tty_only_false(monkeypatch): |
| 303 | """Repo rule: tty_only=True silently drops output in non-TTY hosts.""" |
| 304 | _installed(monkeypatch) |
| 305 | kwargs = {} |
| 306 | monkeypatch.setattr( |
| 307 | brightdata.log, "source_log", lambda name, msg, **k: kwargs.update(k) |
| 308 | ) |
| 309 | monkeypatch.setattr( |
| 310 | subproc, "run_with_timeout", lambda *a, **k: _Result(returncode=1, stderr="Error: 401") |
| 311 | ) |
| 312 | brightdata.run_pipeline("amazon_product_search", ["x"], timeout=30) |
| 313 | assert kwargs.get("tty_only") is False |
| 314 |