| 1 | """Tests for the shared keyless-Reddit throttle (U4) and 429 in-lane retry.""" |
| 2 | |
| 3 | import os |
| 4 | from unittest import mock |
| 5 | |
| 6 | from lib import http, reddit_listing, reddit_rss, render, schema |
| 7 | |
| 8 | |
| 9 | class TestRateLimiter: |
| 10 | def test_burst_does_not_sleep(self): |
| 11 | # A full bucket lets `burst` calls through immediately. |
| 12 | limiter = http.RateLimiter(rate_per_sec=5.0, burst=3) |
| 13 | with mock.patch.object(http.time, "monotonic", return_value=100.0), \ |
| 14 | mock.patch.object(http.time, "sleep") as slept: |
| 15 | limiter.acquire() |
| 16 | limiter.acquire() |
| 17 | limiter.acquire() |
| 18 | slept.assert_not_called() |
| 19 | |
| 20 | def test_sleeps_when_bucket_empty(self): |
| 21 | # burst=1: first call passes, second (same instant) must wait ~1/rate. |
| 22 | limiter = http.RateLimiter(rate_per_sec=2.0, burst=1) |
| 23 | times = iter([100.0, 100.0, 100.0, 100.5]) |
| 24 | with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \ |
| 25 | mock.patch.object(http.time, "sleep") as slept: |
| 26 | limiter.acquire() # consumes the one token |
| 27 | limiter.acquire() # bucket empty -> sleep, then refilled token consumed |
| 28 | slept.assert_called() |
| 29 | waited = slept.call_args.args[0] |
| 30 | assert abs(waited - 0.5) < 1e-6 # (1 token deficit) / 2 per sec |
| 31 | |
| 32 | def test_refill_over_time_avoids_sleep(self): |
| 33 | limiter = http.RateLimiter(rate_per_sec=2.0, burst=1) |
| 34 | # Second call 1s later: bucket refilled (2/s * 1s capped at burst=1) -> no sleep. |
| 35 | times = iter([100.0, 101.0]) |
| 36 | with mock.patch.object(http.time, "monotonic", side_effect=lambda: next(times)), \ |
| 37 | mock.patch.object(http.time, "sleep") as slept: |
| 38 | limiter.acquire() |
| 39 | limiter.acquire() |
| 40 | slept.assert_not_called() |
| 41 | |
| 42 | |
| 43 | class TestRedditKeylessGetText: |
| 44 | def test_acquires_limiter_then_delegates(self): |
| 45 | with mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire") as acq, \ |
| 46 | mock.patch.object(http, "get_text", return_value="body") as gt: |
| 47 | out = http.reddit_keyless_get_text("https://www.reddit.com/x.rss", accept="application/atom+xml") |
| 48 | assert out == "body" |
| 49 | acq.assert_called_once() |
| 50 | gt.assert_called_once() |
| 51 | |
| 52 | def test_reddit_rss_routes_through_throttle(self): |
| 53 | # The RSS tier must use the throttled helper, not raw get_text. |
| 54 | with mock.patch.object(reddit_rss.http, "reddit_keyless_get_text", return_value=None) as throttled: |
| 55 | reddit_rss.search_rss("test query") |
| 56 | assert throttled.called |
| 57 | |
| 58 | |
| 59 | class TestRedditKeylessRateKnob: |
| 60 | def test_default_rate_is_one_per_sec_small_burst(self): |
| 61 | limiter = http.make_reddit_keyless_limiter(environ={}) |
| 62 | assert limiter.rate == 1.0 |
| 63 | assert limiter.capacity == 2 |
| 64 | |
| 65 | def test_env_override(self): |
| 66 | limiter = http.make_reddit_keyless_limiter( |
| 67 | environ={http.REDDIT_KEYLESS_RATE_ENV: "0.25"} |
| 68 | ) |
| 69 | assert limiter.rate == 0.25 |
| 70 | assert limiter.capacity == 2 |
| 71 | |
| 72 | def test_invalid_and_nonpositive_fall_back_to_default(self): |
| 73 | for raw in ("fast", "", "-1", "0", "nan", "inf"): |
| 74 | limiter = http.make_reddit_keyless_limiter( |
| 75 | environ={http.REDDIT_KEYLESS_RATE_ENV: raw} |
| 76 | ) |
| 77 | assert limiter.rate == http.DEFAULT_REDDIT_KEYLESS_RATE, raw |
| 78 | |
| 79 | def test_process_env_syncs_onto_shared_limiter(self, monkeypatch): |
| 80 | monkeypatch.setattr(http.REDDIT_KEYLESS_LIMITER, "rate", 1.0) |
| 81 | monkeypatch.setenv(http.REDDIT_KEYLESS_RATE_ENV, "0.5") |
| 82 | with mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 83 | mock.patch.object(http, "get_text", return_value="ok"): |
| 84 | http.reddit_keyless_get_text("https://www.reddit.com/x.rss") |
| 85 | assert http.REDDIT_KEYLESS_LIMITER.rate == 0.5 |
| 86 | |
| 87 | def test_env_file_value_is_exported_for_limiter(self, tmp_path, monkeypatch): |
| 88 | from lib import env |
| 89 | |
| 90 | config_file = tmp_path / ".env" |
| 91 | config_file.write_text(f"{http.REDDIT_KEYLESS_RATE_ENV}=0.25\n", encoding="utf-8") |
| 92 | config_file.chmod(0o600) |
| 93 | monkeypatch.setattr(env, "CONFIG_DIR", tmp_path) |
| 94 | monkeypatch.setattr(env, "CONFIG_FILE", config_file) |
| 95 | monkeypatch.setenv("LAST30DAYS_CONFIG_DIR", str(tmp_path)) |
| 96 | monkeypatch.delenv(http.REDDIT_KEYLESS_RATE_ENV, raising=False) |
| 97 | monkeypatch.chdir(tmp_path) |
| 98 | with mock.patch.object(env, "_load_keychain", return_value={}), \ |
| 99 | mock.patch.object(env, "_load_pass", return_value={}): |
| 100 | config = env.get_config() |
| 101 | assert config[http.REDDIT_KEYLESS_RATE_ENV] == "0.25" |
| 102 | assert http.parse_reddit_keyless_rate( |
| 103 | config[http.REDDIT_KEYLESS_RATE_ENV] |
| 104 | ) == 0.25 |
| 105 | assert os.environ.get(http.REDDIT_KEYLESS_RATE_ENV) == "0.25" |
| 106 | |
| 107 | |
| 108 | def _record_status(code: int, reason: str) -> None: |
| 109 | http._record_failure(http.HTTPError(f"HTTP {code}: {reason}", code)) |
| 110 | |
| 111 | |
| 112 | class TestRedditKeyless429Retry: |
| 113 | def test_429_then_200_recovers_via_single_retry(self): |
| 114 | bodies = [None, "<feed xmlns='http://www.w3.org/2005/Atom'/>"] |
| 115 | |
| 116 | def fake_get(*_args, **_kwargs): |
| 117 | val = bodies.pop(0) |
| 118 | if val is None: |
| 119 | _record_status(429, "Too Many Requests") |
| 120 | return val |
| 121 | |
| 122 | with mock.patch.object(http, "get_text", side_effect=fake_get), \ |
| 123 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire") as acq, \ |
| 124 | mock.patch.object(http.time, "sleep") as slept, \ |
| 125 | mock.patch.object(http.random, "uniform", return_value=0.0): |
| 126 | text, err = http.reddit_keyless_get_text_retry_429( |
| 127 | "https://www.reddit.com/search.rss" |
| 128 | ) |
| 129 | assert text.startswith("<feed") |
| 130 | assert err is None |
| 131 | assert acq.call_count == 2 |
| 132 | slept.assert_called_once() |
| 133 | assert bodies == [] |
| 134 | |
| 135 | def test_429_then_429_records_failure_and_stops(self): |
| 136 | def fake_get(*_args, **_kwargs): |
| 137 | _record_status(429, "Too Many Requests") |
| 138 | return None |
| 139 | |
| 140 | with mock.patch.object(http, "get_text", side_effect=fake_get) as gt, \ |
| 141 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 142 | mock.patch.object(http.time, "sleep"), \ |
| 143 | mock.patch.object(http.random, "uniform", return_value=0.0), \ |
| 144 | http.capture_failures() as failures: |
| 145 | text, err = http.reddit_keyless_get_text_retry_429( |
| 146 | "https://www.reddit.com/search.rss" |
| 147 | ) |
| 148 | assert text is None |
| 149 | assert err is not None and "429" in err |
| 150 | assert gt.call_count == 2 |
| 151 | assert any(f.status_code == 429 for f in failures) |
| 152 | |
| 153 | def test_non_429_miss_is_not_retried(self): |
| 154 | def fake_get(*_args, **_kwargs): |
| 155 | _record_status(403, "Forbidden") |
| 156 | return None |
| 157 | |
| 158 | with mock.patch.object(http, "get_text", side_effect=fake_get) as gt, \ |
| 159 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 160 | mock.patch.object(http.time, "sleep") as slept, \ |
| 161 | http.capture_failures() as failures: |
| 162 | text, err = http.reddit_keyless_get_text_retry_429( |
| 163 | "https://www.reddit.com/search.rss" |
| 164 | ) |
| 165 | assert text is None |
| 166 | assert "403" in (err or "") |
| 167 | assert gt.call_count == 1 |
| 168 | slept.assert_not_called() |
| 169 | assert any(f.status_code == 403 for f in failures) |
| 170 | |
| 171 | def test_rss_fetch_feed_recovers_after_one_429(self): |
| 172 | feed = ( |
| 173 | '<feed xmlns="http://www.w3.org/2005/Atom"><entry>' |
| 174 | "<title>Recovered</title>" |
| 175 | '<link href="https://www.reddit.com/r/test/comments/abc/x/" />' |
| 176 | "<updated>2026-05-20T00:00:00+00:00</updated></entry></feed>" |
| 177 | ) |
| 178 | bodies = [None, feed] |
| 179 | |
| 180 | def fake_get(*_args, **_kwargs): |
| 181 | val = bodies.pop(0) |
| 182 | if val is None: |
| 183 | _record_status(429, "Too Many Requests") |
| 184 | return val |
| 185 | |
| 186 | with mock.patch.object(http, "get_text", side_effect=fake_get), \ |
| 187 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 188 | mock.patch.object(http.time, "sleep"): |
| 189 | posts = reddit_rss._fetch_feed( |
| 190 | "https://www.reddit.com/search.rss", "Recovered" |
| 191 | ) |
| 192 | assert len(posts) == 1 |
| 193 | assert posts[0]["title"] == "Recovered" |
| 194 | |
| 195 | def test_listing_fetch_recovers_after_one_429(self): |
| 196 | from pathlib import Path |
| 197 | |
| 198 | html = ( |
| 199 | Path(__file__).resolve().parent.parent |
| 200 | / "fixtures" |
| 201 | / "reddit_listing_cards_sample.html" |
| 202 | ).read_text(encoding="utf-8") |
| 203 | bodies = [None, html] |
| 204 | |
| 205 | def fake_get(*_args, **_kwargs): |
| 206 | val = bodies.pop(0) |
| 207 | if val is None: |
| 208 | _record_status(429, "Too Many Requests") |
| 209 | return val |
| 210 | |
| 211 | with mock.patch.object(http, "get_text", side_effect=fake_get), \ |
| 212 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 213 | mock.patch.object(http.time, "sleep"): |
| 214 | items, error = reddit_listing._fetch_one_with_status( |
| 215 | "technology", "hot", "netherlands" |
| 216 | ) |
| 217 | assert error is None |
| 218 | assert items |
| 219 | assert bodies == [] |
| 220 | |
| 221 | def test_listing_fetch_records_double_429(self): |
| 222 | def fake_get(*_args, **_kwargs): |
| 223 | _record_status(429, "Too Many Requests") |
| 224 | return None |
| 225 | |
| 226 | with mock.patch.object(http, "get_text", side_effect=fake_get) as gt, \ |
| 227 | mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire"), \ |
| 228 | mock.patch.object(http.time, "sleep"), \ |
| 229 | http.capture_failures() as failures: |
| 230 | items, error = reddit_listing._fetch_one_with_status( |
| 231 | "technology", "hot", "x" |
| 232 | ) |
| 233 | assert items == [] |
| 234 | assert error is not None and "429" in error |
| 235 | assert gt.call_count == 2 |
| 236 | assert any(f.status_code == 429 for f in failures) |
| 237 | |
| 238 | |
| 239 | class TestPartialOutcomeWording: |
| 240 | def test_rate_limited_partial_does_not_read_as_cutoff(self): |
| 241 | outcome = schema.SourceOutcome( |
| 242 | source="reddit", |
| 243 | state=schema.PARTIAL, |
| 244 | items_returned=8, |
| 245 | detail="HTTP 429: Too Many Requests", |
| 246 | ) |
| 247 | text = render._format_outcome(outcome) |
| 248 | assert "partial after" not in text |
| 249 | assert "8 items returned" in text |
| 250 | assert "some requests rate-limited" in text |
| 251 | assert "HTTP 429" in text |
| 252 | |
| 253 | def test_non_rate_limit_partial_keeps_count_without_429_claim(self): |
| 254 | outcome = schema.SourceOutcome( |
| 255 | source="instagram", |
| 256 | state=schema.PARTIAL, |
| 257 | items_returned=1, |
| 258 | detail="HTTP 400: Bad Request", |
| 259 | ) |
| 260 | text = render._format_outcome(outcome) |
| 261 | assert "partial after" not in text |
| 262 | assert "1 item returned" in text |
| 263 | assert "rate-limited" not in text |
| 264 | assert "HTTP 400" in text |
| 265 |