返回 last30days-skill
test_amazon.py
根目录 / tests / test_amazon.py
1 """Tests for the Amazon source: discovery, enrichment, stats, footer (U2, U3).
2
3 Fixtures mirror live payload shapes pulled 2026-08-13, including the three
4 fields that arrive doubled and the fact that ``max_reviews`` is a ceiling
5 rather than a quota.
6
7 Nothing here spawns a subprocess or touches the network.
8 """
9
10 from __future__ import annotations
11
12 import sys
13 from datetime import datetime, timedelta, timezone
14 from pathlib import Path
15
16 import pytest
17
18 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
19
20 from lib import amazon # noqa: E402
21
22
23 TODAY = datetime(2026, 8, 13, tzinfo=timezone.utc)
24 DOMAIN = "https://www.amazon.com"
25
26
27 def _days_ago(n: int) -> str:
28 return (TODAY - timedelta(days=n)).date().isoformat()
29
30
31 def search_record(**over):
32 base = {
33 "asin": "B0AAA00001",
34 "url": "https://www.amazon.com/Bentgo-Chill/dp/B0AAA00001/ref=sr_1_1?dib=xyz",
35 "name": "Chill Max Leak-Proof XL Bento-Style Lunch Box | Ice Pack Included",
36 "brand": "Bentgo",
37 "sponsored": "false",
38 "rating": 4.4,
39 "num_ratings": 459,
40 "final_price": 39.99,
41 "currency": "USD",
42 }
43 base.update(over)
44 return base
45
46
47 def review_record(days_ago: int, rating: int, **over):
48 review_date = TODAY - timedelta(days=days_ago)
49 # ``%-d`` is POSIX-only and raises on Windows. Build the day component
50 # from the datetime so this fixture exercises the same payload everywhere.
51 posted = f"{review_date.strftime('%B')} {review_date.day}, {review_date.year}"
52 base = {
53 "review_id": f"R{days_ago}{rating}",
54 # Live shape: the date is doubled and prose-wrapped.
55 "review_posted_date": f"{posted}Reviewed in the United States on {posted}",
56 "review_header": "Great box!Great box!",
57 "review_text": f"Review body from {days_ago} days ago.",
58 "rating": rating,
59 "helpful_count": 0,
60 "is_verified": True,
61 "is_amazon_vine": False,
62 "author_name": "A Buyer",
63 "product_rating": 4.4,
64 "product_rating_count": 459,
65 "product_rating_object": {
66 "one_star": 28, "two_star": 9, "three_star": 28,
67 "four_star": 60, "five_star": 335,
68 },
69 }
70 base.update(over)
71 return base
72
73
74 # ------------------------------------------------------- field repair
75
76
77 class TestFieldRepair:
78 def test_doubled_header_is_repaired(self):
79 assert amazon.undouble("Best Box!Best Box!") == "Best Box!"
80
81 def test_comma_doubled_badge_is_repaired(self):
82 assert amazon.undouble("Verified Purchase, Verified Purchase") == "Verified Purchase"
83
84 def test_genuinely_repetitive_text_survives(self):
85 assert amazon.undouble("Great great product") == "Great great product"
86 assert amazon.undouble("Buy one, buy two") == "Buy one, buy two"
87
88 def test_prose_wrapped_date_yields_the_leading_date(self):
89 raw = "August 3, 2026Reviewed in the United States on August 3, 2026"
90 assert amazon.parse_review_date(raw) == "2026-08-03"
91
92 def test_unparseable_date_is_none_not_an_exception(self):
93 assert amazon.parse_review_date("") is None
94 assert amazon.parse_review_date("sometime last year") is None
95 assert amazon.parse_review_date(None) is None
96
97
98 class TestShortName:
99 def test_takes_the_segment_before_the_delimiter(self):
100 assert amazon.short_name("Chill Max XL | Ice Pack Included") == "Chill Max XL"
101
102 def test_strips_a_leading_brand_when_present(self):
103 assert amazon.short_name("Weber Spirit E-325", "Weber") == "Spirit E-325"
104
105 def test_clips_long_names_on_a_word_boundary(self):
106 out = amazon.short_name("Kids Prints Leak-Proof 5-Compartment Bento-Style Box")
107 assert len(out) <= amazon.SHORT_NAME_MAX
108 assert not out.endswith("-")
109 assert " " not in out[-1:]
110
111
112 # ---------------------------------------------------------- discovery
113
114
115 class TestDiscovery:
116 def test_parses_products_with_rating_and_price(self):
117 products = amazon.parse_search_response({"records": [search_record()]}, "bentgo lunch box")
118 assert len(products) == 1
119 product = products[0]
120 assert product["rating"] == 4.4
121 assert product["price"] == 39.99
122 assert product["num_ratings"] == 459
123 assert product["brand"] == "Bentgo"
124
125 def test_urls_are_canonicalized_to_dp_asin(self):
126 """Live URLs carry 200+ chars of session-scoped tracking tail."""
127 products = amazon.parse_search_response({"records": [search_record()]}, "bentgo")
128 assert products[0]["url"] == "https://www.amazon.com/dp/B0AAA00001"
129
130 def test_duplicate_asins_collapse_keeping_the_richest_record(self):
131 records = [
132 search_record(num_ratings=84),
133 search_record(num_ratings=459),
134 search_record(num_ratings=12),
135 ]
136 products = amazon.parse_search_response({"records": records}, "bentgo")
137 assert len(products) == 1
138 assert products[0]["num_ratings"] == 459
139
140 def test_off_keyword_products_are_gated_out(self):
141 records = [
142 search_record(),
143 search_record(asin="B0ZZZ00001", name="Cordless Drill Driver Kit", brand="DeWalt",
144 url="https://www.amazon.com/dp/B0ZZZ00001"),
145 ]
146 products = amazon.parse_search_response({"records": records}, "bentgo lunch box")
147 assert [p["asin"] for p in products] == ["B0AAA00001"]
148
149 def test_non_amazon_and_non_https_urls_are_rejected(self):
150 records = [
151 search_record(asin="B000000001", url="http://www.amazon.com/dp/B000000001"),
152 search_record(asin="B000000002", url="https://evil.example.com/dp/B2"),
153 search_record(asin="B000000003", url="https://www.amazon.com/dp/B000000003"),
154 ]
155 products = amazon.parse_search_response({"records": records}, "bentgo chill max lunch box")
156 assert [p["asin"] for p in products] == ["B000000003"]
157
158 def test_alternate_marketplace_domain_is_honored(self):
159 record = search_record(url="https://www.amazon.co.uk/dp/B0AAA00001")
160 products = amazon.parse_search_response(
161 {"records": [record]}, "bentgo", domain="https://www.amazon.co.uk"
162 )
163 assert products and products[0]["url"].startswith("https://www.amazon.co.uk/dp/")
164
165 def test_sponsored_string_is_recorded_as_bool_never_filtered(self):
166 """R4: the flag is metadata only. Filtering can blank the lane."""
167 records = [
168 search_record(asin="B000000001", sponsored="true", url="https://www.amazon.com/dp/B000000001"),
169 search_record(asin="B000000002", sponsored="false", url="https://www.amazon.com/dp/B000000002"),
170 ]
171 products = amazon.parse_search_response({"records": records}, "bentgo chill max lunch box")
172 assert len(products) == 2
173 assert {p["asin"]: p["sponsored"] for p in products} == {"B000000001": True, "B000000002": False}
174
175 def test_error_envelope_yields_no_products(self):
176 assert amazon.parse_search_response({"records": [], "error": "401"}, "x") == []
177
178
179 class TestTargetSelection:
180 def _pool(self):
181 return amazon.parse_search_response(
182 {
183 "records": [
184 search_record(asin="C000000001", brand="Fimibuke", num_ratings=901,
185 name="60oz Leakproof Bento Lunch Box",
186 url="https://www.amazon.com/dp/C000000001"),
187 search_record(asin="B000000001", brand="Bentgo", num_ratings=821,
188 name="Kids Insulated Lunch Bag",
189 url="https://www.amazon.com/dp/B000000001"),
190 search_record(asin="B000000002", brand="Bentgo", num_ratings=710,
191 name="MicroSteel Bento Lunch Box",
192 url="https://www.amazon.com/dp/B000000002"),
193 search_record(asin="B000000003", brand="Bentgo", num_ratings=623,
194 name="Classic Stackable Lunch Box",
195 url="https://www.amazon.com/dp/B000000003"),
196 ]
197 },
198 "bentgo lunch box",
199 )
200
201 def test_brand_topic_excludes_competitors_buying_the_keyword(self):
202 """The guard against paying to review a rival's product."""
203 targets = amazon.select_enrichment_targets(self._pool(), limit=3, keyword="bentgo lunch box")
204 assert [t["asin"] for t in targets] == ["B000000001", "B000000002", "B000000003"]
205 assert all(t["brand"] == "Bentgo" for t in targets)
206
207 def test_category_topic_stays_unfiltered_and_ranks_on_merit(self):
208 targets = amazon.select_enrichment_targets(self._pool(), limit=3, keyword="kids lunch box")
209 assert targets[0]["asin"] == "C000000001"
210
211 def test_infer_brand_needs_the_keyword_to_name_it(self):
212 pool = self._pool()
213 assert amazon.infer_brand(pool, "bentgo lunch box") == "Bentgo"
214 assert amazon.infer_brand(pool, "best kids lunch box") == ""
215
216 def test_near_identical_variants_do_not_take_two_pulls(self):
217 pool = amazon.parse_search_response(
218 {
219 "records": [
220 search_record(asin="V000000001", num_ratings=901, name="60oz Leakproof Box | Blue",
221 url="https://www.amazon.com/dp/V000000001"),
222 search_record(asin="V000000002", num_ratings=900, name="60oz Leakproof Box | Pink",
223 url="https://www.amazon.com/dp/V000000002"),
224 search_record(asin="V000000003", num_ratings=500, name="Chill Max XL Box",
225 url="https://www.amazon.com/dp/V000000003"),
226 ]
227 },
228 "bentgo lunch box",
229 )
230 targets = amazon.select_enrichment_targets(pool, limit=2, keyword="bentgo lunch box")
231 assert [t["asin"] for t in targets] == ["V000000001", "V000000003"]
232
233 def test_zero_limit_selects_nothing(self):
234 assert amazon.select_enrichment_targets(self._pool(), limit=0) == []
235
236
237 # --------------------------------------------------------- enrichment
238
239
240 class TestReviewParsing:
241 def test_reviews_become_comments_with_stats(self):
242 response = {"records": [review_record(3, 5), review_record(10, 4)]}
243 comments, stats = amazon.parse_reviews(response)
244 assert len(comments) == 2
245 assert stats["product_rating"] == 4.4
246 assert stats["product_rating_count"] == 459
247 assert stats["star_distribution"]["five_star"] == 335
248
249 def test_comments_carry_the_keys_remap_would_strip(self):
250 """Rating, date, and verified are exactly what this source needs."""
251 comments, _ = amazon.parse_reviews({"records": [review_record(3, 5)]})
252 comment = comments[0]
253 assert set(comment) >= {"score", "excerpt", "rating", "date", "verified"}
254 assert comment["rating"] == 5
255 assert comment["date"] == _days_ago(3)
256
257 def test_doubled_header_is_repaired_in_the_comment_title(self):
258 comments, _ = amazon.parse_reviews({"records": [review_record(3, 5)]})
259 assert comments[0]["title"] == "Great box!"
260
261 def test_woven_sample_is_newest_first(self):
262 """R2a: recency is enforced client-side, not assumed from the API."""
263 response = {"records": [
264 review_record(400, 5), review_record(2, 3), review_record(45, 4),
265 ]}
266 comments, _ = amazon.parse_reviews(response)
267 assert [c["date"] for c in comments] == [_days_ago(2), _days_ago(45), _days_ago(400)]
268
269 def test_empty_payload_is_not_an_error(self):
270 assert amazon.parse_reviews({"records": []}) == ([], {})
271
272
273 class TestEnrichmentLane:
274 def _products(self, n=4):
275 return [
276 {"asin": f"B00000000{i}", "url": f"https://www.amazon.com/dp/B00000000{i}",
277 "name": f"Product {i}", "short_name": f"Product {i}",
278 "brand": "Bentgo", "num_ratings": 900 - i, "rating": 4.4}
279 for i in range(n)
280 ]
281
282 def test_quick_depth_spawns_no_review_pulls(self):
283 calls = []
284 out, status = amazon.enrich_with_reviews(
285 self._products(), depth="quick",
286 fetcher=lambda url: calls.append(url) or {"records": []},
287 )
288 assert calls == []
289 assert len(out) == 4
290 assert status is None # quick depth is not a degraded outcome
291
292 def test_default_depth_pulls_exactly_three(self):
293 calls = []
294 _, status = amazon.enrich_with_reviews(
295 self._products(), depth="default",
296 fetcher=lambda url: calls.append(url) or {"records": [review_record(2, 5)]},
297 )
298 assert len(calls) == 3
299 assert status is None
300
301 def test_deep_depth_pulls_five(self):
302 calls = []
303 _, status = amazon.enrich_with_reviews(
304 self._products(6), depth="deep",
305 fetcher=lambda url: calls.append(url) or {"records": []},
306 )
307 assert len(calls) == 5
308
309 def test_cap_is_fifty_per_pull(self):
310 seen = {}
311
312 def fetcher(url):
313 return {"records": []}
314
315 # The cap reaches the CLI through fetch_reviews; assert the constant
316 # and the plumbed default together.
317 assert amazon.MAX_REVIEWS == 50
318 import lib.brightdata as bd
319 original = bd.run_pipeline
320 bd.run_pipeline = lambda p, params, **k: seen.update(params=params) or {"records": []}
321 try:
322 amazon.fetch_reviews("https://www.amazon.com/dp/B000000001")
323 finally:
324 bd.run_pipeline = original
325 assert seen["params"] == ["https://www.amazon.com/dp/B000000001", "50"]
326
327 def test_reviews_attach_to_the_right_product(self):
328 out, status = amazon.enrich_with_reviews(
329 self._products(3), depth="default",
330 fetcher=lambda url: {"records": [review_record(2, 5, review_id=url)]},
331 )
332 assert all(p.get("top_comments") for p in out)
333 assert out[0]["product_rating_count"] == 459
334 assert status is None
335
336 def test_one_failing_pull_does_not_discard_its_siblings(self):
337 def fetcher(url):
338 if url.endswith("B000000001"):
339 return {"records": [], "error": "snapshot failed"}
340 return {"records": [review_record(2, 5)]}
341
342 out, status = amazon.enrich_with_reviews(self._products(3), depth="default", fetcher=fetcher)
343 by_asin = {p["asin"]: p for p in out}
344 assert not by_asin["B000000001"].get("top_comments")
345 assert by_asin["B000000000"].get("top_comments")
346 assert by_asin["B000000002"].get("top_comments")
347 assert status is None # partial success is not reported as status
348
349 def test_one_raising_pull_does_not_kill_the_lane(self):
350 def fetcher(url):
351 if url.endswith("B000000001"):
352 raise RuntimeError("boom")
353 return {"records": [review_record(2, 5)]}
354
355 out, status = amazon.enrich_with_reviews(self._products(3), depth="default", fetcher=fetcher)
356 assert sum(1 for p in out if p.get("top_comments")) == 2
357 assert status is None # partial success is not reported as status
358
359 def test_dropped_straggler_keeps_its_product_with_search_stats(self):
360 """A deadline drop must never delete the product from the report.
361
362 Use a patched short LANE_DEADLINE to trigger the straggler case without
363 relying on crumb budgets (which now skip the lane entirely).
364 """
365 import time as _time
366 import unittest.mock
367
368 def slow(url):
369 if url.endswith("B000000000"):
370 _time.sleep(3)
371 return {"records": [review_record(2, 5)]}
372
373 # Patch LANE_DEADLINE to 1s so the slow product times out but other
374 # products have time to complete. elapsed=0 keeps the full 1s budget.
375 with unittest.mock.patch.object(amazon, "LANE_DEADLINE", 1):
376 out, status = amazon.enrich_with_reviews(
377 self._products(2), depth="default", fetcher=slow,
378 elapsed=0.0,
379 )
380 by_asin = {p["asin"]: p for p in out}
381 assert set(by_asin) == {"B000000000", "B000000001"}
382 # Search-record stats survive on the dropped product.
383 assert by_asin["B000000000"]["rating"] == 4.4
384 assert by_asin["B000000000"]["num_ratings"] == 900
385 # One straggler dropped, so status should be "review lane timed out"
386 # (since all 3 pulls didn't complete, but some did)
387 # Actually, B000000001 completed, so status is None
388 # Let me check: if completed_count > 0, status is None
389
390 def test_exhausted_wall_clock_skips_the_lane_entirely(self):
391 calls = []
392 out, status = amazon.enrich_with_reviews(
393 self._products(), depth="default",
394 fetcher=lambda url: calls.append(url) or {"records": []},
395 elapsed=amazon.FOREGROUND_CONTRACT,
396 )
397 assert calls == []
398 assert status == "review lane skipped (budget 0s)"
399
400 def test_crumb_budget_skips_not_fires_doomed_pulls(self):
401 """Regression test for the Bentgo bug: elapsed=269 must skip, not fire doomed 11s pulls.
402
403 The bug: a multi-source run took 269s before reaching Amazon enrichment,
404 leaving only 11s of budget (300 - 269 - 20 = 11). All Bright Data pulls
405 timed out (cli_timeout = max(5, timeout-10) = 1s), spending credits
406 without returning reviews.
407
408 The fix: crumb budgets (below MIN_USEFUL_REVIEW_BUDGET=90) return 0,
409 skipping the lane entirely instead of firing doomed short pulls.
410 """
411 calls = []
412 out, status = amazon.enrich_with_reviews(
413 self._products(), depth="default",
414 fetcher=lambda url: calls.append(url) or {"records": []},
415 elapsed=269.0,
416 )
417 # Fetcher should never be called
418 assert calls == []
419 # Products keep their search stats (no top_comments)
420 assert all(not p.get("top_comments") for p in out)
421 assert all(p.get("rating") == 4.4 for p in out)
422 assert all(p.get("num_ratings") for p in out)
423 # Status should indicate the lane was skipped
424 assert status == "review lane skipped (budget 0s)"
425
426 def test_early_elapsed_gets_full_budget(self):
427 """A quick search (40s elapsed) should get the full LANE_DEADLINE budget."""
428 timeouts_seen = []
429
430 def capturing_fetcher(url):
431 return {"records": [review_record(2, 5)]}
432
433 # Patch fetch_reviews to capture the timeout
434 original_fetch = amazon.fetch_reviews
435 captured_timeout = []
436
437 def mock_fetch(url, *, max_reviews=50, config=None, timeout=180):
438 captured_timeout.append(timeout)
439 return {"records": [review_record(2, 5)]}
440
441 amazon.fetch_reviews = mock_fetch
442 try:
443 out, status = amazon.enrich_with_reviews(
444 self._products(1), depth="default",
445 elapsed=40.0,
446 )
447 finally:
448 amazon.fetch_reviews = original_fetch
449
450 assert captured_timeout, "fetch_reviews was not called"
451 # elapsed=40 → remaining = 300-40-20 = 240 → clamped to LANE_DEADLINE=180
452 assert captured_timeout[0] == amazon.LANE_DEADLINE
453 assert status is None
454
455 def test_all_pulls_dropped_reports_timed_out_status(self):
456 """When all pulls drop (none complete), status should be 'review lane timed out'."""
457 import time as _time
458 import unittest.mock
459
460 def very_slow(url):
461 _time.sleep(5) # Longer than the deadline
462 return {"records": [review_record(2, 5)]}
463
464 # Use a very short deadline so all pulls time out
465 with unittest.mock.patch.object(amazon, "LANE_DEADLINE", 1):
466 with unittest.mock.patch.object(amazon, "MIN_USEFUL_REVIEW_BUDGET", 1):
467 out, status = amazon.enrich_with_reviews(
468 self._products(2), depth="default", fetcher=very_slow,
469 elapsed=0.0,
470 )
471
472 # No products should have top_comments (all dropped)
473 assert all(not p.get("top_comments") for p in out)
474 # Status should indicate timeout
475 assert status == "review lane timed out"
476
477
478 # --------------------------------------------------------------- stats
479
480
481 class TestStats:
482 def test_five_star_share_from_the_distribution_object(self):
483 share = amazon.five_star_share(
484 {"one_star": 28, "two_star": 9, "three_star": 28, "four_star": 60, "five_star": 335}
485 )
486 assert round(share * 100) == 73
487
488 def test_five_star_share_is_none_without_a_distribution(self):
489 assert amazon.five_star_share({}) is None
490
491 def test_recent_window_counts_only_dated_reviews_inside_it(self):
492 comments = [
493 {"date": _days_ago(2), "rating": 4},
494 {"date": _days_ago(29), "rating": 4},
495 {"date": _days_ago(31), "rating": 1},
496 {"date": None, "rating": 5},
497 ]
498 window = amazon.recent_window_stats(comments, today=TODAY)
499 assert window["recent_n"] == 2
500 assert window["recent_avg"] == 4
501
502 @pytest.mark.parametrize(
503 "ratings,expected",
504 [
505 ([1, 2, 3, 4, 5], "down"), # avg 3.0 vs 4.4
506 ([5, 5, 5, 5, 5], "up"), # avg 5.0 vs 4.4
507 ([4, 4, 5, 4, 5], "flat"), # avg 4.4 vs 4.4
508 ],
509 )
510 def test_drift_direction(self, ratings, expected):
511 product = {
512 "product_rating": 4.4,
513 "product_rating_count": 459,
514 "top_comments": [
515 {"date": _days_ago(i + 1), "rating": r} for i, r in enumerate(ratings)
516 ],
517 }
518 assert amazon.product_stats(product, today=TODAY)["drift"] == expected
519
520 def test_below_threshold_sample_renders_quiet_not_a_drift(self):
521 """Live census: a 50-cap pull can land only a handful in-window."""
522 product = {
523 "product_rating": 4.4,
524 "top_comments": [{"date": _days_ago(i + 1), "rating": 1} for i in range(4)],
525 }
526 assert amazon.product_stats(product, today=TODAY)["drift"] == "quiet"
527
528 def test_threshold_is_exactly_five(self):
529 product = {
530 "product_rating": 4.4,
531 "top_comments": [{"date": _days_ago(i + 1), "rating": 1} for i in range(5)],
532 }
533 assert amazon.product_stats(product, today=TODAY)["drift"] == "down"
534
535 def test_no_baseline_renders_new(self):
536 assert amazon.product_stats({"top_comments": []}, today=TODAY)["drift"] == "new"
537
538 def test_review_pull_rating_count_supersedes_the_search_record(self):
539 """Search counts are variant-level and can undercount 100x."""
540 stats = amazon.product_stats(
541 {"num_ratings": 84, "product_rating_count": 8446, "product_rating": 4.7},
542 today=TODAY,
543 )
544 assert stats["ratings_total"] == 8446
545
546 def test_reproduces_the_live_chill_max_reading(self):
547 """End-to-end against the real 2026-08-13 payload shape."""
548 records = (
549 [review_record(i, r) for i, r in ((1, 5), (2, 1), (5, 5), (9, 4), (11, 4))]
550 + [review_record(200, 5), review_record(400, 5)]
551 )
552 comments, stats = amazon.parse_reviews({"records": records})
553 product = {"short_name": "Chill Max XL", **stats, "top_comments": comments}
554 out = amazon.product_stats(product, today=TODAY)
555 assert out["all_time"] == 4.4
556 assert out["ratings_total"] == 459
557 assert round(out["five_star_share"] * 100) == 73
558 assert out["recent_n"] == 5
559 assert out["recent_avg"] == 3.8
560 assert out["drift"] == "down"
561
562
563 # -------------------------------------------------------------- footer
564
565
566 class TestFooterEntry:
567 def test_negative_drift_gets_the_arrow_marker(self):
568 entry = amazon.footer_entry(
569 {"short_name": "Chill Max XL", "all_time": 4.4, "recent_avg": 3.8, "drift": "down"}
570 )
571 assert entry == "Chill Max XL 4.4★→3.8★ ↓"
572
573 def test_positive_drift_gets_no_marker(self):
574 entry = amazon.footer_entry(
575 {"short_name": "Deluxe Bag", "all_time": 4.7, "recent_avg": 5.0, "drift": "up"}
576 )
577 assert entry == "Deluxe Bag 4.7★→5.0★"
578 assert "↓" not in entry
579
580 def test_quiet_state_shows_the_baseline_without_an_arrow(self):
581 entry = amazon.footer_entry(
582 {"short_name": "Genesis E-325", "all_time": 4.4, "recent_avg": None, "drift": "quiet"}
583 )
584 assert entry == "Genesis E-325 4.4★ quiet"
585 assert "→" not in entry
586
587 def test_new_state_claims_no_baseline(self):
588 entry = amazon.footer_entry({"short_name": "BLUEY Set", "all_time": None, "drift": "new"})
589 assert entry == "BLUEY Set new"
590
591 def test_quote_renders_only_on_negative_drift(self):
592 sagging = amazon.footer_entry(
593 {"short_name": "Chill Max XL", "all_time": 4.4, "recent_avg": 3.8, "drift": "down"},
594 quote="the lid jams",
595 )
596 assert sagging == 'Chill Max XL 4.4★→3.8★ ↓ "the lid jams"'
597 healthy = amazon.footer_entry(
598 {"short_name": "Deluxe Bag", "all_time": 4.7, "recent_avg": 5.0, "drift": "up"},
599 quote="the lid jams",
600 )
601 assert '"' not in healthy
602
603 def test_absent_quote_renders_the_clean_numeric_entry(self):
604 entry = amazon.footer_entry(
605 {"short_name": "Chill Max XL", "all_time": 4.4, "recent_avg": 3.8, "drift": "down"},
606 quote="",
607 )
608 assert entry == "Chill Max XL 4.4★→3.8★ ↓"
609
610
611 class _Item:
612 """Minimal SourceItem stand-in for the enrichment adapter."""
613
614 def __init__(self, asin, **meta):
615 self.source = "amazon"
616 self.url = f"https://www.amazon.com/dp/{asin}"
617 self.title = meta.get("name", asin)
618 self.metadata = {"asin": asin, "short_name": asin, "brand": "Bentgo", **meta}
619
620
621 class TestSourceItemEnrichment:
622 def test_reviews_and_stats_land_on_item_metadata(self, monkeypatch):
623 # Review records are relative to the fixture date, not the wall clock.
624 monkeypatch.setattr(amazon, "_today", lambda: TODAY)
625 items = [_Item("B000000001"), _Item("B000000002")]
626 amazon.enrich_source_items(
627 items, depth="default", keyword="bentgo lunch box",
628 fetcher=lambda url: {"records": [
629 review_record(i + 1, r) for i, r in enumerate([1, 1, 1, 1, 1])
630 ]},
631 )
632 for item in items:
633 assert item.metadata["top_comments"]
634 assert item.metadata["stats"]["drift"] == "down"
635
636 def test_non_amazon_items_are_untouched(self):
637 other = _Item("B000000001")
638 other.source = "reddit"
639 amazon.enrich_source_items([other], depth="default", fetcher=lambda url: {"records": []})
640 assert "top_comments" not in other.metadata
641
642 def test_already_enriched_items_are_not_re_pulled(self):
643 """enrich_source_items no-ops when top_comments is already set.
644
645 This is critical for the thin-retry path: Phase 1 enriches products at
646 search time, then thin retry (Phase 2b) may return the same ASINs. The
647 pipeline passes skip_amazon_enrichment=True in thin retry, so products
648 arrive at finalize without re-enrichment. Finalize calls enrich_source_items,
649 which skips already-enriched items (top_comments set) and only enriches
650 genuinely new products. This prevents duplicate Bright Data pulls.
651 """
652 item = _Item("B000000001", top_comments=[{"excerpt": "cached", "score": 0, "rating": 5, "date": None}])
653 calls = []
654 amazon.enrich_source_items(
655 [item], depth="default",
656 fetcher=lambda url: calls.append(url) or {"records": []},
657 )
658 assert calls == []
659
660 def test_quick_depth_touches_nothing(self):
661 items = [_Item("B000000001")]
662 calls = []
663 amazon.enrich_source_items(
664 items, depth="quick",
665 fetcher=lambda url: calls.append(url) or {"records": []},
666 )
667 assert calls == []
668 assert "top_comments" not in items[0].metadata
669
670
671 class TestStatsFromItem:
672 def test_uses_the_cached_block_when_enrichment_already_ran(self):
673 item = _Item("B000000001", stats={"short_name": "Cached", "drift": "up"})
674 assert amazon.stats_from_item(item)["short_name"] == "Cached"
675
676 def test_recomputes_from_metadata_when_absent(self):
677 """Mock runs and replayed fixtures skip enrichment entirely."""
678 item = _Item(
679 "B1",
680 product_rating=4.4,
681 product_rating_count=459,
682 star_distribution={"one_star": 28, "two_star": 9, "three_star": 28,
683 "four_star": 60, "five_star": 335},
684 top_comments=[{"date": _days_ago(i + 1), "rating": 1} for i in range(5)],
685 )
686 stats = amazon.stats_from_item(item, today=TODAY)
687 assert stats["drift"] == "down"
688 assert stats["all_time"] == 4.4
689 assert round(stats["five_star_share"] * 100) == 73
690
691
692 class _FooterItem:
693 def __init__(self, short_name, *, reviews=0, all_time=4.4, recent=None,
694 drift="quiet", quote=None):
695 self.source = "amazon"
696 self.url = "https://www.amazon.com/dp/X"
697 self.title = short_name
698 self.metadata = {
699 "asin": short_name,
700 "stats": {
701 "short_name": short_name, "all_time": all_time,
702 "recent_avg": recent, "drift": drift,
703 "reviews_pulled": reviews, "ratings_total": 100,
704 "five_star_share": 0.7, "recent_n": 5 if recent else 0,
705 "url": self.url,
706 },
707 }
708 if quote:
709 self.metadata["pulse_quote"] = quote
710
711
712 def _report(items, *, keyword="bentgo lunch box"):
713 from lib import schema
714 report = object.__new__(schema.Report)
715 object.__setattr__(report, "items_by_source", {"amazon": items})
716 object.__setattr__(report, "artifacts", {"amazon_query": keyword})
717 object.__setattr__(report, "source_status", {})
718 return report
719
720
721 class TestFooterLine:
722 def _line(self, items, **kw):
723 from lib import render
724 return render._amazon_footer_line(_report(items, **kw))
725
726 def test_only_sampled_products_get_a_slot(self):
727 """A dozen discovered products must not become a dozen entries."""
728 items = [_FooterItem("Sampled", reviews=20, recent=3.8, drift="down")]
729 items += [_FooterItem(f"Unsampled{i}") for i in range(9)]
730 line = self._line(items)
731 assert "Sampled 4.4★→3.8★ ↓" in line
732 assert "Unsampled" not in line
733 # The count still reports everything discovered.
734 assert line.startswith("📦 Amazon: 10 products │")
735
736 def test_duplicate_variant_names_are_collapsed(self):
737 items = [
738 _FooterItem("Kids Bento", reviews=20, recent=4.9, drift="up"),
739 _FooterItem("Kids Bento", reviews=20, recent=4.8, drift="up"),
740 ]
741 assert self._line(items).count("Kids Bento") == 1
742
743 def test_quick_depth_renders_the_inventory_form(self):
744 items = [_FooterItem("A"), _FooterItem("B")]
745 line = self._line(items)
746 assert "→" not in line
747 assert "average" in line and "ratings" in line
748
749 def test_footer_renders_no_quote_today(self):
750 """The engine renders this line before the model sees the report, so
751 there is no weave-time path for a model-written quote. Deferred."""
752 items = [_FooterItem("Sagging", reviews=20, recent=3.8, drift="down",
753 quote="the lid jams")]
754 line = self._line(items)
755 assert "↓" in line
756 assert '"' not in line
757
758 def test_footer_entry_still_accepts_a_quote_for_a_future_writer(self):
759 entry = amazon.footer_entry(
760 {"short_name": "X", "all_time": 4.4, "recent_avg": 3.8, "drift": "down"},
761 quote="the lid jams",
762 )
763 assert '"the lid jams"' in entry
764
765 def test_empty_result_names_the_keyword(self):
766 assert self._line([]) == '📦 Amazon: no products matched "bentgo lunch box"'
767
768 def test_no_line_at_all_when_the_source_never_ran(self):
769 assert self._line([], keyword="") is None
770
771 def test_malformed_asin_records_are_rejected(self):
772 """The ASIN is interpolated into a URL that is refetched and rendered."""
773 records = [
774 search_record(asin="../../etc/passwd", url="https://www.amazon.com/dp/x"),
775 search_record(asin="B0AAA0000", url="https://www.amazon.com/dp/y"), # 9 chars
776 search_record(asin="B0AAA000012", url="https://www.amazon.com/dp/z"), # 11 chars
777 search_record(asin="B0AAA00001", url="https://www.amazon.com/dp/ok"),
778 ]
779 products = amazon.parse_search_response({"records": records}, "bentgo chill max lunch box")
780 assert [p["asin"] for p in products] == ["B0AAA00001"]
781
782 def test_canonical_url_falls_back_when_the_asin_is_malformed(self):
783 original = "https://www.amazon.com/dp/legit"
784 assert amazon.canonical_product_url(original, "not-an-asin", DOMAIN) == original
785
786 def test_option_shaped_keyword_is_rejected_before_the_cli_runs(self):
787 """A leading dash would be parsed as a flag, not a search term."""
788 calls = []
789 import lib.brightdata as bd
790 original = bd.run_pipeline
791 bd.run_pipeline = lambda *a, **k: calls.append(a) or {"records": []}
792 try:
793 out = amazon.search_products("--help")
794 finally:
795 bd.run_pipeline = original
796 assert calls == []
797 assert "may not begin" in out["error"]
798
799
800 class TestReviewFindingRegressions:
801 """Regressions caught by the correctness review pass."""
802
803 def test_brand_inference_survives_mixed_casing(self):
804 """One vendor spelled two ways must not disable the guard."""
805 pool = [{"brand": "Bentgo"}, {"brand": "BENTGO"}, {"brand": "Umi"}]
806 assert amazon.infer_brand(pool, "bentgo lunch box") == "Bentgo"
807
808 def test_multi_word_brands_are_matched(self):
809 pool = [{"brand": "Hydro Flask"}, {"brand": "Iron Flask"}]
810 assert amazon.infer_brand(pool, "hydro flask water bottle") == "Hydro Flask"
811
812 def test_two_distinct_brands_in_the_keyword_stay_ambiguous(self):
813 pool = [{"brand": "Yeti"}, {"brand": "Stanley"}]
814 assert amazon.infer_brand(pool, "yeti vs stanley tumbler") == ""
815
816 def test_short_name_respects_word_boundaries_on_the_brand(self):
817 """A bare startswith() ate into sub-brands and coincidental prefixes."""
818 assert amazon.short_name("AnkerWork B600 Video Bar", "Anker") == "AnkerWork B600"
819 assert amazon.short_name("Chillax Bento Lunch Box", "Chill") == "Chillax Bento"
820
821 def test_undouble_leaves_short_repeated_words_alone(self):
822 assert amazon.undouble("ByeBye") == "ByeBye"
823 assert amazon.undouble("NoNo") == "NoNo"
824 # Real doubling of a whole headline still repairs.
825 assert amazon.undouble("Best Box!Best Box!") == "Best Box!"
826 assert amazon.undouble("Great value for money.Great value for money.") == \
827 "Great value for money."
828
829 def test_lane_budget_shrinks_as_the_run_clock_advances(self):
830 """The clamp was dead code until `elapsed` was threaded through."""
831 # Fresh run gets full LANE_DEADLINE
832 assert amazon._remaining_lane_budget(0.0) == amazon.LANE_DEADLINE
833 # Mid-run (100s elapsed) still has 180s leftover (300-100-20=180), clamped to LANE_DEADLINE
834 mid = amazon._remaining_lane_budget(100.0)
835 assert mid == amazon.LANE_DEADLINE
836 # Below floor (300-200-20=80 < MIN_USEFUL_REVIEW_BUDGET=90) → 0
837 assert amazon._remaining_lane_budget(200.0) == 0
838 # Way past contract → 0
839 assert amazon._remaining_lane_budget(295.0) == 0
840
841 def test_lane_budget_floor_prevents_doomed_pulls(self):
842 """Crumb budgets return 0, not the crumbs.
843
844 This is the fix for the Bentgo bug: elapsed=269 left only 11s of budget,
845 causing Bright Data pulls to time out (cli_timeout = max(5, timeout-10)
846 → 1s timeout). Now any budget below MIN_USEFUL_REVIEW_BUDGET returns 0.
847 """
848 # elapsed=269 → remaining = 300-269-20 = 11 < MIN_USEFUL=90 → 0
849 assert amazon._remaining_lane_budget(269.0) == 0
850 # Just above floor: 300-190-20=90 == MIN_USEFUL → 90 (not 0)
851 assert amazon._remaining_lane_budget(190.0) == amazon.MIN_USEFUL_REVIEW_BUDGET
852 # Just below floor: 300-191-20=89 < MIN_USEFUL=90 → 0
853 assert amazon._remaining_lane_budget(191.0) == 0
854
855 def test_lane_budget_constants_are_sane(self):
856 """Guard against accidental constant drift breaking the logic."""
857 assert amazon.MIN_USEFUL_REVIEW_BUDGET == 90
858 assert amazon.LANE_DEADLINE == 180
859 assert amazon.MIN_USEFUL_REVIEW_BUDGET < amazon.LANE_DEADLINE
860
861 def test_enrichment_refreshes_the_variant_level_rating_count(self):
862 """Search counts undercount badly; the pull's count is authoritative."""
863 item = _Item("B000000001", name="Deluxe Bag", num_ratings=84)
864 item.title = "Bentgo Deluxe Bag - 4.7/5 (84 ratings)"
865 item.engagement = {"ratings": 84}
866 amazon.enrich_source_items(
867 [item], depth="default",
868 fetcher=lambda url: {"records": [
869 review_record(i + 1, 5, product_rating=4.7, product_rating_count=8446)
870 for i in range(5)
871 ]},
872 )
873 assert item.engagement["ratings"] == 8446
874 assert "8,446 ratings" in item.title
875 assert "84 ratings" not in item.title
876 assert item.metadata["stats"]["ratings_total"] == 8446
877
878
879 class TestSavedArtifactCompleteness:
880 """The saved report is the copy users keep; it must not drop evidence."""
881
882 def _saved(self, sources):
883 from lib import render, schema
884 report = object.__new__(schema.Report)
885 for field, value in {
886 "topic": "bentgo", "range_from": "2026-07-14", "range_to": "2026-08-13",
887 "generated_at": "2026-08-13", "clusters": [], "ranked_candidates": [],
888 "items_by_source": sources, "errors_by_source": {}, "source_status": {},
889 "freshness_verdicts": [], "warnings": [], "artifacts": {},
890 "library_context": [], "drill_of": None,
891 "provider_runtime": schema.ProviderRuntime(
892 reasoning_provider="local", planner_model="m", rerank_model="m",
893 ),
894 "query_plan": schema.QueryPlan(
895 intent="product", freshness_mode="balanced_recent", cluster_mode="none",
896 raw_topic="bentgo", subqueries=[],
897 source_weights={s: 1.0 for s in sources},
898 ),
899 }.items():
900 object.__setattr__(report, field, value)
901 return render.render_full(report)
902
903 def _item(self, source, item_id, engagement):
904 from lib import schema
905 return schema.SourceItem(
906 item_id=item_id, source=source, title=f"{source} item", body="b",
907 url="https://example.com", author="A", container=None,
908 published_at="2026-08-10", date_confidence="high",
909 engagement=engagement, relevance_hint=0.5, why_relevant="",
910 snippet="", metadata={},
911 )
912
913 def test_amazon_items_appear_in_the_per_source_dump(self):
914 """Regression: a hardcoded source list silently dropped this section."""
915 out = self._saved({"amazon": [self._item("amazon", "B0AAA00001", {"ratings": 459})]})
916 assert "### Amazon (1 items)" in out
917 assert "B0AAA00001" in out
918
919 def test_a_source_absent_from_the_fixed_order_still_renders(self):
920 """The list is display order, not the source registry."""
921 out = self._saved({"trustpilot": [self._item("trustpilot", "TP1", {"reviews": 12})]})
922 assert "TP1" in out
923
924 def test_engagement_is_not_blank_for_a_non_allowlisted_metric(self):
925 out = self._saved({"amazon": [self._item("amazon", "B0AAA00001", {"ratings": 459})]})
926 assert "459 ratings" in out
927
928 def test_allowlisted_sources_keep_their_existing_engagement_format(self):
929 """The fall-through must not add previously-unshown keys."""
930 out = self._saved({"reddit": [self._item(
931 "reddit", "R1", {"score": 120, "num_comments": 48, "upvote_ratio": 0.91}
932 )]})
933 assert "120 score, 48 num_comments" in out
934 assert "upvote_ratio" not in out
935
935 lines PYTHON