| 1 | """Diverse-topic evaluation harness for the three default-on Printing Press |
| 2 | sources (arxiv, techmeme, trustpilot). |
| 3 | |
| 4 | Two layers: |
| 5 | |
| 6 | 1. Deterministic fire/quiet matrix over five diverse topics drawn from real |
| 7 | run history. Asserts each source's *gating* decision without calling any CLI |
| 8 | -- this is the part that proves "Trustpilot fires only on company topics" and |
| 9 | "arXiv stays quiet on stale/off-topic matches". |
| 10 | |
| 11 | 2. Opt-in --live mode (set LAST30DAYS_EVAL_LIVE=1) that actually invokes the |
| 12 | installed CLIs against the same topics, prints a sample for human review, and |
| 13 | asserts the negative controls stay quiet. Skipped by default so CI stays |
| 14 | deterministic and offline. |
| 15 | """ |
| 16 | |
| 17 | from __future__ import annotations |
| 18 | |
| 19 | import os |
| 20 | import shutil |
| 21 | from datetime import datetime, timedelta, timezone |
| 22 | |
| 23 | import pytest |
| 24 | |
| 25 | from lib import arxiv, techmeme, trustpilot |
| 26 | |
| 27 | |
| 28 | # Five diverse topics from real run history, with the expected fire/quiet call |
| 29 | # for each source. None = "not asserted deterministically" (depends on live |
| 30 | # corpus; checked only in --live mode). |
| 31 | TOPICS = { |
| 32 | "AI coding agents": {"arxiv": True, "techmeme": True, "trustpilot": False}, |
| 33 | "agent memory": {"arxiv": True, "techmeme": True, "trustpilot": False}, |
| 34 | "Nothing Phone": {"arxiv": None, "techmeme": None, "trustpilot": True}, |
| 35 | "ChowNow": {"arxiv": None, "techmeme": None, "trustpilot": True}, |
| 36 | "Golden State Warriors": {"arxiv": False, "techmeme": None, "trustpilot": False}, |
| 37 | } |
| 38 | |
| 39 | NOW = datetime(2026, 6, 27, tzinfo=timezone.utc) |
| 40 | |
| 41 | |
| 42 | # ---- Layer 1: deterministic gating matrix ---- |
| 43 | |
| 44 | @pytest.mark.parametrize("topic,expected", [(t, e["trustpilot"]) for t, e in TOPICS.items()]) |
| 45 | def test_trustpilot_brand_gate_matrix(topic, expected): |
| 46 | """Trustpilot's brand-shape gate is the primary quiet-keeper.""" |
| 47 | assert trustpilot.is_brand_shaped(topic) is expected |
| 48 | |
| 49 | |
| 50 | def test_arxiv_quiet_on_offtopic_stale_match(): |
| 51 | """The "Golden State Warriors" control resolves to a 2017 stats paper; |
| 52 | arXiv's recency cutoff drops it, keeping the source quiet off-topic.""" |
| 53 | stale = "2017-06-12T12:00:00Z" |
| 54 | resp = {"results": [{ |
| 55 | "id": "http://arxiv.org/abs/1706.03442v2", |
| 56 | "title": "Do Steph Curry and Klay Thompson Have Hot Hands?", |
| 57 | "summary": "An analysis of Golden State Warriors shooting.", |
| 58 | "published": stale, |
| 59 | "links": [{"rel": "alternate", "href": "https://arxiv.org/abs/1706.03442v2"}], |
| 60 | }]} |
| 61 | assert arxiv.parse_arxiv_response(resp, query="Golden State Warriors", today=NOW) == [] |
| 62 | |
| 63 | |
| 64 | def test_arxiv_fires_on_recent_ontopic(): |
| 65 | recent = (NOW - timedelta(days=20)).strftime("%Y-%m-%dT12:00:00Z") |
| 66 | resp = {"results": [{ |
| 67 | "id": "http://arxiv.org/abs/2606.1v1", |
| 68 | "title": "Is Agent Memory a Database?", |
| 69 | "summary": "Rethinking data foundations for long-term AI agent memory.", |
| 70 | "published": recent, |
| 71 | "authors": [{"name": "A"}], |
| 72 | "links": [{"rel": "alternate", "href": "https://arxiv.org/abs/2606.1v1"}], |
| 73 | }]} |
| 74 | items = arxiv.parse_arxiv_response(resp, query="agent memory", today=NOW) |
| 75 | assert len(items) == 1 |
| 76 | |
| 77 | |
| 78 | def test_techmeme_drops_header_rows_keeps_stories(): |
| 79 | resp = {"results": [ |
| 80 | {"num": 1, "source": "techcrunch.com", "headline": "TechCrunch", "link": "http://techcrunch.com/"}, |
| 81 | {"num": 2, "source": "techcrunch.com", |
| 82 | "headline": "OpenAI ships a new coding agent for developers today", |
| 83 | "link": "https://www.techmeme.com/x"}, |
| 84 | ]} |
| 85 | items = techmeme.parse_techmeme_response(resp, query="AI coding agents") |
| 86 | assert len(items) == 1 |
| 87 | assert items[0]["title"].startswith("OpenAI") |
| 88 | |
| 89 | |
| 90 | # ---- Layer 2: opt-in live exploration ---- |
| 91 | |
| 92 | _LIVE = os.environ.get("LAST30DAYS_EVAL_LIVE", "").strip().lower() in ("1", "true", "yes") |
| 93 | |
| 94 | |
| 95 | def _have(binary: str) -> bool: |
| 96 | return shutil.which(binary) is not None |
| 97 | |
| 98 | |
| 99 | @pytest.mark.skipif(not _LIVE, reason="set LAST30DAYS_EVAL_LIVE=1 to run live source exploration") |
| 100 | def test_live_fire_matrix(capsys): |
| 101 | """Live: run the real CLIs against the diverse topics, print samples, and |
| 102 | assert the negative controls stay quiet.""" |
| 103 | fd, td = "2026-05-28", "2026-06-27" |
| 104 | lines = [] |
| 105 | for topic, expected in TOPICS.items(): |
| 106 | lines.append(f"\n=== {topic} ===") |
| 107 | |
| 108 | if _have("arxiv-pp-cli"): |
| 109 | items = arxiv.parse_arxiv_response( |
| 110 | arxiv.search_arxiv(topic, fd, td), query=topic) |
| 111 | lines.append(f" arXiv: {len(items)} papers") |
| 112 | for it in items[:2]: |
| 113 | lines.append(f" - {it['title'][:80]}") |
| 114 | if expected["arxiv"] is False: |
| 115 | assert items == [], f"arXiv should stay quiet on {topic!r}" |
| 116 | |
| 117 | if _have("techmeme-pp-cli"): |
| 118 | items = techmeme.parse_techmeme_response( |
| 119 | techmeme.search_techmeme(topic, fd, td), query=topic) |
| 120 | lines.append(f" Techmeme: {len(items)} headlines") |
| 121 | for it in items[:2]: |
| 122 | lines.append(f" - {it['title'][:80]}") |
| 123 | |
| 124 | if _have("trustpilot-pp-cli"): |
| 125 | items = trustpilot.parse_trustpilot_response( |
| 126 | trustpilot.search_trustpilot(topic, fd, td), query=topic) |
| 127 | lines.append(f" Trustpilot: {len(items)} companies") |
| 128 | for it in items[:1]: |
| 129 | lines.append(f" - {it['title'][:80]}") |
| 130 | if expected["trustpilot"] is False: |
| 131 | assert items == [], f"Trustpilot should stay quiet on {topic!r}" |
| 132 | |
| 133 | with capsys.disabled(): |
| 134 | print("\n".join(lines)) |
| 135 |