| 1 | """Tests for relevance-blended Top Community Comments ranking (issue #641). |
| 2 | |
| 3 | Off-topic high-traffic threads should not crowd out on-topic lower-voted threads. |
| 4 | """ |
| 5 | |
| 6 | import math |
| 7 | |
| 8 | from lib import render, schema, signals |
| 9 | |
| 10 | |
| 11 | def _candidate( |
| 12 | *, |
| 13 | source: str = "reddit", |
| 14 | title: str = "Post", |
| 15 | url: str = "https://example.com/1", |
| 16 | local_relevance: float = 0.8, |
| 17 | top_comments=None, |
| 18 | ) -> schema.Candidate: |
| 19 | source_items = [] |
| 20 | if top_comments is not None: |
| 21 | source_items.append( |
| 22 | schema.SourceItem( |
| 23 | item_id=f"si-{url}", |
| 24 | source=source, |
| 25 | title=title, |
| 26 | body="", |
| 27 | url=url, |
| 28 | metadata={"top_comments": top_comments}, |
| 29 | ) |
| 30 | ) |
| 31 | c = schema.Candidate( |
| 32 | candidate_id=f"c-{url}", |
| 33 | item_id=f"i-{url}", |
| 34 | source=source, |
| 35 | title=title, |
| 36 | url=url, |
| 37 | snippet="", |
| 38 | subquery_labels=["q1"], |
| 39 | native_ranks={source: 1}, |
| 40 | local_relevance=local_relevance, |
| 41 | freshness=50, |
| 42 | engagement=0, |
| 43 | source_quality=0.5, |
| 44 | rrf_score=0.01, |
| 45 | source_items=source_items, |
| 46 | ) |
| 47 | c.final_score = 50.0 |
| 48 | return c |
| 49 | |
| 50 | |
| 51 | def _make_report(*candidates) -> schema.Report: |
| 52 | return schema.Report( |
| 53 | topic="test topic", |
| 54 | range_from="2026-01-01", |
| 55 | range_to="2026-01-31", |
| 56 | generated_at="2026-01-31T00:00:00Z", |
| 57 | provider_runtime=schema.ProviderRuntime( |
| 58 | reasoning_provider="test", planner_model="test", rerank_model="test", |
| 59 | ), |
| 60 | query_plan=schema.QueryPlan( |
| 61 | intent="general", freshness_mode="balanced_recent", |
| 62 | cluster_mode="debate", raw_topic="test", subqueries=[], |
| 63 | source_weights={}, |
| 64 | ), |
| 65 | ranked_candidates=list(candidates), |
| 66 | clusters=[], |
| 67 | items_by_source={}, |
| 68 | errors_by_source={}, |
| 69 | ) |
| 70 | |
| 71 | |
| 72 | class TestTopCommentsRelevanceGate: |
| 73 | def test_below_floor_candidates_excluded(self): |
| 74 | """Candidates below RELEVANCE_FLOOR must not contribute comments.""" |
| 75 | off_topic = _candidate( |
| 76 | url="https://example.com/offtopic", |
| 77 | local_relevance=0.05, # below RELEVANCE_FLOOR=0.1 |
| 78 | top_comments=[{"body": "viral off-topic comment nine thousand votes", "score": 9000}], |
| 79 | ) |
| 80 | on_topic = [ |
| 81 | _candidate( |
| 82 | url=f"https://example.com/ontopic-{idx}", |
| 83 | local_relevance=0.7, |
| 84 | top_comments=[{"body": f"directly about the topic comment {idx}", "score": 50 - idx}], |
| 85 | ) |
| 86 | for idx in range(5) |
| 87 | ] |
| 88 | report = _make_report(off_topic, *on_topic) |
| 89 | lines = render._render_top_comments(report) |
| 90 | combined = "\n".join(lines) |
| 91 | assert "viral off-topic comment" not in combined |
| 92 | assert "directly about the topic" in combined |
| 93 | |
| 94 | def test_high_relevance_beats_low_relevance_despite_moderate_vote_lead(self): |
| 95 | """On-topic comment (high relevance) should outrank a moderately more-voted off-topic one. |
| 96 | |
| 97 | Vote normalization: reddit ref = log1p(2000) ≈ 7.6 |
| 98 | - 200 upvotes: log1p(200)/7.6 ≈ 0.70 |
| 99 | - 50 upvotes: log1p(50)/7.6 ≈ 0.52 |
| 100 | |
| 101 | Blended scores (60% vote, 40% relevance): |
| 102 | - viral_offtopic (200 votes, rel=0.12): 0.6*0.70 + 0.4*0.12 = 0.468 |
| 103 | - on_topic (50 votes, rel=0.90): 0.6*0.52 + 0.4*0.90 = 0.672 |
| 104 | """ |
| 105 | viral_offtopic = _candidate( |
| 106 | url="https://example.com/viral", |
| 107 | local_relevance=0.12, # just above RELEVANCE_FLOOR |
| 108 | top_comments=[{"body": "Viral off-topic comment with moderate vote lead", "score": 200}], |
| 109 | ) |
| 110 | ontopic_modest = _candidate( |
| 111 | url="https://example.com/ontopic", |
| 112 | local_relevance=0.90, |
| 113 | top_comments=[{"body": "On-topic comment fewer votes but on-topic", "score": 50}], |
| 114 | ) |
| 115 | report = _make_report(viral_offtopic, ontopic_modest) |
| 116 | lines = render._render_top_comments(report) |
| 117 | combined = "\n".join(lines) |
| 118 | assert combined.index("On-topic comment") < combined.index("Viral off-topic") |
| 119 | |
| 120 | def test_sparse_topics_keep_comments_when_floor_would_remove_everything(self): |
| 121 | """Sparse niche topics should still surface comments below the soft relevance floor.""" |
| 122 | low_relevance_a = _candidate( |
| 123 | url="https://example.com/sparse-a", |
| 124 | local_relevance=0.08, |
| 125 | top_comments=[{"body": "Sparse topic comment still relevant enough to show", "score": 70}], |
| 126 | ) |
| 127 | low_relevance_b = _candidate( |
| 128 | url="https://example.com/sparse-b", |
| 129 | local_relevance=0.07, |
| 130 | top_comments=[{"body": "Another sparse topic comment below floor", "score": 50}], |
| 131 | ) |
| 132 | low_relevance_c = _candidate( |
| 133 | url="https://example.com/sparse-c", |
| 134 | local_relevance=0.05, |
| 135 | top_comments=[{"body": "Third sparse topic comment below floor", "score": 30}], |
| 136 | ) |
| 137 | report = _make_report(low_relevance_a, low_relevance_b, low_relevance_c) |
| 138 | |
| 139 | lines = render._render_top_comments(report) |
| 140 | combined = "\n".join(lines) |
| 141 | |
| 142 | assert "Sparse topic comment" in combined |
| 143 | assert "Another sparse topic comment" in combined |
| 144 | |
| 145 | def test_maximally_viral_low_relevance_comment_loses_to_on_topic_comment(self): |
| 146 | """A clamped 5000-vote comment should still lose to a highly relevant 50-vote one.""" |
| 147 | viral_low_relevance = _candidate( |
| 148 | url="https://example.com/max-viral", |
| 149 | local_relevance=0.12, |
| 150 | top_comments=[{"body": "Maximally viral but barely related comment", "score": 5000}], |
| 151 | ) |
| 152 | on_topic = _candidate( |
| 153 | url="https://example.com/on-topic-boundary", |
| 154 | local_relevance=0.90, |
| 155 | top_comments=[{"body": "Highly relevant lower voted boundary comment", "score": 50}], |
| 156 | ) |
| 157 | report = _make_report(viral_low_relevance, on_topic) |
| 158 | |
| 159 | lines = render._render_top_comments(report) |
| 160 | combined = "\n".join(lines) |
| 161 | |
| 162 | assert combined.index("Highly relevant") < combined.index("Maximally viral") |
| 163 | |
| 164 | def test_no_duplicate_comments(self): |
| 165 | """Same comment body must appear at most once even if two candidates share the text.""" |
| 166 | cand_a = _candidate( |
| 167 | url="https://example.com/a", |
| 168 | local_relevance=0.8, |
| 169 | top_comments=[{"body": "Shared comment body appears in two different threads", "score": 100}], |
| 170 | ) |
| 171 | cand_b = _candidate( |
| 172 | url="https://example.com/b", |
| 173 | local_relevance=0.8, |
| 174 | top_comments=[ |
| 175 | {"body": "Shared comment body appears in two different threads", "score": 80}, |
| 176 | {"body": "Second distinct comment that is different from the first", "score": 60}, |
| 177 | ], |
| 178 | ) |
| 179 | report = _make_report(cand_a, cand_b) |
| 180 | lines = render._render_top_comments(report) |
| 181 | occurrences = sum(1 for line in lines if "Shared comment body" in line) |
| 182 | assert occurrences == 1 |
| 183 |