| 1 | """Tests for crowd-vote weighting in the fun judge (Best Takes). |
| 2 | |
| 3 | Covers: |
| 4 | - U2: signals.top_comment_vote_signal (per-platform normalized [0,1] vote signal) |
| 5 | - U1: rerank._extract_comment_text_scored / _build_fun_prompt (votes in the LLM prompt) |
| 6 | - U4: rerank._apply_single_fun_fallback (fallback uses the vote signal) |
| 7 | - U3: render._render_best_takes (relevance-gated, confidence-scaled, bounded nudge) |
| 8 | """ |
| 9 | |
| 10 | import pytest |
| 11 | |
| 12 | from lib import schema, signals |
| 13 | from lib.rerank import ( |
| 14 | _apply_single_fun_fallback, |
| 15 | _build_fun_prompt, |
| 16 | _extract_comment_text_scored, |
| 17 | ) |
| 18 | from lib import render |
| 19 | |
| 20 | |
| 21 | def _candidate( |
| 22 | *, |
| 23 | source: str = "reddit", |
| 24 | title: str = "Some Title", |
| 25 | snippet: str = "", |
| 26 | top_comments=None, |
| 27 | fun_score=None, |
| 28 | local_relevance: float = 0.8, |
| 29 | explanation: str | None = None, |
| 30 | final_score: float = 50.0, |
| 31 | engagement=0.0, |
| 32 | ) -> schema.Candidate: |
| 33 | source_items = [] |
| 34 | if top_comments is not None: |
| 35 | source_items.append( |
| 36 | schema.SourceItem( |
| 37 | item_id="si-1", |
| 38 | source=source, |
| 39 | title=title, |
| 40 | body="", |
| 41 | url=f"https://example.com/{source}/1", |
| 42 | metadata={"top_comments": top_comments}, |
| 43 | ) |
| 44 | ) |
| 45 | c = schema.Candidate( |
| 46 | candidate_id="c-1", |
| 47 | item_id="i-1", |
| 48 | source=source, |
| 49 | title=title, |
| 50 | url=f"https://example.com/{source}/1", |
| 51 | snippet=snippet, |
| 52 | subquery_labels=["q1"], |
| 53 | native_ranks={source: 1}, |
| 54 | local_relevance=local_relevance, |
| 55 | freshness=50, |
| 56 | engagement=engagement, |
| 57 | source_quality=0.5, |
| 58 | rrf_score=0.01, |
| 59 | source_items=source_items, |
| 60 | ) |
| 61 | c.fun_score = fun_score |
| 62 | c.explanation = explanation |
| 63 | c.final_score = final_score |
| 64 | return c |
| 65 | |
| 66 | |
| 67 | # --------------------------------------------------------------------------- U2 |
| 68 | |
| 69 | class TestTopCommentVoteSignal: |
| 70 | def test_cross_platform_comparable(self): |
| 71 | """A 66-upvote Reddit comment and a 22,821-like TikTok comment land on a |
| 72 | comparable scale -- neither platform dominates by raw count.""" |
| 73 | reddit = _candidate(source="reddit", top_comments=[{"body": "x", "score": 66}]) |
| 74 | tiktok = _candidate(source="tiktok", top_comments=[{"body": "x", "score": 22821}]) |
| 75 | rs = signals.top_comment_vote_signal(reddit) |
| 76 | ts = signals.top_comment_vote_signal(tiktok) |
| 77 | # Both substantial, and TikTok's 22k does not swamp Reddit's 66 by 100x. |
| 78 | assert 0.3 < rs < 1.0 |
| 79 | assert 0.3 < ts <= 1.0 |
| 80 | assert ts / max(rs, 1e-9) < 2.5 |
| 81 | |
| 82 | def test_zero_or_missing_votes(self): |
| 83 | assert signals.top_comment_vote_signal(_candidate(top_comments=[{"body": "x"}])) == 0.0 |
| 84 | assert signals.top_comment_vote_signal(_candidate(top_comments=[{"body": "x", "score": 0}])) == 0.0 |
| 85 | assert signals.top_comment_vote_signal(_candidate(top_comments=None)) == 0.0 |
| 86 | |
| 87 | def test_monotonic_within_platform(self): |
| 88 | low = _candidate(source="reddit", top_comments=[{"body": "x", "score": 10}]) |
| 89 | high = _candidate(source="reddit", top_comments=[{"body": "x", "score": 5000}]) |
| 90 | assert signals.top_comment_vote_signal(high) > signals.top_comment_vote_signal(low) |
| 91 | |
| 92 | def test_bounded_zero_to_one(self): |
| 93 | for score in (1, 100, 6100, 39000, 10_000_000): |
| 94 | sig = signals.top_comment_vote_signal( |
| 95 | _candidate(source="tiktok", top_comments=[{"body": "x", "score": score}]) |
| 96 | ) |
| 97 | assert 0.0 <= sig <= 1.0 |
| 98 | |
| 99 | |
| 100 | # --------------------------------------------------------------------------- U1 |
| 101 | |
| 102 | class TestVotesInFunPrompt: |
| 103 | def test_scored_extract_prefixes_vote_count(self): |
| 104 | c = _candidate(top_comments=[{"body": "the crowd loved this", "score": 14200}]) |
| 105 | text = _extract_comment_text_scored(c) |
| 106 | assert "[+14200]" in text |
| 107 | assert "the crowd loved this" in text |
| 108 | |
| 109 | def test_scored_extract_no_score_no_prefix(self): |
| 110 | c = _candidate(top_comments=[{"body": "no score here"}]) |
| 111 | text = _extract_comment_text_scored(c) |
| 112 | assert "no score here" in text |
| 113 | assert "[+" not in text |
| 114 | |
| 115 | def test_scored_extract_negative_score_no_malformed_prefix(self): |
| 116 | """A negative score must not emit a misleading [+-N] prefix (Greptile #592).""" |
| 117 | c = _candidate(top_comments=[{"body": "downvoted line", "score": -3}]) |
| 118 | text = _extract_comment_text_scored(c) |
| 119 | assert "downvoted line" in text |
| 120 | assert "[+-3]" not in text |
| 121 | assert "[+" not in text |
| 122 | |
| 123 | def test_prompt_contains_traction_guidance(self): |
| 124 | c = _candidate(top_comments=[{"body": "lol", "score": 99}]) |
| 125 | prompt = _build_fun_prompt("test topic", [c]) |
| 126 | assert "[+99]" in prompt |
| 127 | # The judge is told votes = traction, not funniness. |
| 128 | assert "TRACTION" in prompt |
| 129 | |
| 130 | |
| 131 | # --------------------------------------------------------------------------- U4 |
| 132 | |
| 133 | class TestFallbackUsesVoteSignal: |
| 134 | def test_high_vote_top_comment_scores_higher(self): |
| 135 | low = _candidate(source="reddit", title="t", snippet="s", top_comments=[{"body": "ok", "score": 5}]) |
| 136 | high = _candidate(source="reddit", title="t", snippet="s", top_comments=[{"body": "ok", "score": 4000}]) |
| 137 | _apply_single_fun_fallback(low) |
| 138 | _apply_single_fun_fallback(high) |
| 139 | assert high.fun_score > low.fun_score |
| 140 | |
| 141 | def test_fallback_bounded(self): |
| 142 | c = _candidate(source="tiktok", top_comments=[{"body": "lol bruh", "score": 10_000_000}]) |
| 143 | _apply_single_fun_fallback(c) |
| 144 | assert 0.0 <= c.fun_score <= 100.0 |
| 145 | |
| 146 | def test_fallback_without_votes_still_scores(self): |
| 147 | c = _candidate(title="hilarious bit", snippet="", top_comments=[{"body": "bruh"}]) |
| 148 | _apply_single_fun_fallback(c) |
| 149 | assert c.fun_score is not None and c.fun_score > 0 |
| 150 | |
| 151 | |
| 152 | # --------------------------------------------------------------------------- U3 |
| 153 | |
| 154 | def _render(cands, level="medium"): |
| 155 | p = render._FUN_LEVELS[level] |
| 156 | return "\n".join( |
| 157 | render._render_best_takes(cands, limit=p["limit"], threshold=p["threshold"], vote_weight=p["vote_weight"]) |
| 158 | ) |
| 159 | |
| 160 | |
| 161 | class TestBestTakesVoteWeighting: |
| 162 | # Best Takes displays the candidate TITLE when the top comment body is longer |
| 163 | # than it, so these tests use distinctive long titles + longer comment bodies |
| 164 | # and assert on the titles. |
| 165 | def test_relevance_gate_excludes_entity_miss(self): |
| 166 | """An off-topic-but-viral comment (entity-miss) never reaches Best Takes, |
| 167 | even with a huge vote count and a high fun_score.""" |
| 168 | offtopic = _candidate( |
| 169 | source="youtube", title="JamesMayReactsClip", fun_score=85, |
| 170 | top_comments=[{"body": "James May is a great man and a true friend", "score": 39000}], |
| 171 | explanation="fallback-local-score (entity-miss demotion)", final_score=0.0, |
| 172 | ) |
| 173 | ontopic_a = _candidate(source="reddit", title="VelcroShirtReturn", fun_score=88, |
| 174 | top_comments=[{"body": "the velcro on my camp chair bag ate it", "score": 66}]) |
| 175 | ontopic_b = _candidate(source="reddit", title="BuriedInBaggies", fun_score=82, |
| 176 | top_comments=[{"body": "when i die bury me in my baggies", "score": 73}]) |
| 177 | out = _render([offtopic, ontopic_a, ontopic_b]) |
| 178 | assert "JamesMayReactsClip" not in out |
| 179 | assert "VelcroShirtReturn" in out |
| 180 | |
| 181 | def test_zero_final_score_excluded(self): |
| 182 | dead = _candidate(title="DeadZeroScore", fun_score=90, final_score=0.0, |
| 183 | top_comments=[{"body": "high voted but score zero item", "score": 100}]) |
| 184 | a = _candidate(title="FunnyAlpha", fun_score=88, top_comments=[{"body": "funny alpha line here", "score": 50}]) |
| 185 | b = _candidate(title="FunnyBeta", fun_score=85, top_comments=[{"body": "funny beta line here", "score": 50}]) |
| 186 | out = _render([dead, a, b]) |
| 187 | assert "DeadZeroScore" not in out |
| 188 | |
| 189 | def test_funny_floor_blocks_high_vote_unfunny(self): |
| 190 | """A high-voted but unfunny comment (fun below the floor) is excluded.""" |
| 191 | rant = _candidate(title="LawyerRant", fun_score=15, |
| 192 | top_comments=[{"body": "pay a lawyer to send a letter to their legal dept", "score": 1720}]) |
| 193 | a = _candidate(title="FunnyAlpha", fun_score=80, top_comments=[{"body": "funny alpha line here", "score": 40}]) |
| 194 | b = _candidate(title="FunnyBeta", fun_score=78, top_comments=[{"body": "funny beta line here", "score": 40}]) |
| 195 | out = _render([rant, a, b]) |
| 196 | assert "LawyerRant" not in out |
| 197 | |
| 198 | def test_votes_promote_funnyish_over_threshold(self): |
| 199 | """A funny-ish on-topic comment (fun 55) with strong on-topic votes clears |
| 200 | the medium threshold (70) via the effective score -- the empty-Best-Takes fix.""" |
| 201 | promoted = _candidate(source="reddit", title="PromotedGem", fun_score=55, local_relevance=1.0, |
| 202 | top_comments=[{"body": "a promoted gem the crowd loved", "score": 6000}]) |
| 203 | other = _candidate(source="reddit", title="AlreadyFunny", fun_score=72, |
| 204 | top_comments=[{"body": "already funny on its own merit", "score": 10}]) |
| 205 | # Without votes, PromotedGem (fun 55) would not clear medium's 70 threshold. |
| 206 | baseline = _render([ |
| 207 | _candidate(source="reddit", title="PromotedGem", fun_score=55, |
| 208 | top_comments=[{"body": "a promoted gem the crowd loved"}]), |
| 209 | other, |
| 210 | ]) |
| 211 | assert "PromotedGem" not in baseline |
| 212 | out = _render([promoted, other]) |
| 213 | assert "PromotedGem" in out |
| 214 | |
| 215 | def test_bounded_amplification_does_not_overturn_humor_gap(self): |
| 216 | """fun 90 / tiny votes still ranks above fun 55 / max votes at medium.""" |
| 217 | gem = _candidate(source="reddit", title="GhostOfYvonGem", fun_score=90, local_relevance=1.0, |
| 218 | top_comments=[{"body": "the ghost of yvon weeps for this funko pop civilization", "score": 26}]) |
| 219 | viral = _candidate(source="tiktok", title="MidButViral", fun_score=55, local_relevance=1.0, |
| 220 | top_comments=[{"body": "mid but extremely viral comment here", "score": 50000}]) |
| 221 | out = _render([gem, viral]) |
| 222 | assert out.index("GhostOfYvonGem") < out.index("MidButViral") |
| 223 | |
| 224 | def test_meaningful_at_medium_orders_by_votes(self): |
| 225 | """Equal fun_score, different votes -> ordered by votes, and the effect is |
| 226 | more than a hairline tiebreaker at medium.""" |
| 227 | hi = _candidate(source="reddit", title="HighVotedItem", fun_score=72, local_relevance=1.0, |
| 228 | top_comments=[{"body": "high voted comment body here", "score": 5000}]) |
| 229 | lo = _candidate(source="reddit", title="LowVotedItem", fun_score=72, local_relevance=1.0, |
| 230 | top_comments=[{"body": "low voted comment body here", "score": 5}]) |
| 231 | eff_hi = render._effective_fun_score(hi, render._FUN_LEVELS["medium"]["vote_weight"]) |
| 232 | eff_lo = render._effective_fun_score(lo, render._FUN_LEVELS["medium"]["vote_weight"]) |
| 233 | assert eff_hi - eff_lo > 5.0 # meaningful, not a tiebreaker |
| 234 | out = _render([hi, lo]) |
| 235 | assert out.index("HighVotedItem") < out.index("LowVotedItem") |
| 236 | |
| 237 | def test_level_scaling_high_more_than_low(self): |
| 238 | c = _candidate(source="reddit", fun_score=72, local_relevance=1.0, |
| 239 | top_comments=[{"body": "x", "score": 5000}]) |
| 240 | eff_low = render._effective_fun_score(c, render._FUN_LEVELS["low"]["vote_weight"]) |
| 241 | eff_high = render._effective_fun_score(c, render._FUN_LEVELS["high"]["vote_weight"]) |
| 242 | base = c.fun_score |
| 243 | assert (eff_high - base) > (eff_low - base) |
| 244 | |
| 245 | def test_confidence_scaling(self): |
| 246 | weight = render._FUN_LEVELS["medium"]["vote_weight"] |
| 247 | high_conf = _candidate(source="reddit", fun_score=72, local_relevance=1.0, |
| 248 | top_comments=[{"body": "x", "score": 5000}]) |
| 249 | low_conf = _candidate(source="reddit", fun_score=72, local_relevance=0.2, |
| 250 | top_comments=[{"body": "x", "score": 5000}]) |
| 251 | assert render._effective_fun_score(high_conf, weight) > render._effective_fun_score(low_conf, weight) |
| 252 | |
| 253 | def test_crowd_boost_tag_when_votes_lift_ranking(self): |
| 254 | """A vote-boosted item is flagged '+crowd' so a lower-fun item ranking |
| 255 | above a higher-fun one reads correctly (Greptile #592).""" |
| 256 | boosted = _candidate(source="reddit", title="BoostedItem", fun_score=72, local_relevance=1.0, |
| 257 | top_comments=[{"body": "crowd loved this line a lot", "score": 6000}]) |
| 258 | plain = _candidate(source="reddit", title="PlainItem", fun_score=90, |
| 259 | top_comments=[{"body": "very funny on its own merit here"}]) |
| 260 | out = _render([boosted, plain]) |
| 261 | assert "+crowd" in out |
| 262 | # The plain (no-votes) item carries no crowd tag. |
| 263 | plain_line = [ln for ln in out.splitlines() if "PlainItem" in ln][0] |
| 264 | assert "+crowd" not in plain_line |
| 265 | |
| 266 | def test_no_votes_preserves_pure_fun_ordering(self): |
| 267 | """With no comment votes, Best Takes ordering matches pure fun_score (no regression).""" |
| 268 | a = _candidate(source="reddit", title="FunnyAlpha", fun_score=90, top_comments=[{"body": "funny alpha line here"}]) |
| 269 | b = _candidate(source="reddit", title="FunnyBeta", fun_score=80, top_comments=[{"body": "funny beta line here"}]) |
| 270 | out = _render([a, b]) |
| 271 | assert out.index("FunnyAlpha") < out.index("FunnyBeta") |
| 272 |