| 1 | """Tests for x_judge.py — corpus judging for retrieve-judge-retry.""" |
| 2 | |
| 3 | import pytest |
| 4 | |
| 5 | from skills.last30days.scripts.lib import x_judge |
| 6 | |
| 7 | |
| 8 | class TestJudgeXCorpus: |
| 9 | """Tests for judge_x_corpus.""" |
| 10 | |
| 11 | def test_empty_corpus_returns_not_flood(self): |
| 12 | """Empty corpus should not be flagged as off-topic flood.""" |
| 13 | result = x_judge.judge_x_corpus([], "Rome") |
| 14 | assert result["on_topic_ratio"] == 1.0 |
| 15 | assert not result["is_off_topic_flood"] |
| 16 | assert result["on_topic_items"] == [] |
| 17 | assert result["off_topic_items"] == [] |
| 18 | |
| 19 | def test_on_topic_corpus_passes(self): |
| 20 | """Corpus with mostly on-topic posts should pass.""" |
| 21 | items = [ |
| 22 | {"author_handle": "mamboitaliano__", "text": "Beautiful Rome sunset"}, |
| 23 | {"author_handle": "mamboitaliano__", "text": "Trevi Fountain in Rome"}, |
| 24 | {"author_handle": "mamboitaliano__", "text": "Rome Colosseum today"}, |
| 25 | {"author_handle": "turismoromaweb", "text": "Visit Rome Italy"}, |
| 26 | ] |
| 27 | result = x_judge.judge_x_corpus(items, "Rome") |
| 28 | assert result["on_topic_ratio"] >= 0.5 |
| 29 | assert not result["is_off_topic_flood"] |
| 30 | assert len(result["on_topic_items"]) >= 3 |
| 31 | |
| 32 | def test_off_topic_flood_detected(self): |
| 33 | """Corpus with mostly off-topic posts should be flagged as flood.""" |
| 34 | # Rome fixture: 8 on-topic posts, 32 off-topic posts |
| 35 | on_topic = [ |
| 36 | {"author_handle": "mamboitaliano__", "text": "Beautiful Rome sunset"}, |
| 37 | {"author_handle": "mamboitaliano__", "text": "Trevi Fountain in Rome"}, |
| 38 | {"author_handle": "mamboitaliano__", "text": "Rome Colosseum today"}, |
| 39 | {"author_handle": "turismoromaweb", "text": "Visit Rome Italy"}, |
| 40 | {"author_handle": "romecityphoto", "text": "Rome city at night"}, |
| 41 | {"author_handle": "visitrome", "text": "Rome vacation tips"}, |
| 42 | {"author_handle": "italytravel", "text": "Rome and Florence tour"}, |
| 43 | {"author_handle": "rometraveler", "text": "Rome landmarks to see"}, |
| 44 | ] |
| 45 | off_topic = [ |
| 46 | {"author_handle": "visegrad24", "text": "Zelensky addresses NATO summit"}, |
| 47 | {"author_handle": "visegrad24", "text": "Poland border update"}, |
| 48 | {"author_handle": "visegrad24", "text": "Ukraine military news"}, |
| 49 | {"author_handle": "PrettyCitiesX", "text": "London bridge at sunset"}, |
| 50 | {"author_handle": "PrettyCitiesX", "text": "Paris Eiffel tower"}, |
| 51 | {"author_handle": "PrettyCitiesX", "text": "Tokyo skyline"}, |
| 52 | {"author_handle": "earthserenityy", "text": "Mountain landscape photo"}, |
| 53 | {"author_handle": "earthserenityy", "text": "Beach sunset vibes"}, |
| 54 | ] |
| 55 | # Repeat off-topic to get 32 items |
| 56 | all_off_topic = off_topic * 4 |
| 57 | items = on_topic + all_off_topic |
| 58 | |
| 59 | result = x_judge.judge_x_corpus(items, "Rome") |
| 60 | assert result["on_topic_ratio"] < 0.4 |
| 61 | assert result["is_off_topic_flood"] |
| 62 | assert len(result["on_topic_items"]) == 8 |
| 63 | |
| 64 | def test_no_substring_collision_ai_in_said(self): |
| 65 | """'AI' as a topic token should NOT match 'said' (no substring collisions). |
| 66 | |
| 67 | This verifies that topic judging uses proper word tokenization, not |
| 68 | substring matching. If substring matching were used, 'ai' would match |
| 69 | inside 'said', causing false positives. |
| 70 | """ |
| 71 | items = [ |
| 72 | {"author_handle": "offtopic", "text": "He said something interesting"}, |
| 73 | {"author_handle": "offtopic", "text": "She said the weather was nice"}, |
| 74 | {"author_handle": "offtopic", "text": "They said it was great"}, |
| 75 | ] |
| 76 | result = x_judge.judge_x_corpus(items, "AI") |
| 77 | # None of these posts contain "AI" as a word, so all should be off-topic |
| 78 | assert len(result["on_topic_items"]) == 0 |
| 79 | assert len(result["off_topic_items"]) == 3 |
| 80 | |
| 81 | def test_no_substring_collision_rome_in_promoter(self): |
| 82 | """'rome' as a topic token should NOT match 'promoter' (no substring collisions).""" |
| 83 | items = [ |
| 84 | {"author_handle": "offtopic", "text": "A viral promoter of products"}, |
| 85 | {"author_handle": "offtopic", "text": "She is a band promoter"}, |
| 86 | ] |
| 87 | result = x_judge.judge_x_corpus(items, "Rome") |
| 88 | # None of these posts contain "Rome" as a word |
| 89 | assert len(result["on_topic_items"]) == 0 |
| 90 | assert len(result["off_topic_items"]) == 2 |
| 91 | |
| 92 | def test_us_pronoun_does_not_match_us_topic(self): |
| 93 | """Lowercase 'us' pronoun should not match 'US' country topic.""" |
| 94 | items = [ |
| 95 | {"author_handle": "offtopic", "text": "Tell us what you think"}, |
| 96 | {"author_handle": "offtopic", "text": "Join us for the event"}, |
| 97 | ] |
| 98 | # "US" topic vs "us" pronoun in text - case-sensitive check filters |
| 99 | result = x_judge.judge_x_corpus(items, "US") |
| 100 | assert len(result["on_topic_items"]) == 0 |
| 101 | assert len(result["off_topic_items"]) == 2 |
| 102 | |
| 103 | def test_us_acronym_matches_us_topic(self): |
| 104 | """Uppercase 'US' country acronym in text matches 'US' topic.""" |
| 105 | items = [ |
| 106 | {"author_handle": "ontopic", "text": "The US is expanding trade"}, |
| 107 | {"author_handle": "ontopic", "text": "US policy update"}, |
| 108 | ] |
| 109 | result = x_judge.judge_x_corpus(items, "US") |
| 110 | assert len(result["on_topic_items"]) == 2 |
| 111 | assert len(result["off_topic_items"]) == 0 |
| 112 | |
| 113 | def test_us_with_context_matches(self): |
| 114 | """'US economy' matches posts about US economy.""" |
| 115 | items = [ |
| 116 | {"author_handle": "ontopic", "text": "The US economy is growing"}, |
| 117 | {"author_handle": "ontopic", "text": "Economy news from America"}, |
| 118 | ] |
| 119 | result = x_judge.judge_x_corpus(items, "US economy") |
| 120 | # First post has "US" (acronym) and "economy" → on-topic |
| 121 | # Second post has "economy" only → on-topic |
| 122 | assert len(result["on_topic_items"]) >= 1 |
| 123 | |
| 124 | def test_multi_token_us_query_with_pronoun(self): |
| 125 | """'US economy' should NOT match text with pronoun 'us' and 'economy'.""" |
| 126 | items = [ |
| 127 | {"author_handle": "offtopic", "text": "Tell us about the economy"}, |
| 128 | {"author_handle": "offtopic", "text": "Let us discuss economy trends"}, |
| 129 | ] |
| 130 | result = x_judge.judge_x_corpus(items, "US economy") |
| 131 | # These have lowercase "us" (pronoun) not "US" (country) |
| 132 | # Should match on "economy" alone (50% coverage), which may or may not |
| 133 | # pass the relevance floor depending on thresholds |
| 134 | # The key test is that "us" doesn't contribute to the match |
| 135 | for item in result["on_topic_items"]: |
| 136 | # If on-topic, it should be due to "economy" match only |
| 137 | assert "economy" in item["text"].lower() |
| 138 | |
| 139 | def test_handle_stats_computed(self): |
| 140 | """Handle stats should track on-topic and total counts.""" |
| 141 | items = [ |
| 142 | {"author_handle": "handle_a", "text": "Rome is beautiful"}, |
| 143 | {"author_handle": "handle_a", "text": "Paris is nice"}, |
| 144 | {"author_handle": "handle_b", "text": "Rome vacation"}, |
| 145 | {"author_handle": "handle_b", "text": "Rome food"}, |
| 146 | ] |
| 147 | result = x_judge.judge_x_corpus(items, "Rome") |
| 148 | assert result["handle_stats"]["handle_a"]["total"] == 2 |
| 149 | assert result["handle_stats"]["handle_b"]["total"] == 2 |
| 150 | # handle_b has 2 on-topic posts about Rome |
| 151 | assert result["handle_stats"]["handle_b"]["on_topic"] == 2 |
| 152 | |
| 153 | |
| 154 | class TestPromotableHandles: |
| 155 | """Tests for promotable_handles — split FROM promotion.""" |
| 156 | |
| 157 | def test_explicit_handles_always_promoted(self): |
| 158 | """Explicit handles (--x-handle) should always be promoted.""" |
| 159 | items = [ |
| 160 | {"author_handle": "steipete", "text": "random post about nothing"}, |
| 161 | ] |
| 162 | explicit, extracted = x_judge.promotable_handles( |
| 163 | items, "quantum widgets", ["steipete"], explicit_handles=["steipete"] |
| 164 | ) |
| 165 | assert "steipete" in explicit |
| 166 | assert "steipete" not in extracted |
| 167 | |
| 168 | def test_extracted_handle_promoted_with_on_topic_posts(self): |
| 169 | """Extracted handle with ≥2 on-topic posts should be promoted.""" |
| 170 | items = [ |
| 171 | {"author_handle": "mamboitaliano__", "text": "Rome sunset beautiful"}, |
| 172 | {"author_handle": "mamboitaliano__", "text": "Rome Colosseum tour"}, |
| 173 | {"author_handle": "mamboitaliano__", "text": "Trevi Fountain Rome"}, |
| 174 | ] |
| 175 | explicit, extracted = x_judge.promotable_handles( |
| 176 | items, "Rome", ["mamboitaliano__"], explicit_handles=[] |
| 177 | ) |
| 178 | assert "mamboitaliano__" in extracted |
| 179 | assert len(explicit) == 0 |
| 180 | |
| 181 | def test_extracted_handle_not_promoted_with_off_topic_posts(self): |
| 182 | """Extracted handle with off-topic posts should not be promoted.""" |
| 183 | items = [ |
| 184 | {"author_handle": "visegrad24", "text": "Zelensky addresses summit"}, |
| 185 | {"author_handle": "visegrad24", "text": "Poland military update"}, |
| 186 | {"author_handle": "visegrad24", "text": "Ukraine border news"}, |
| 187 | # One post mentions Rome but mostly off-topic |
| 188 | {"author_handle": "visegrad24", "text": "Rome, Italy in caption"}, |
| 189 | ] |
| 190 | explicit, extracted = x_judge.promotable_handles( |
| 191 | items, "Rome", ["visegrad24"], explicit_handles=[] |
| 192 | ) |
| 193 | assert "visegrad24" not in extracted |
| 194 | assert len(explicit) == 0 |
| 195 | |
| 196 | def test_explicit_handle_not_in_extracted_still_promoted(self): |
| 197 | """Explicit handle not in extracted list should still be promoted.""" |
| 198 | items = [] # No items for this handle |
| 199 | explicit, extracted = x_judge.promotable_handles( |
| 200 | items, "topic", [], explicit_handles=["explicit_handle"] |
| 201 | ) |
| 202 | assert "explicit_handle" in explicit |
| 203 | |
| 204 | def test_min_hits_threshold(self): |
| 205 | """Handle needs MIN_ON_TOPIC_HITS to be promoted.""" |
| 206 | items = [ |
| 207 | {"author_handle": "onepost", "text": "Rome is nice"}, # Only 1 on-topic |
| 208 | ] |
| 209 | explicit, extracted = x_judge.promotable_handles( |
| 210 | items, "Rome", ["onepost"], explicit_handles=[] |
| 211 | ) |
| 212 | # Not enough on-topic hits (needs ≥2) |
| 213 | assert "onepost" not in extracted |
| 214 | |
| 215 | |
| 216 | class TestShouldRetryXSearch: |
| 217 | """Tests for should_retry_x_search.""" |
| 218 | |
| 219 | def test_skip_retry_on_quick_depth(self): |
| 220 | """Quick depth should skip retry.""" |
| 221 | items = [ |
| 222 | {"author_handle": "off_topic", "text": "completely unrelated content"}, |
| 223 | ] * 40 |
| 224 | assert not x_judge.should_retry_x_search(items, "Rome", depth="quick") |
| 225 | |
| 226 | def test_trigger_retry_on_off_topic_flood(self): |
| 227 | """Off-topic flood should trigger retry.""" |
| 228 | off_topic = [ |
| 229 | {"author_handle": "off_topic", "text": "completely unrelated content"}, |
| 230 | ] * 40 |
| 231 | assert x_judge.should_retry_x_search(off_topic, "Rome", depth="default") |
| 232 | |
| 233 | def test_no_retry_on_good_corpus(self): |
| 234 | """On-topic corpus should not trigger retry.""" |
| 235 | on_topic = [ |
| 236 | {"author_handle": "romefan", "text": "Rome vacation photos"}, |
| 237 | {"author_handle": "romefan", "text": "Rome colosseum visit"}, |
| 238 | {"author_handle": "romefan", "text": "Rome food tour"}, |
| 239 | {"author_handle": "romefan", "text": "Rome sunset beautiful"}, |
| 240 | ] |
| 241 | assert not x_judge.should_retry_x_search(on_topic, "Rome", depth="default") |
| 242 | |
| 243 | |
| 244 | class TestPruneOffTopicItems: |
| 245 | """Tests for prune_off_topic_items.""" |
| 246 | |
| 247 | def test_prunes_off_topic_keeps_on_topic(self): |
| 248 | """Should prune off-topic items and keep on-topic ones.""" |
| 249 | items = [ |
| 250 | {"author_handle": "on", "text": "Rome vacation"}, |
| 251 | {"author_handle": "off", "text": "Paris travel"}, |
| 252 | {"author_handle": "on", "text": "Rome colosseum"}, |
| 253 | {"author_handle": "off", "text": "London sights"}, |
| 254 | ] |
| 255 | result = x_judge.prune_off_topic_items(items, "Rome") |
| 256 | assert len(result) == 2 |
| 257 | assert all("Rome" in item["text"] for item in result) |
| 258 | |
| 259 | def test_empty_after_prune_is_ok(self): |
| 260 | """Pruning all items should return empty list.""" |
| 261 | items = [ |
| 262 | {"author_handle": "off", "text": "completely unrelated"}, |
| 263 | {"author_handle": "off", "text": "nothing to do with topic"}, |
| 264 | ] |
| 265 | result = x_judge.prune_off_topic_items(items, "Rome") |
| 266 | assert result == [] |
| 267 | |
| 268 | |
| 269 | class TestRomeScenario: |
| 270 | """Integration tests for the Rome failure scenario from 2026-08-14.""" |
| 271 | |
| 272 | @pytest.fixture |
| 273 | def rome_off_topic_corpus(self): |
| 274 | """The Rome failure corpus: 8 on-topic, 32 off-topic.""" |
| 275 | on_topic = [ |
| 276 | {"author_handle": "mamboitaliano__", "text": "Beautiful Rome sunset over the Tiber"}, |
| 277 | {"author_handle": "mamboitaliano__", "text": "Trevi Fountain in Rome today"}, |
| 278 | {"author_handle": "mamboitaliano__", "text": "Rome Colosseum morning visit"}, |
| 279 | {"author_handle": "Turismoromaweb", "text": "Visit Rome Italy for vacation"}, |
| 280 | {"author_handle": "romecityguide", "text": "Rome city center walking tour"}, |
| 281 | {"author_handle": "italyrome", "text": "Rome and Vatican day trip"}, |
| 282 | {"author_handle": "romephotos", "text": "Rome architecture stunning"}, |
| 283 | {"author_handle": "rometravel", "text": "Rome travel tips for tourists"}, |
| 284 | ] |
| 285 | off_topic = [ |
| 286 | # visegrad24 - geopolitics account, thin Rome connection |
| 287 | {"author_handle": "visegrad24", "text": "Zelensky addresses NATO summit"}, |
| 288 | {"author_handle": "visegrad24", "text": "Poland border security update"}, |
| 289 | {"author_handle": "visegrad24", "text": "Ukraine military operations today"}, |
| 290 | {"author_handle": "visegrad24", "text": "Russia sanctions news breaking"}, |
| 291 | # PrettyCitiesX - city aesthetics account, has other cities |
| 292 | {"author_handle": "PrettyCitiesX", "text": "London bridge at sunset"}, |
| 293 | {"author_handle": "PrettyCitiesX", "text": "Paris Eiffel tower night"}, |
| 294 | {"author_handle": "PrettyCitiesX", "text": "Tokyo skyline beautiful"}, |
| 295 | {"author_handle": "PrettyCitiesX", "text": "New York Central Park"}, |
| 296 | # earthserenityy - nature photos, no Rome |
| 297 | {"author_handle": "earthserenityy", "text": "Mountain landscape photography"}, |
| 298 | {"author_handle": "earthserenityy", "text": "Beach sunset vibes serene"}, |
| 299 | {"author_handle": "earthserenityy", "text": "Forest morning fog mystical"}, |
| 300 | {"author_handle": "earthserenityy", "text": "Ocean waves crashing shore"}, |
| 301 | # AS Roma sports content (Rome collision) |
| 302 | {"author_handle": "asromafc", "text": "AS Roma match tonight Serie A"}, |
| 303 | {"author_handle": "asromafc", "text": "Roma vs Juventus preview"}, |
| 304 | {"author_handle": "asromafc", "text": "Forza Roma goal highlights"}, |
| 305 | {"author_handle": "asromafc", "text": "AS Roma transfer rumors"}, |
| 306 | # Rome Odunze collision |
| 307 | {"author_handle": "nflupdates", "text": "Rome Odunze catches touchdown"}, |
| 308 | {"author_handle": "nflupdates", "text": "Bears WR Rome Odunze stats"}, |
| 309 | {"author_handle": "fantasyfb", "text": "Start Rome Odunze this week"}, |
| 310 | {"author_handle": "fantasyfb", "text": "Rome Odunze fantasy value"}, |
| 311 | ] |
| 312 | # Duplicate off-topic to reach ~32 items |
| 313 | return on_topic + off_topic + off_topic[:12] |
| 314 | |
| 315 | def test_rome_corpus_detected_as_off_topic(self, rome_off_topic_corpus): |
| 316 | """Rome failure corpus should be detected as off-topic flood.""" |
| 317 | result = x_judge.judge_x_corpus(rome_off_topic_corpus, "Rome") |
| 318 | # ~8/40 = 0.2, well below 0.4 threshold |
| 319 | assert result["on_topic_ratio"] < 0.4 |
| 320 | assert result["is_off_topic_flood"] |
| 321 | |
| 322 | def test_mamboitaliano_promotable(self, rome_off_topic_corpus): |
| 323 | """mamboitaliano__ with on-topic Rome posts should be promotable.""" |
| 324 | explicit, extracted = x_judge.promotable_handles( |
| 325 | rome_off_topic_corpus, |
| 326 | "Rome", |
| 327 | ["mamboitaliano__", "visegrad24", "PrettyCitiesX"], |
| 328 | explicit_handles=[], |
| 329 | ) |
| 330 | assert "mamboitaliano__" in extracted |
| 331 | |
| 332 | def test_visegrad24_not_promotable(self, rome_off_topic_corpus): |
| 333 | """visegrad24 with off-topic posts should NOT be promotable.""" |
| 334 | explicit, extracted = x_judge.promotable_handles( |
| 335 | rome_off_topic_corpus, |
| 336 | "Rome", |
| 337 | ["mamboitaliano__", "visegrad24", "PrettyCitiesX"], |
| 338 | explicit_handles=[], |
| 339 | ) |
| 340 | assert "visegrad24" not in extracted |
| 341 | assert "PrettyCitiesX" not in extracted |
| 342 | |
| 343 | def test_turismoromaweb_promotable_if_explicit(self, rome_off_topic_corpus): |
| 344 | """Turismoromaweb should be promoted if explicitly specified.""" |
| 345 | explicit, extracted = x_judge.promotable_handles( |
| 346 | rome_off_topic_corpus, |
| 347 | "Rome", |
| 348 | ["Turismoromaweb"], |
| 349 | explicit_handles=["Turismoromaweb"], |
| 350 | ) |
| 351 | assert "Turismoromaweb" in explicit |
| 352 |