| 1 | """Shared token-overlap relevance scoring for search result ranking. |
| 2 | |
| 3 | The score is intentionally query-centric: |
| 4 | - exact phrase matches should score very high |
| 5 | - partial matches should pay a meaningful penalty |
| 6 | - matches on generic words alone ("odds", "review") should not pass as relevant |
| 7 | """ |
| 8 | |
| 9 | import re |
| 10 | from typing import List, Optional, Set |
| 11 | |
| 12 | from . import cjk |
| 13 | |
| 14 | # Stopwords for relevance computation (common English words that dilute token overlap) |
| 15 | STOPWORDS = frozenset({ |
| 16 | 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', |
| 17 | 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', |
| 18 | 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', |
| 19 | 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', |
| 20 | 'all', 'just', 'get', 'has', 'have', 'was', 'will', |
| 21 | # Hebrew function words / prepositions / conjunctions |
| 22 | 'את', 'של', 'על', 'עם', 'אל', 'כי', 'לא', 'הוא', 'היא', 'הם', |
| 23 | 'הן', 'אנו', 'אנחנו', 'זה', 'זו', 'זאת', 'כל', 'יש', 'אין', |
| 24 | 'כבר', 'רק', 'גם', 'כן', 'אם', 'או', 'אבל', 'כך', 'מה', 'מי', |
| 25 | 'איך', 'למה', 'כמה', 'היה', 'הייתה', 'היו', 'יהיה', 'יהיו', |
| 26 | # Hebrew definite article / prefixes appearing as standalone tokens after split |
| 27 | 'ה', 'ב', 'ל', 'מ', 'כ', 'ו', 'ש', |
| 28 | }) | cjk.CHINESE_STOPWORDS |
| 29 | |
| 30 | # Shared relevance-ranking thresholds for the Reddit pipelines (keyed + keyless). |
| 31 | # Single source of truth so both paths apply identical thresholds to the same |
| 32 | # query. RELEVANCE_FLOOR: posts below this are off-topic; the zero-overlap tail is |
| 33 | # dropped when anything relevant remains. MIN_ON_TOPIC: how many posts must clear |
| 34 | # the soft floor before it is applied wholesale. |
| 35 | RELEVANCE_FLOOR = 0.1 |
| 36 | MIN_ON_TOPIC = 5 |
| 37 | |
| 38 | |
| 39 | # Synonym groups for relevance scoring (bidirectional expansion) |
| 40 | # Superset of all platform-specific synonym dicts |
| 41 | SYNONYMS = { |
| 42 | 'hip': {'rap', 'hiphop'}, |
| 43 | 'hop': {'rap', 'hiphop'}, |
| 44 | 'rap': {'hip', 'hop', 'hiphop'}, |
| 45 | 'hiphop': {'rap', 'hip', 'hop'}, |
| 46 | 'js': {'javascript'}, |
| 47 | 'javascript': {'js'}, |
| 48 | 'ts': {'typescript'}, |
| 49 | 'typescript': {'ts'}, |
| 50 | 'ai': {'artificial', 'intelligence'}, |
| 51 | 'ml': {'machine', 'learning'}, |
| 52 | 'react': {'reactjs'}, |
| 53 | 'reactjs': {'react'}, |
| 54 | 'svelte': {'sveltejs'}, |
| 55 | 'sveltejs': {'svelte'}, |
| 56 | 'vue': {'vuejs'}, |
| 57 | 'vuejs': {'vue'}, |
| 58 | } |
| 59 | |
| 60 | # Generic query words that should not carry relevance on their own. |
| 61 | # They still help when paired with stronger entity/topic matches. |
| 62 | # |
| 63 | # The second group is scaffolding emitted by planner's ranking-query templates |
| 64 | # ("What recent evidence from the last 30 days is most relevant to X?" and its |
| 65 | # siblings). Those words are not the topic, but every one of them was being |
| 66 | # counted as an informative query token, which capped achievable coverage at the |
| 67 | # topic's share of the query and demoted on-topic posts. Kept here rather than |
| 68 | # stripped in the planner so any caller building a similar natural-language |
| 69 | # ranking query gets the same treatment. |
| 70 | # |
| 71 | # Domain nouns from those same templates (production, market, workflows, |
| 72 | # experience, signals, ...) are deliberately absent: they can legitimately be a |
| 73 | # user's topic, and demoting them globally would hurt every source. |
| 74 | # tests/test_ranking_query_scaffolding.py pins that split. |
| 75 | LOW_SIGNAL_QUERY_TOKENS = frozenset({ |
| 76 | 'advice', 'animation', 'animations', 'best', 'chance', 'chances', |
| 77 | 'code', 'compare', 'comparison', 'differences', 'explain', 'guide', |
| 78 | 'guides', 'how', 'latest', 'news', 'odds', 'opinion', 'opinions', |
| 79 | 'prediction', 'predictions', 'probability', 'probabilities', 'prompt', |
| 80 | 'prompting', 'prompts', 'rate', 'review', 'reviews', 'thoughts', |
| 81 | 'tip', 'tips', 'tutorial', 'tutorials', 'update', 'updates', 'use', |
| 82 | 'using', 'versus', 'vs', 'worth', |
| 83 | # planner ranking-query scaffolding |
| 84 | '30', 'current', 'days', 'describing', 'especially', 'evidence', 'exist', |
| 85 | 'follow', 'hands', 'last', 'matter', 'most', 'new', 'people', 'real', |
| 86 | 'recent', 'relevant', 'running', 'up', 'world', |
| 87 | }) |
| 88 | |
| 89 | |
| 90 | def tokenize(text: str) -> Set[str]: |
| 91 | """Lowercase, strip punctuation, remove stopwords, drop single-char tokens. |
| 92 | |
| 93 | Expands tokens with synonyms for better cross-domain matching. |
| 94 | |
| 95 | Chinese text is segmented via cjk.segment (jieba or character bigrams) so |
| 96 | overlap scoring works on Chinese sources; ASCII text keeps the original |
| 97 | whitespace path. |
| 98 | """ |
| 99 | words = cjk.segment(text) |
| 100 | tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} |
| 101 | expanded = set(tokens) |
| 102 | for t in tokens: |
| 103 | if t in SYNONYMS: |
| 104 | expanded.update(SYNONYMS[t]) |
| 105 | return expanded |
| 106 | |
| 107 | |
| 108 | def _normalize_phrase(text: str) -> str: |
| 109 | """Normalize text for phrase containment checks.""" |
| 110 | return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split()) |
| 111 | |
| 112 | |
| 113 | class PreparedQuery: |
| 114 | """Precomputed query shape reused across items in a stream. |
| 115 | |
| 116 | Built once per ranking_query; reused by token_overlap_relevance so the |
| 117 | per-item normalize/score loops don't re-tokenize the same query N times. |
| 118 | """ |
| 119 | |
| 120 | __slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase") |
| 121 | |
| 122 | def __init__(self, query: str) -> None: |
| 123 | self.raw = query |
| 124 | self.q_tokens = tokenize(query) |
| 125 | informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS} |
| 126 | self.informative_q_tokens = informative or self.q_tokens |
| 127 | self.normalized_phrase = _normalize_phrase(query) |
| 128 | |
| 129 | |
| 130 | def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery: |
| 131 | return query if isinstance(query, PreparedQuery) else PreparedQuery(query) |
| 132 | |
| 133 | |
| 134 | def token_overlap_relevance( |
| 135 | query: "str | PreparedQuery", |
| 136 | text: str, |
| 137 | hashtags: Optional[List[str]] = None, |
| 138 | ) -> float: |
| 139 | """Compute a query-centric relevance score between 0.0 and 1.0. |
| 140 | |
| 141 | The score combines: |
| 142 | - query coverage |
| 143 | - informative-token coverage |
| 144 | - a small precision term to penalize extra noise |
| 145 | - an exact phrase bonus |
| 146 | |
| 147 | Generic tokens alone are capped below typical relevance filter thresholds. |
| 148 | |
| 149 | Args: |
| 150 | query: Search query |
| 151 | text: Content text to match against |
| 152 | hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated |
| 153 | hashtags are split to match query tokens (e.g. "claudecode" matches "claude"). |
| 154 | |
| 155 | Returns: |
| 156 | Float between 0.0 and 1.0 (0.5 for empty queries) |
| 157 | """ |
| 158 | prepared = _as_prepared(query) |
| 159 | q_tokens = prepared.q_tokens |
| 160 | |
| 161 | # Combine text and hashtags for matching |
| 162 | combined = text |
| 163 | if hashtags: |
| 164 | combined = f"{text} {' '.join(hashtags)}" |
| 165 | t_tokens = tokenize(combined) |
| 166 | |
| 167 | # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code") |
| 168 | if hashtags: |
| 169 | for tag in hashtags: |
| 170 | tag_lower = tag.lower() |
| 171 | for qt in q_tokens: |
| 172 | if qt in tag_lower and qt != tag_lower: |
| 173 | t_tokens.add(qt) |
| 174 | |
| 175 | if not q_tokens: |
| 176 | return 0.5 # Neutral fallback for empty/stopword-only queries |
| 177 | |
| 178 | overlap_tokens = q_tokens & t_tokens |
| 179 | overlap = len(overlap_tokens) |
| 180 | if overlap == 0: |
| 181 | return 0.0 |
| 182 | |
| 183 | informative_q_tokens = prepared.informative_q_tokens |
| 184 | |
| 185 | coverage = overlap / len(q_tokens) |
| 186 | informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens) |
| 187 | precision_denominator = min(len(t_tokens), len(q_tokens) + 4) or 1 |
| 188 | precision = overlap / precision_denominator |
| 189 | |
| 190 | phrase_bonus = 0.0 |
| 191 | normalized_query = prepared.normalized_phrase |
| 192 | normalized_text = _normalize_phrase(combined) |
| 193 | if normalized_query: |
| 194 | contained = normalized_query in normalized_text |
| 195 | if not contained and cjk.has_cjk(normalized_query): |
| 196 | # CJK has no inter-word spaces, so a multi-token Chinese query like |
| 197 | # "国产大模型 测评" never appears verbatim in continuous source text |
| 198 | # ("...国产大模型的最新测评"). Retry the containment with spaces |
| 199 | # removed so the phrase bonus isn't permanently dead for Chinese. |
| 200 | # Gated on has_cjk so English ("react hooks") keeps space-sensitive |
| 201 | # matching and doesn't gain spurious bonuses from concatenation. |
| 202 | contained = normalized_query.replace(" ", "") in normalized_text.replace(" ", "") |
| 203 | if contained: |
| 204 | phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16 |
| 205 | |
| 206 | base = ( |
| 207 | 0.55 * (coverage ** 1.35) + |
| 208 | 0.25 * informative_overlap + |
| 209 | 0.20 * precision |
| 210 | ) |
| 211 | |
| 212 | # If we only matched generic query words, keep the score below the |
| 213 | # normal relevance filter threshold so these do not survive by default. |
| 214 | if informative_q_tokens and not (informative_q_tokens & t_tokens): |
| 215 | return round(min(0.24, base), 2) |
| 216 | |
| 217 | return round(min(1.0, base + phrase_bonus), 2) |
| 218 |