| 1 | """Deterministic topic naming and junk-shape classification for discovery. |
| 2 | |
| 3 | Discovery mode surfaces short, named, content-worthy topics instead of raw |
| 4 | post titles. This module is the pure-function, stdlib-only stage-1 fallback |
| 5 | for that pipeline (used when no LLM is available, and as the deterministic |
| 6 | baseline the LLM path is judged against): |
| 7 | |
| 8 | - ``distill_topic_name(title, snippet)`` distills a listing title into a |
| 9 | 2-6 word searchable topic name: question/framing scaffolding is stripped, |
| 10 | proper-noun / digit-bearing entity phrases are preferred and emitted as an |
| 11 | ORDERED phrase in title order (never a bag of words), and the cleaned, |
| 12 | truncated title is the final fallback so the result is never empty for any |
| 13 | title with word content. |
| 14 | - ``is_junk_shape(title, snippet)`` flags listing shapes that should never |
| 15 | become topics: help-me questions, beginner asks, and first-person musings. |
| 16 | Launch titles ("Show HN: ...") and entity-bearing news statements are not |
| 17 | junk. |
| 18 | |
| 19 | Both functions take plain strings and return plain values - no candidate |
| 20 | objects, no config, no I/O - so they are trivially testable and reusable. |
| 21 | |
| 22 | Names produced here are used downstream as short search queries and grounding |
| 23 | strings, so they never carry trailing punctuation or quote characters. Per the |
| 24 | head-token convention, callers must never assume a distilled name appears as a |
| 25 | contiguous substring of any document. |
| 26 | |
| 27 | Token conventions (stopwords, capital/digit entity signals) are inherited from |
| 28 | ``entity_extract`` and extended here; unlike ``extract_text_entities`` this |
| 29 | module preserves title order and original casing because the output is a |
| 30 | human-readable phrase, not a matching set. Non-Latin (CJK) titles never crash: |
| 31 | they carry no Latin entity signal, so they fall through to the cleaned-title |
| 32 | path, capped at ``_MAX_NAME_CHARS``. |
| 33 | """ |
| 34 | |
| 35 | from __future__ import annotations |
| 36 | |
| 37 | import re |
| 38 | from typing import List, NamedTuple, Optional |
| 39 | |
| 40 | from .entity_extract import ENTITY_STOPWORDS |
| 41 | |
| 42 | _MAX_NAME_WORDS = 6 |
| 43 | _MAX_NAME_CHARS = 80 |
| 44 | |
| 45 | # Extends the shared entity stopwords with pronouns, auxiliaries, contractions |
| 46 | # and musing filler that read as capitalized sentence-openers in titles but are |
| 47 | # never entities ("My", "Everyone", "Don't", ...). Deliberate casualty: the |
| 48 | # acronyms "US" and "IT" are swallowed by their pronoun homographs. |
| 49 | _ANCHOR_STOPWORDS = frozenset(ENTITY_STOPWORDS) | frozenset({ |
| 50 | "i", "i'm", "i've", "i'd", "i'll", "me", "my", "mine", "myself", |
| 51 | "we", "we're", "we've", "our", "ours", "us", |
| 52 | "you", "you're", "your", "yours", |
| 53 | "am", "were", "be", "why", "when", "where", "which", "whom", "whose", |
| 54 | "does", "did", "doing", "done", "should", "shall", "may", "might", "must", |
| 55 | "if", "or", "so", "as", "any", "anyone", "anybody", "someone", "somebody", |
| 56 | "everyone", "everybody", "nobody", "none", "no", "yes", |
| 57 | "please", "thanks", "thank", "really", "actually", "very", "well", |
| 58 | "while", "during", "still", "even", "ever", "never", "always", |
| 59 | "don't", "dont", "can't", "cant", "won't", "wont", "isn't", "isnt", |
| 60 | "aren't", "arent", "doesn't", "doesnt", "didn't", "didnt", |
| 61 | "it's", "that's", "there's", "here's", "let's", "what's", "who's", "how's", |
| 62 | "mean", "means", "meant", "same", "thing", "things", "stuff", |
| 63 | "way", "ways", "lot", "lots", "kind", "sort", |
| 64 | "today", "yesterday", "tomorrow", |
| 65 | }) |
| 66 | |
| 67 | # Characters stripped from token edges for display (internal hyphens/dots in |
| 68 | # "open-source" / "example.com" survive). Includes unicode dashes/ellipsis. |
| 69 | _EDGE_CHARS = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~–—…" |
| 70 | _TRAILING_JUNK = ".,;:!?…'\"`- " |
| 71 | |
| 72 | _POSSESSIVE_RE = re.compile(r"(?<=\w)'s\b", re.IGNORECASE) |
| 73 | _DOUBLE_QUOTE_RE = re.compile(r"[\"“”„«»]") |
| 74 | _LONE_APOSTROPHE_RE = re.compile(r"(?<!\w)'|'(?!\w)") |
| 75 | _SENTENCE_END_RE = re.compile(r"[.!?;:,]$") |
| 76 | |
| 77 | # Framing scaffolding stripped (iteratively) from the start of a title before |
| 78 | # naming: forum labels, interrogative openers, first-person setup, politeness |
| 79 | # filler, and leading articles. Junk *classification* has its own patterns |
| 80 | # below; these only clean the string we name from. |
| 81 | _SCAFFOLD_RES = [re.compile(p, re.IGNORECASE) for p in ( |
| 82 | r"^(show hn|ask hn|tell hn|launch hn|psa|eli5|tifu|til|discussion|" |
| 83 | r"question|help|advice|update|rant|vent|meta)\s*[:\-–—]\s*", |
| 84 | r"^(how|what|when|where|which|why|who)\s+" |
| 85 | r"(do|does|did|is|are|was|were|am|can|could|should|would|will|to|i|we|you|your|my|one)\s+", |
| 86 | r"^(is|are|does|do|did|can|could|should|would|will|has|have|am)\s+" |
| 87 | r"(there|it|this|anyone|anybody|someone|somebody|we|you|i|they|my|your)\s+", |
| 88 | r"^(i|we)\s+(think|believe|feel|guess|wonder|noticed|realized|have run|" |
| 89 | r"have been|have|had|am|was|were|just|finally|recently|need|want|" |
| 90 | r"would like|tried|keep|built|made|created|wrote|spent)\s+", |
| 91 | r"^(i'm|i've|i'd|we're|we've)\s+", |
| 92 | r"^my\s+(coworker|co-worker|colleague|boss|friend|manager|team|company|" |
| 93 | r"startup|wife|husband|partner|mom|dad|mother|father|brother|sister|" |
| 94 | r"son|daughter|kid|kids|roommate|neighbor)\s+\w+\s+", |
| 95 | r"^(hey|hi|hello|guys|folks|please|okay|ok|so|honestly|serious question)[,!\s]\s*", |
| 96 | r"^(a|an|the)\s+", |
| 97 | )] |
| 98 | |
| 99 | # --- junk-shape markers (matched against the cleaned, lowercased title) ----- |
| 100 | |
| 101 | _LAUNCH_RE = re.compile(r"^(show hn|launch hn)\b") |
| 102 | # Leading interrogatives: wh-words count only with a question follow-through |
| 103 | # ("What is the best..." is junk; "What Gemma 4 means..." is an explainer). |
| 104 | _WH_JUNK_RE = re.compile( |
| 105 | r"^(how|what|why|when|where|which|who)\s+" |
| 106 | r"(do|does|did|is|are|was|were|am|can|could|should|would|will|to|i|we|you|your|my|one)\b" |
| 107 | ) |
| 108 | _AUX_JUNK_RE = re.compile( |
| 109 | r"^(is|are|does|do|did|can|could|should|would|will|has|have|am)\s+" |
| 110 | r"(there|it|this|anyone|anybody|someone|somebody|we|you|i|they|my|your)\b" |
| 111 | ) |
| 112 | _HELP_RE = re.compile( |
| 113 | r"\bneed (some |a little )?(help|advice)\b|\bplease help\b|\bhelp me\b|" |
| 114 | r"^help\b|\bany (advice|recommendation|recommendations|suggestions|recs|tips)\b|" |
| 115 | r"\blooking for (advice|recommendations|suggestions|help|tips)\b|" |
| 116 | r"\bwhere (do|should|would) (i|we) (even )?(start|begin)\b|\bwhere to start\b|" |
| 117 | r"\bbeginner (question|here)\b|\bnoob (question|here)\b|" |
| 118 | r"\btotal beginner\b|\bcomplete beginner\b|\bam i missing something\b|" |
| 119 | r"\brecommend me\b" |
| 120 | ) |
| 121 | _MUSING_RE = re.compile( |
| 122 | r"^(i think|i feel|i believe|i guess|i wonder|i have been|i've been|i keep|" |
| 123 | r"my thoughts|thoughts on|unpopular opinion|hot take|am i the only one|" |
| 124 | r"is it just me|anyone else|does anyone else|rant|vent|change my mind|cmv)\b" |
| 125 | ) |
| 126 | _EVERYONE_RE = re.compile( |
| 127 | r"\beveryone (is|does|says|seems|keeps|wants)\b.{0,80}\bbut (do|are|can|should|will|did) we\b" |
| 128 | ) |
| 129 | |
| 130 | |
| 131 | class _Token(NamedTuple): |
| 132 | display: str # edge-punctuation-stripped, original casing |
| 133 | lower: str |
| 134 | is_anchor: bool # proper-noun / digit / acronym entity signal |
| 135 | breaks_after: bool # sentence/clause boundary follows this token |
| 136 | |
| 137 | |
| 138 | def distill_topic_name(title: str, snippet: str = "") -> str: |
| 139 | """Distill a listing title (+ optional snippet) into a 2-6 word topic name. |
| 140 | |
| 141 | The name is an ordered phrase built from entity anchors in title order, |
| 142 | safe to use as a short search query: <= 6 words, <= 80 chars, no trailing |
| 143 | punctuation, no quote characters. Never empty for any input with word |
| 144 | content (the sole exception: title AND snippet contain no word characters, |
| 145 | which returns ""). |
| 146 | """ |
| 147 | base = _normalize(title) or _normalize(snippet) |
| 148 | if not base: |
| 149 | return "" |
| 150 | |
| 151 | stripped = _strip_scaffolding(base) |
| 152 | tokens = _tokenize(stripped) |
| 153 | if not tokens: |
| 154 | tokens = _tokenize(base) |
| 155 | if not tokens: |
| 156 | return "" |
| 157 | words = [t.display for t in tokens] |
| 158 | |
| 159 | # Already-short titles pass through unless a stronger entity phrase is |
| 160 | # buried mid-title (first word not an anchor while anchors exist). |
| 161 | if len(words) <= _MAX_NAME_WORDS and (tokens[0].is_anchor or not any(t.is_anchor for t in tokens)): |
| 162 | return _finalize(words) |
| 163 | |
| 164 | phrase = _entity_phrase(tokens) |
| 165 | if phrase: |
| 166 | return _finalize(phrase) |
| 167 | |
| 168 | # Title had no entity anchors: try the snippet's leading entity phrase. |
| 169 | if snippet: |
| 170 | snippet_tokens = _tokenize(_strip_scaffolding(_normalize(snippet))) |
| 171 | snippet_phrase = _entity_phrase(snippet_tokens) |
| 172 | if snippet_phrase: |
| 173 | return _finalize(snippet_phrase) |
| 174 | |
| 175 | # Final fallback: cleaned title truncated to the word cap. |
| 176 | return _finalize(words[:_MAX_NAME_WORDS]) |
| 177 | |
| 178 | |
| 179 | def is_junk_shape(title: str, snippet: str = "") -> bool: |
| 180 | """True when the listing shape is not content-worthy. |
| 181 | |
| 182 | Rule-based markers: leading interrogatives, help/advice asks, first-person |
| 183 | musings, and trailing "?" with no named entity in the title. Launch titles |
| 184 | ("Show HN: ...") and entity-bearing news statements are not junk. The |
| 185 | snippet is consulted only when the title itself has no entity anchors. |
| 186 | """ |
| 187 | cleaned = _normalize(title) |
| 188 | if not cleaned: |
| 189 | cleaned = _normalize(snippet) |
| 190 | if not cleaned: |
| 191 | return True # nothing nameable at all |
| 192 | lower = cleaned.lower() |
| 193 | |
| 194 | if _LAUNCH_RE.search(lower): |
| 195 | return False |
| 196 | if _WH_JUNK_RE.search(lower) or _AUX_JUNK_RE.search(lower): |
| 197 | return True |
| 198 | if _HELP_RE.search(lower) or _MUSING_RE.search(lower) or _EVERYONE_RE.search(lower): |
| 199 | return True |
| 200 | |
| 201 | has_entity = any(t.is_anchor for t in _tokenize(cleaned)) |
| 202 | if lower.endswith(("?", "？")) and not has_entity: |
| 203 | return True |
| 204 | if not has_entity and snippet: |
| 205 | snippet_lower = _normalize(snippet).lower() |
| 206 | if (_HELP_RE.search(snippet_lower) or _MUSING_RE.search(snippet_lower) |
| 207 | or _AUX_JUNK_RE.search(snippet_lower) or _EVERYONE_RE.search(snippet_lower)): |
| 208 | return True |
| 209 | return False |
| 210 | |
| 211 | |
| 212 | # --------------------------------------------------------------------------- |
| 213 | |
| 214 | |
| 215 | def _normalize(text: str) -> str: |
| 216 | """Collapse whitespace, drop quote characters, fold possessives ("4's" -> "4").""" |
| 217 | if not text: |
| 218 | return "" |
| 219 | text = text.replace("’", "'").replace("‘", "'").replace("`", "'").replace("´", "'") |
| 220 | text = _POSSESSIVE_RE.sub("", text) |
| 221 | text = _DOUBLE_QUOTE_RE.sub(" ", text) |
| 222 | text = _LONE_APOSTROPHE_RE.sub(" ", text) |
| 223 | return " ".join(text.split()) |
| 224 | |
| 225 | |
| 226 | def _strip_scaffolding(text: str) -> str: |
| 227 | """Iteratively strip question/framing scaffolding from the title start.""" |
| 228 | for _ in range(6): |
| 229 | before = text |
| 230 | for pattern in _SCAFFOLD_RES: |
| 231 | text = pattern.sub("", text, count=1).lstrip(" ,-") |
| 232 | if text == before: |
| 233 | break |
| 234 | return text.strip() |
| 235 | |
| 236 | |
| 237 | def _is_anchor(display: str) -> bool: |
| 238 | """Entity signal per entity_extract conventions: capitals, digits, acronyms.""" |
| 239 | if not display: |
| 240 | return False |
| 241 | if display.lower() in _ANCHOR_STOPWORDS: |
| 242 | return False |
| 243 | if any(c.isdigit() for c in display): |
| 244 | return True |
| 245 | if len(display) < 2: |
| 246 | return False |
| 247 | if display[0].isupper(): |
| 248 | return True |
| 249 | return any(c.isupper() for c in display[1:]) # iPhone, gpt4all-style |
| 250 | |
| 251 | |
| 252 | def _tokenize(text: str) -> List[_Token]: |
| 253 | """Split into display tokens, tagging entity anchors and clause boundaries.""" |
| 254 | tokens: List[_Token] = [] |
| 255 | for raw in text.split(): |
| 256 | display = raw.strip(_EDGE_CHARS) |
| 257 | if not display: |
| 258 | # Pure-punctuation token (a bare dash, "..."): clause boundary. |
| 259 | if tokens: |
| 260 | tokens[-1] = tokens[-1]._replace(breaks_after=True) |
| 261 | continue |
| 262 | tokens.append(_Token( |
| 263 | display=display, |
| 264 | lower=display.lower(), |
| 265 | is_anchor=_is_anchor(display), |
| 266 | breaks_after=bool(_SENTENCE_END_RE.search(raw)), |
| 267 | )) |
| 268 | return tokens |
| 269 | |
| 270 | |
| 271 | def _entity_phrase(tokens: List[_Token]) -> Optional[List[str]]: |
| 272 | """Build an ordered phrase from entity-anchor runs, in title order. |
| 273 | |
| 274 | Adjacent anchor runs separated by <= 2 contentful (non-stopword, |
| 275 | non-boundary) words are merged with their connecting words kept, so the |
| 276 | phrase stays readable ("AI agent handle Slack", not "AI Slack"). Runs are |
| 277 | then concatenated in title order up to the word cap. |
| 278 | """ |
| 279 | runs: List[tuple[int, int]] = [] # inclusive (start, end) token indices |
| 280 | i = 0 |
| 281 | while i < len(tokens): |
| 282 | if tokens[i].is_anchor: |
| 283 | j = i |
| 284 | while j + 1 < len(tokens) and tokens[j + 1].is_anchor and not tokens[j].breaks_after: |
| 285 | j += 1 |
| 286 | runs.append((i, j)) |
| 287 | i = j + 1 |
| 288 | else: |
| 289 | i += 1 |
| 290 | if not runs: |
| 291 | return None |
| 292 | |
| 293 | merged = [runs[0]] |
| 294 | for start, end in runs[1:]: |
| 295 | prev_start, prev_end = merged[-1] |
| 296 | gap = tokens[prev_end + 1:start] |
| 297 | if ( |
| 298 | 0 < len(gap) <= 2 |
| 299 | and not tokens[prev_end].breaks_after |
| 300 | and all(g.lower not in _ANCHOR_STOPWORDS and not g.breaks_after for g in gap) |
| 301 | ): |
| 302 | merged[-1] = (prev_start, end) |
| 303 | else: |
| 304 | merged.append((start, end)) |
| 305 | |
| 306 | words: List[str] = [] |
| 307 | last_index: Optional[int] = None |
| 308 | for start, end in merged: |
| 309 | span = [t.display for t in tokens[start:end + 1]] |
| 310 | if not words and len(span) > _MAX_NAME_WORDS: |
| 311 | span = span[:_MAX_NAME_WORDS] |
| 312 | end = start + _MAX_NAME_WORDS - 1 |
| 313 | if len(words) + len(span) > _MAX_NAME_WORDS: |
| 314 | break |
| 315 | words.extend(span) |
| 316 | last_index = end |
| 317 | |
| 318 | # Readability extension: pull in one attached plural noun ("Slack replies"). |
| 319 | if words and len(words) < _MAX_NAME_WORDS and last_index is not None: |
| 320 | nxt = tokens[last_index + 1] if last_index + 1 < len(tokens) else None |
| 321 | if ( |
| 322 | nxt is not None |
| 323 | and not tokens[last_index].breaks_after |
| 324 | and not nxt.is_anchor |
| 325 | and nxt.display.islower() |
| 326 | and nxt.display.endswith("s") |
| 327 | and nxt.lower not in _ANCHOR_STOPWORDS |
| 328 | ): |
| 329 | words.append(nxt.display) |
| 330 | |
| 331 | return words or None |
| 332 | |
| 333 | |
| 334 | def _finalize(words: List[str]) -> str: |
| 335 | """Join to a query-safe name: char cap, no trailing punctuation or quotes.""" |
| 336 | name = " ".join(w for w in words if w).strip() |
| 337 | if len(name) > _MAX_NAME_CHARS: |
| 338 | name = name[:_MAX_NAME_CHARS].rstrip() |
| 339 | name = name.strip(_TRAILING_JUNK) |
| 340 | return " ".join(name.split()) |
| 341 |