| 1 | """Keyless Reddit listing scrape via shreddit /svc partials — with real scores. |
| 2 | |
| 3 | The subreddit listing partial |
| 4 | ``/svc/shreddit/community-more-posts/{sort}/?name={sub}[&t={range}]`` serves |
| 5 | HTTP 200 with no API key and **server-renders each post's upvote score**, which |
| 6 | neither RSS nor the comments endpoint provides. Each post is a |
| 7 | ``<shreddit-post>`` element whose start-tag attributes carry ``score``, |
| 8 | ``comment-count``, ``post-title``, ``permalink``, ``author``, ``subreddit-name`` |
| 9 | and ``created-timestamp``. |
| 10 | |
| 11 | This is the keyless source of post-level upvotes. It works for normal users on |
| 12 | ordinary connections (verified), so reddit_keyless uses it both as a scored |
| 13 | discovery source and to backfill scores onto RSS-discovered posts. |
| 14 | """ |
| 15 | |
| 16 | import html as _html |
| 17 | import re |
| 18 | import sys |
| 19 | from datetime import datetime, timezone |
| 20 | from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError |
| 21 | from typing import Any, Dict, List, Optional, Set |
| 22 | |
| 23 | from . import http |
| 24 | from .relevance import token_overlap_relevance, tokenize |
| 25 | |
| 26 | # Generic domain terms that are excluded from the keyword gate — matches |
| 27 | # pipeline._DISCOVERY_GENERIC_DOMAIN_TERMS (duplicated to avoid circular import). |
| 28 | _DISCOVERY_GENERIC_DOMAIN_TERMS: Set[str] = { |
| 29 | "ai", "artificial", "intelligence", "tech", "technology", "trending", "trend", |
| 30 | } |
| 31 | |
| 32 | # Listing sorts pulled per subreddit, by depth. |
| 33 | LISTING_SORTS = { |
| 34 | "quick": ["top"], |
| 35 | "default": ["top", "hot"], |
| 36 | "deep": ["top", "hot", "new"], |
| 37 | } |
| 38 | DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50} |
| 39 | TIMEFRAME = "month" |
| 40 | MAX_WORKERS = 4 |
| 41 | LISTING_TIMEOUT = 15 |
| 42 | |
| 43 | _POST_CARD = re.compile(r"<shreddit-post(?=[\s>])[^>]*>") |
| 44 | |
| 45 | |
| 46 | def _log(msg: str) -> None: |
| 47 | sys.stderr.write(f"[RedditListing] {msg}\n") |
| 48 | sys.stderr.flush() |
| 49 | |
| 50 | |
| 51 | def _matches_discovery_domain(domain: str, text: str) -> bool: |
| 52 | """Require a distinctive domain term, not a generic token such as ``AI``. |
| 53 | |
| 54 | Duplicated from pipeline._matches_discovery_domain to avoid circular imports. |
| 55 | The rule must stay in sync: pipeline.py owns the authoritative version and |
| 56 | test_reddit_listing.py verifies parity. |
| 57 | """ |
| 58 | def terms(value: str) -> Set[str]: |
| 59 | words: Set[str] = set() |
| 60 | for word in tokenize(value): |
| 61 | words.add(word) |
| 62 | if len(word) > 4 and word.endswith("s") and not word.endswith("ss"): |
| 63 | words.add(word[:-1]) |
| 64 | return words |
| 65 | |
| 66 | domain_terms = terms(domain) |
| 67 | anchors = domain_terms - _DISCOVERY_GENERIC_DOMAIN_TERMS |
| 68 | return bool((anchors or domain_terms) & terms(text)) |
| 69 | |
| 70 | |
| 71 | def _attr(tag: str, name: str) -> Optional[str]: |
| 72 | m = re.search(rf'\b{name}="([^"]*)"', tag) |
| 73 | return _html.unescape(m.group(1)) if m else None |
| 74 | |
| 75 | |
| 76 | def _to_date(value: Optional[str]) -> Optional[str]: |
| 77 | if not value: |
| 78 | return None |
| 79 | try: |
| 80 | return datetime.fromisoformat(value.strip()).date().isoformat() |
| 81 | except (ValueError, TypeError): |
| 82 | return None |
| 83 | |
| 84 | |
| 85 | def _to_epoch(value: Optional[str]) -> Optional[float]: |
| 86 | if not value: |
| 87 | return None |
| 88 | try: |
| 89 | dt = datetime.fromisoformat(value.strip()) |
| 90 | if dt.tzinfo is None: |
| 91 | dt = dt.replace(tzinfo=timezone.utc) |
| 92 | return dt.timestamp() |
| 93 | except (ValueError, TypeError): |
| 94 | return None |
| 95 | |
| 96 | |
| 97 | def _post_id(permalink: str) -> str: |
| 98 | m = re.search(r"/comments/([A-Za-z0-9]+)", permalink or "") |
| 99 | return m.group(1) if m else "" |
| 100 | |
| 101 | |
| 102 | _ERROR_PATTERN = re.compile(r"^r/(\S+)\s+(\S+):", re.IGNORECASE) |
| 103 | |
| 104 | |
| 105 | def _shreddit_error_recovered(error: str, successes: Set[tuple[str, str]]) -> bool: |
| 106 | """Return True if the error's (sub, sort) pair is in the successes set. |
| 107 | |
| 108 | Error format: "r/{sub} {sort}: {message}". |
| 109 | """ |
| 110 | m = _ERROR_PATTERN.match(error) |
| 111 | if not m: |
| 112 | return False |
| 113 | sub, sort = m.group(1).lower(), m.group(2).lower() |
| 114 | return (sub, sort) in successes |
| 115 | |
| 116 | |
| 117 | def parse_cards(html_text: str, query: str = "") -> List[Dict[str, Any]]: |
| 118 | """Parse <shreddit-post> cards into normalized post dicts with real scores.""" |
| 119 | posts: List[Dict[str, Any]] = [] |
| 120 | for m in _POST_CARD.finditer(html_text or ""): |
| 121 | tag = m.group(0) |
| 122 | permalink = _attr(tag, "permalink") or "" |
| 123 | if "/comments/" not in permalink: |
| 124 | continue |
| 125 | try: |
| 126 | score = int(_attr(tag, "score") or 0) |
| 127 | except ValueError: |
| 128 | score = 0 |
| 129 | try: |
| 130 | num_comments = int(_attr(tag, "comment-count") or 0) |
| 131 | except ValueError: |
| 132 | num_comments = 0 |
| 133 | title = _attr(tag, "post-title") or "" |
| 134 | author = _attr(tag, "author") or "[deleted]" |
| 135 | subreddit = _attr(tag, "subreddit-name") or "" |
| 136 | created = _attr(tag, "created-timestamp") |
| 137 | url = f"https://www.reddit.com{permalink}" |
| 138 | |
| 139 | posts.append({ |
| 140 | "id": "", |
| 141 | "title": title, |
| 142 | "url": url, |
| 143 | "score": score, |
| 144 | "num_comments": num_comments, |
| 145 | "subreddit": subreddit, |
| 146 | "created_utc": _to_epoch(created), |
| 147 | "author": author if author not in ("[deleted]", "[removed]") else "[deleted]", |
| 148 | "selftext": "", |
| 149 | "date": _to_date(created), |
| 150 | "engagement": { |
| 151 | "score": score, |
| 152 | "num_comments": num_comments, |
| 153 | "upvote_ratio": None, |
| 154 | }, |
| 155 | "relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0, |
| 156 | "why_relevant": "Reddit listing", |
| 157 | "metadata": {"post_id": _post_id(permalink)}, |
| 158 | }) |
| 159 | return posts |
| 160 | |
| 161 | |
| 162 | def _listing_url(subreddit: str, sort: str, timeframe: str = TIMEFRAME) -> str: |
| 163 | sub = subreddit.removeprefix("r/").strip() |
| 164 | if sub.lower() == "all": |
| 165 | url = f"https://www.reddit.com/r/all/{sort}/" |
| 166 | if sort == "top": |
| 167 | url += f"?t={timeframe}" |
| 168 | return url |
| 169 | url = f"https://www.reddit.com/svc/shreddit/community-more-posts/{sort}/?name={sub}" |
| 170 | if sort == "top": |
| 171 | url += f"&t={timeframe}" |
| 172 | return url |
| 173 | |
| 174 | |
| 175 | def _fetch_one( |
| 176 | subreddit: str, |
| 177 | sort: str, |
| 178 | query: str, |
| 179 | timeframe: str = TIMEFRAME, |
| 180 | ) -> List[Dict[str, Any]]: |
| 181 | items, _ = _fetch_one_with_status(subreddit, sort, query, timeframe) |
| 182 | return items |
| 183 | |
| 184 | |
| 185 | def _fetch_one_with_status( |
| 186 | subreddit: str, |
| 187 | sort: str, |
| 188 | query: str, |
| 189 | timeframe: str = TIMEFRAME, |
| 190 | ) -> tuple[List[Dict[str, Any]], Optional[str]]: |
| 191 | try: |
| 192 | # retry_429 records a terminal miss into the pipeline sink (issue #899) |
| 193 | # and retries a 429 once through the limiter (issue #985). An empty |
| 194 | # body ("") is a real empty listing; None never is. |
| 195 | text, error = http.reddit_keyless_get_text_retry_429( |
| 196 | _listing_url(subreddit, sort, timeframe), |
| 197 | timeout=LISTING_TIMEOUT, |
| 198 | accept="text/html", |
| 199 | ) |
| 200 | if text is None: |
| 201 | return [], (error or "no response") |
| 202 | return parse_cards(text, query), None |
| 203 | except Exception as e: |
| 204 | _log(f"listing fetch failed r/{subreddit} {sort}: {e}") |
| 205 | return [], str(e) |
| 206 | |
| 207 | |
| 208 | def _result_timeout(batch_size: int) -> float: |
| 209 | """Per-future wait: the fetch's own timeout plus the bucket's queue depth.""" |
| 210 | return LISTING_TIMEOUT + 5 + http.reddit_keyless_wait_allowance(batch_size) |
| 211 | |
| 212 | |
| 213 | def fetch_listings( |
| 214 | subreddits: List[str], |
| 215 | depth: str = "default", |
| 216 | query: str = "", |
| 217 | sorts: Optional[List[str]] = None, |
| 218 | timeframe: str = TIMEFRAME, |
| 219 | ) -> List[Dict[str, Any]]: |
| 220 | """Fetch scored post cards across subreddits × sorts. |
| 221 | |
| 222 | Returns deduped normalized posts (with real scores), unranked/unsliced — |
| 223 | the caller merges these with other sources, ranks, and slices. |
| 224 | |
| 225 | ``sorts`` overrides the depth-derived sort set. Dedicated-subreddit lanes |
| 226 | pass ``["top", "hot", "new"]`` so fresh threads (which the top-of-month |
| 227 | listing misses) are caught with their scores regardless of depth. |
| 228 | """ |
| 229 | if not subreddits: |
| 230 | return [] |
| 231 | sorts = sorts or LISTING_SORTS.get(depth, LISTING_SORTS["default"]) |
| 232 | jobs = [(sub, sort) for sub in subreddits for sort in sorts] |
| 233 | all_posts: List[Dict[str, Any]] = [] |
| 234 | with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor: |
| 235 | # submit_with_context, not executor.submit — see the note in |
| 236 | # fetch_discovery_listings below (issue #899). |
| 237 | futures = {http.submit_with_context(executor, _fetch_one, sub, sort, query, timeframe): (sub, sort) |
| 238 | for sub, sort in jobs} |
| 239 | for future in futures: |
| 240 | try: |
| 241 | all_posts.extend(future.result(timeout=_result_timeout(len(jobs)))) |
| 242 | except (Exception, FuturesTimeoutError) as e: |
| 243 | _log(f"listing future failed: {e}") |
| 244 | |
| 245 | seen: set = set() |
| 246 | unique: List[Dict[str, Any]] = [] |
| 247 | for p in all_posts: |
| 248 | if p["url"] not in seen: |
| 249 | seen.add(p["url"]) |
| 250 | unique.append(p) |
| 251 | return unique |
| 252 | |
| 253 | |
| 254 | def fetch_discovery_listings( |
| 255 | subreddits: List[str], |
| 256 | *, |
| 257 | query: str, |
| 258 | depth: str = "default", |
| 259 | ) -> Dict[str, Any]: |
| 260 | """Fetch rising/top-week listings while preserving per-feed failures. |
| 261 | |
| 262 | When shreddit fails and arctic-shift recovers, errors are cleared only for |
| 263 | subreddits whose posts survive the keyword gate. If query is empty (global |
| 264 | ``--discover`` with no domain), the gate is skipped and any arctic result |
| 265 | counts as recovery. |
| 266 | """ |
| 267 | if not subreddits: |
| 268 | return {"items": [], "errors": []} |
| 269 | jobs = [(subreddit, sort) for subreddit in subreddits for sort in ("rising", "top")] |
| 270 | items: List[Dict[str, Any]] = [] |
| 271 | errors: List[str] = [] |
| 272 | # Track which (sub, sort) pairs shreddit successfully delivered posts for. |
| 273 | # Used to decide which errors to clear — Arctic can supplement but cannot |
| 274 | # "recover" a failed hot/top/new/rising lane (it's recency-only). |
| 275 | shreddit_successes: Set[tuple[str, str]] = set() |
| 276 | with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor: |
| 277 | # submit_with_context, not executor.submit: a plain submit starts the |
| 278 | # worker with an empty context, dropping the pipeline's |
| 279 | # capture_failures() sink so a listing's 429/403 is silently discarded |
| 280 | # and the source reports a clean no-results (issue #899). |
| 281 | futures = { |
| 282 | http.submit_with_context( |
| 283 | executor, _fetch_one_with_status, subreddit, sort, query, "week" |
| 284 | ): (subreddit, sort) |
| 285 | for subreddit, sort in jobs |
| 286 | } |
| 287 | for future, (subreddit, sort) in futures.items(): |
| 288 | try: |
| 289 | fetched, error = future.result(timeout=_result_timeout(len(jobs))) |
| 290 | except (Exception, FuturesTimeoutError) as exc: |
| 291 | errors.append(f"r/{subreddit} {sort}: {exc}") |
| 292 | continue |
| 293 | items.extend(fetched) |
| 294 | if error: |
| 295 | errors.append(f"r/{subreddit} {sort}: {error}") |
| 296 | elif fetched: |
| 297 | # Shreddit succeeded for this (sub, sort) lane. |
| 298 | shreddit_successes.add((subreddit.lower(), sort.lower())) |
| 299 | |
| 300 | seen: set[str] = set() |
| 301 | unique = [] |
| 302 | for item in items: |
| 303 | if item["url"] in seen: |
| 304 | continue |
| 305 | seen.add(item["url"]) |
| 306 | unique.append(item) |
| 307 | |
| 308 | # Supplement with arctic-shift for all requested subreddits. Shreddit's |
| 309 | # per-sort success/failure is opaque (individual rising/top lanes can fail |
| 310 | # while others succeed), so arctic provides coverage for any failed lanes. |
| 311 | # Deduplication ensures no redundant posts when shreddit fully succeeded. |
| 312 | from . import reddit_arctic |
| 313 | arctic_items = reddit_arctic.fetch_listings( |
| 314 | subreddits, depth=depth, query=query, sorts=("rising", "top") |
| 315 | ) |
| 316 | if arctic_items: |
| 317 | _log(f"discovery arctic supplement: {len(arctic_items)} posts") |
| 318 | # Apply the same keyword gate that pipeline._fetch_discovery_source |
| 319 | # uses downstream. When query is empty (global --discover), skip the |
| 320 | # gate — there's no keyword to match, and the river feed IS the signal. |
| 321 | if query: |
| 322 | arctic_items = [ |
| 323 | item for item in arctic_items |
| 324 | if _matches_discovery_domain( |
| 325 | query, |
| 326 | f"{item.get('title') or ''} {item.get('selftext') or ''}", |
| 327 | ) |
| 328 | ] |
| 329 | # Merge arctic items into unique list, deduping by URL. |
| 330 | added = 0 |
| 331 | for item in arctic_items: |
| 332 | if item["url"] not in seen: |
| 333 | seen.add(item["url"]) |
| 334 | unique.append(item) |
| 335 | added += 1 |
| 336 | if added: |
| 337 | _log(f"discovery arctic supplement added {added} new posts") |
| 338 | |
| 339 | # Clear errors only for (sub, sort) pairs where shreddit succeeded. |
| 340 | # Arctic supplements recency posts but cannot "recover" a failed hot/top/ |
| 341 | # rising lane — it has no sort lanes. Errors for failed shreddit lanes are |
| 342 | # preserved even when another sort for the same subreddit succeeded. |
| 343 | if errors and shreddit_successes: |
| 344 | errors = [ |
| 345 | e for e in errors |
| 346 | if not _shreddit_error_recovered(e, shreddit_successes) |
| 347 | ] |
| 348 | return {"items": unique, "errors": errors} |
| 349 | |
| 350 | |
| 351 | def score_index(subreddits: List[str], depth: str = "default") -> Dict[str, Dict[str, int]]: |
| 352 | """Build a {post_id: {score, num_comments}} map from subreddit listings. |
| 353 | |
| 354 | Used to backfill real scores onto posts discovered via RSS, which carries |
| 355 | no engagement numbers. |
| 356 | """ |
| 357 | index: Dict[str, Dict[str, int]] = {} |
| 358 | for p in fetch_listings(subreddits, depth=depth): |
| 359 | pid = p.get("metadata", {}).get("post_id") or _post_id(p["url"]) |
| 360 | if pid: |
| 361 | index[pid] = {"score": p["score"], "num_comments": p["num_comments"]} |
| 362 | return index |
| 363 |