| 1 | """Keyless Reddit discovery via public RSS/Atom feeds. |
| 2 | |
| 3 | Reddit's ``.json`` search endpoints now return HTTP 403 (shreddit anti-bot). |
| 4 | RSS feeds still serve HTTP 200 with no API key, so this module uses them for |
| 5 | post discovery, replacing ``reddit_public.search`` as the free search path. |
| 6 | |
| 7 | Two feed families are combined and deduped: |
| 8 | - search: /search.rss?q=... and /r/{sub}/search.rss?q=...&restrict_sr=on |
| 9 | - listing: /r/{sub}/{top,hot}.rss?t=month |
| 10 | |
| 11 | RSS entries carry no engagement score, so ``score``/``num_comments`` start at 0 |
| 12 | and are backfilled during shreddit enrichment (see reddit_shreddit.py). Output |
| 13 | dicts match the normalized shape emitted by ``reddit_public._parse_posts`` so |
| 14 | downstream code (pipeline, renderer) is unaffected. |
| 15 | """ |
| 16 | |
| 17 | import sys |
| 18 | import xml.etree.ElementTree as ET |
| 19 | from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError |
| 20 | from datetime import datetime, timezone |
| 21 | from typing import Any, Dict, List, Optional |
| 22 | from urllib.parse import quote_plus |
| 23 | |
| 24 | from . import http |
| 25 | from .relevance import token_overlap_relevance |
| 26 | |
| 27 | ATOM = "{http://www.w3.org/2005/Atom}" |
| 28 | |
| 29 | # Mirror reddit_public depth-aware limits so the two free paths behave alike. |
| 30 | DEPTH_LIMITS = { |
| 31 | "quick": 10, |
| 32 | "default": 25, |
| 33 | "deep": 50, |
| 34 | } |
| 35 | |
| 36 | # Listing sorts pulled per subreddit (in addition to search), for volume. |
| 37 | LISTING_SORTS = { |
| 38 | "quick": ["top"], |
| 39 | "default": ["top", "hot"], |
| 40 | "deep": ["top", "hot", "new"], |
| 41 | } |
| 42 | |
| 43 | MAX_WORKERS = 4 |
| 44 | FEED_TIMEOUT = 15 |
| 45 | |
| 46 | |
| 47 | def _log(msg: str) -> None: |
| 48 | sys.stderr.write(f"[RedditRSS] {msg}\n") |
| 49 | sys.stderr.flush() |
| 50 | |
| 51 | |
| 52 | def _iso_to_date(value: Optional[str]) -> Optional[str]: |
| 53 | """Parse an ISO-8601 timestamp (e.g. 2026-05-20T18:48:31+00:00) to YYYY-MM-DD.""" |
| 54 | if not value: |
| 55 | return None |
| 56 | try: |
| 57 | dt = datetime.fromisoformat(value.strip()) |
| 58 | return dt.date().isoformat() |
| 59 | except (ValueError, TypeError): |
| 60 | return None |
| 61 | |
| 62 | |
| 63 | def _iso_to_epoch(value: Optional[str]) -> Optional[float]: |
| 64 | if not value: |
| 65 | return None |
| 66 | try: |
| 67 | dt = datetime.fromisoformat(value.strip()) |
| 68 | if dt.tzinfo is None: |
| 69 | dt = dt.replace(tzinfo=timezone.utc) |
| 70 | return dt.timestamp() |
| 71 | except (ValueError, TypeError): |
| 72 | return None |
| 73 | |
| 74 | |
| 75 | def _subreddit_from(category: str, url: str) -> str: |
| 76 | """Derive subreddit name from the entry category or, failing that, the URL.""" |
| 77 | if category: |
| 78 | return category |
| 79 | # URL form: https://www.reddit.com/r/{sub}/comments/{id}/... |
| 80 | parts = url.split("/r/", 1) |
| 81 | if len(parts) == 2: |
| 82 | return parts[1].split("/", 1)[0] |
| 83 | return "" |
| 84 | |
| 85 | |
| 86 | def _parse_feed(xml_text: str, query: str = "") -> List[Dict[str, Any]]: |
| 87 | """Parse an Atom feed string into normalized post dicts. Never raises.""" |
| 88 | if not xml_text: |
| 89 | return [] |
| 90 | try: |
| 91 | root = ET.fromstring(xml_text) |
| 92 | except ET.ParseError as e: |
| 93 | _log(f"feed parse error: {e}") |
| 94 | return [] |
| 95 | |
| 96 | posts: List[Dict[str, Any]] = [] |
| 97 | for entry in root.iter(f"{ATOM}entry"): |
| 98 | link_el = entry.find(f"{ATOM}link") |
| 99 | url = link_el.get("href", "").strip() if link_el is not None else "" |
| 100 | if not url or "/comments/" not in url: |
| 101 | continue |
| 102 | |
| 103 | title_el = entry.find(f"{ATOM}title") |
| 104 | title = (title_el.text or "").strip() if title_el is not None else "" |
| 105 | |
| 106 | author = "" |
| 107 | author_el = entry.find(f"{ATOM}author/{ATOM}name") |
| 108 | if author_el is not None and author_el.text: |
| 109 | author = author_el.text.strip().removeprefix("/u/").removeprefix("u/") |
| 110 | if author in ("[deleted]", "[removed]", ""): |
| 111 | author = "[deleted]" |
| 112 | |
| 113 | cat_el = entry.find(f"{ATOM}category") |
| 114 | category = cat_el.get("term", "").strip() if cat_el is not None else "" |
| 115 | subreddit = _subreddit_from(category, url) |
| 116 | |
| 117 | updated_el = entry.find(f"{ATOM}updated") |
| 118 | updated = (updated_el.text or "").strip() if updated_el is not None else "" |
| 119 | |
| 120 | content_el = entry.find(f"{ATOM}content") |
| 121 | selftext = "" |
| 122 | if content_el is not None and content_el.text: |
| 123 | # Strip the simplest HTML; renderer only needs an excerpt. |
| 124 | import re as _re |
| 125 | selftext = _re.sub(r"<[^>]+>", " ", content_el.text) |
| 126 | selftext = _re.sub(r"\s+", " ", selftext).strip()[:500] |
| 127 | |
| 128 | relevance = round(token_overlap_relevance(query, title), 3) if query else 0.0 |
| 129 | |
| 130 | posts.append({ |
| 131 | "id": "", # assigned after dedup |
| 132 | "title": title, |
| 133 | "url": url, |
| 134 | "score": 0, # backfilled by shreddit enrichment |
| 135 | "num_comments": 0, # backfilled by shreddit enrichment |
| 136 | "subreddit": subreddit, |
| 137 | "created_utc": _iso_to_epoch(updated), |
| 138 | "author": author, |
| 139 | "selftext": selftext, |
| 140 | "date": _iso_to_date(updated), |
| 141 | "engagement": { |
| 142 | "score": 0, |
| 143 | "num_comments": 0, |
| 144 | "upvote_ratio": None, |
| 145 | }, |
| 146 | "relevance": relevance, |
| 147 | "why_relevant": "Reddit RSS", |
| 148 | "metadata": {}, |
| 149 | }) |
| 150 | |
| 151 | return posts |
| 152 | |
| 153 | |
| 154 | def _build_urls(query: str, depth: str, subreddits: Optional[List[str]]) -> List[str]: |
| 155 | """Build the keyless RSS feed URLs to fan out across.""" |
| 156 | q = quote_plus(query) |
| 157 | urls: List[str] = [ |
| 158 | f"https://www.reddit.com/search.rss?q={q}&sort=relevance&t=month" |
| 159 | ] |
| 160 | for raw_sub in (subreddits or []): |
| 161 | sub = raw_sub.removeprefix("r/").strip() |
| 162 | if not sub: |
| 163 | continue |
| 164 | urls.append( |
| 165 | f"https://www.reddit.com/r/{sub}/search.rss" |
| 166 | f"?q={q}&restrict_sr=on&sort=relevance&t=month" |
| 167 | ) |
| 168 | for sort in LISTING_SORTS.get(depth, LISTING_SORTS["default"]): |
| 169 | urls.append(f"https://www.reddit.com/r/{sub}/{sort}.rss?t=month") |
| 170 | return urls |
| 171 | |
| 172 | |
| 173 | def _fetch_feed(url: str, query: str) -> List[Dict[str, Any]]: |
| 174 | """Fetch and parse one feed. Never raises.""" |
| 175 | try: |
| 176 | text = http.reddit_keyless_get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml") |
| 177 | return _parse_feed(text, query) if text else [] |
| 178 | except Exception as e: # defensive: a single bad feed must not sink the run |
| 179 | _log(f"feed fetch failed for {url}: {e}") |
| 180 | return [] |
| 181 | |
| 182 | |
| 183 | def search_rss( |
| 184 | query: str, |
| 185 | depth: str = "default", |
| 186 | subreddits: Optional[List[str]] = None, |
| 187 | ) -> List[Dict[str, Any]]: |
| 188 | """Discover Reddit posts for a query via keyless RSS feeds. |
| 189 | |
| 190 | Args: |
| 191 | query: Search query string |
| 192 | depth: 'quick', 'default', or 'deep' — controls result limit and feeds |
| 193 | subreddits: Optional pre-resolved subreddit names (without r/) to target |
| 194 | |
| 195 | Returns: |
| 196 | List of normalized post dicts (deduped by URL, capped by depth), |
| 197 | with placeholder scores to be backfilled during enrichment. |
| 198 | Empty list on any failure. |
| 199 | """ |
| 200 | limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"]) |
| 201 | urls = _build_urls(query, depth, subreddits) |
| 202 | |
| 203 | all_posts: List[Dict[str, Any]] = [] |
| 204 | workers = min(MAX_WORKERS, len(urls)) or 1 |
| 205 | with ThreadPoolExecutor(max_workers=workers) as executor: |
| 206 | # submit_with_context, not executor.submit: a plain submit starts the |
| 207 | # worker with an empty context, dropping the pipeline's |
| 208 | # capture_failures() sink so a feed's 429/403 is silently discarded and |
| 209 | # the source reports a clean no-results (issue #899). |
| 210 | futures = { |
| 211 | http.submit_with_context(executor, _fetch_feed, url, query): url |
| 212 | for url in urls |
| 213 | } |
| 214 | for future in futures: |
| 215 | try: |
| 216 | all_posts.extend(future.result(timeout=FEED_TIMEOUT + 5)) |
| 217 | except (Exception, FuturesTimeoutError) as e: |
| 218 | _log(f"feed future failed: {e}") |
| 219 | |
| 220 | # Dedupe by URL (first occurrence wins). |
| 221 | seen: set = set() |
| 222 | unique: List[Dict[str, Any]] = [] |
| 223 | for post in all_posts: |
| 224 | if post["url"] not in seen: |
| 225 | seen.add(post["url"]) |
| 226 | unique.append(post) |
| 227 | |
| 228 | for i, post in enumerate(unique): |
| 229 | post["id"] = f"R{i + 1}" |
| 230 | |
| 231 | return unique[:limit] |
| 232 |