| 1 | """Reddit public ``.json`` search module (demoted to keyless Tier 0). |
| 2 | |
| 3 | Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts |
| 4 | (shreddit anti-bot), so this is no longer the primary free path. The keyless |
| 5 | pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot |
| 6 | Tier 0 attempt — a residential machine may occasionally get a 200 — before |
| 7 | falling through to RSS discovery (reddit_rss.py) and shreddit comment |
| 8 | enrichment (reddit_shreddit.py). |
| 9 | |
| 10 | ``search_reddit_public`` is retained as a compatibility shim that delegates to |
| 11 | the keyless pipeline, so existing callers (pipeline.py) need no change. |
| 12 | |
| 13 | Endpoints (Tier 0): |
| 14 | - Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit} |
| 15 | - Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month |
| 16 | |
| 17 | Handles 429 rate limits with exponential backoff, HTML anti-bot responses, |
| 18 | network timeouts, and missing subreddits. |
| 19 | """ |
| 20 | |
| 21 | import gzip |
| 22 | import json |
| 23 | import sys |
| 24 | import time |
| 25 | import urllib.error |
| 26 | import urllib.parse |
| 27 | import urllib.request |
| 28 | from typing import Any, Dict, List, Optional |
| 29 | |
| 30 | from lib import http |
| 31 | |
| 32 | |
| 33 | USER_AGENT = ( |
| 34 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " |
| 35 | "AppleWebKit/537.36 (KHTML, like Gecko) " |
| 36 | "Chrome/124.0.0.0 Safari/537.36" |
| 37 | ) |
| 38 | |
| 39 | # Depth-aware limits for thread counts |
| 40 | DEPTH_LIMITS = { |
| 41 | "quick": 10, |
| 42 | "default": 25, |
| 43 | "deep": 50, |
| 44 | } |
| 45 | |
| 46 | MAX_RETRIES = 3 |
| 47 | BASE_BACKOFF = 2.0 # seconds |
| 48 | |
| 49 | |
| 50 | def _log(msg: str): |
| 51 | """Log to stderr.""" |
| 52 | sys.stderr.write(f"[RedditPublic] {msg}\n") |
| 53 | sys.stderr.flush() |
| 54 | |
| 55 | |
| 56 | def _url_encode(text: str) -> str: |
| 57 | """URL-encode a query string.""" |
| 58 | return urllib.parse.quote_plus(text) |
| 59 | |
| 60 | |
| 61 | def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]: |
| 62 | """Fetch JSON from a URL with retry on 429 and error handling. |
| 63 | |
| 64 | Returns parsed JSON dict, or None on unrecoverable failure. |
| 65 | """ |
| 66 | headers = { |
| 67 | "User-Agent": USER_AGENT, |
| 68 | "Accept": "application/json", |
| 69 | "Accept-Language": "en-US,en;q=0.9", |
| 70 | "Accept-Encoding": "gzip, deflate", |
| 71 | "Connection": "keep-alive", |
| 72 | } |
| 73 | req = urllib.request.Request(url, headers=headers) |
| 74 | |
| 75 | for attempt in range(MAX_RETRIES): |
| 76 | try: |
| 77 | with urllib.request.urlopen(req, timeout=timeout) as resp: |
| 78 | content_type = resp.headers.get("Content-Type", "") |
| 79 | if "json" not in content_type and "text/html" in content_type: |
| 80 | _log(f"Anti-bot HTML response (Content-Type: {content_type})") |
| 81 | return None |
| 82 | |
| 83 | raw = resp.read() |
| 84 | if resp.headers.get("Content-Encoding", "").lower() == "gzip": |
| 85 | raw = gzip.decompress(raw) |
| 86 | body = raw.decode("utf-8") |
| 87 | return json.loads(body) |
| 88 | |
| 89 | except urllib.error.HTTPError as e: |
| 90 | if e.code == 429: |
| 91 | # Reddit answers an anonymous 429 with x-ratelimit-reset and no |
| 92 | # Retry-After; honour either. See http.retry_delay_from_headers. |
| 93 | delay = http.retry_delay_from_headers( |
| 94 | getattr(e, "headers", None), |
| 95 | BASE_BACKOFF * (2 ** attempt), |
| 96 | ) |
| 97 | _log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s") |
| 98 | if attempt < MAX_RETRIES - 1: |
| 99 | time.sleep(delay) |
| 100 | continue |
| 101 | # Last attempt exhausted |
| 102 | _log("429 retries exhausted") |
| 103 | return None |
| 104 | elif e.code == 404: |
| 105 | _log(f"404 not found: {url}") |
| 106 | return None |
| 107 | elif e.code == 403: |
| 108 | _log(f"403 forbidden: {url}") |
| 109 | return None |
| 110 | else: |
| 111 | _log(f"HTTP {e.code}: {e.reason}") |
| 112 | return None |
| 113 | |
| 114 | except (urllib.error.URLError, OSError, TimeoutError) as e: |
| 115 | _log(f"Network error: {e}") |
| 116 | return None |
| 117 | |
| 118 | except json.JSONDecodeError as e: |
| 119 | _log(f"JSON decode error: {e}") |
| 120 | return None |
| 121 | |
| 122 | return None |
| 123 | |
| 124 | |
| 125 | def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: |
| 126 | """Parse Reddit listing JSON into normalized post dicts.""" |
| 127 | if not data: |
| 128 | return [] |
| 129 | |
| 130 | children = data.get("data", {}).get("children", []) |
| 131 | posts = [] |
| 132 | |
| 133 | for child in children: |
| 134 | if child.get("kind") != "t3": |
| 135 | continue |
| 136 | post = child.get("data", {}) |
| 137 | permalink = str(post.get("permalink", "")).strip() |
| 138 | if not permalink or "/comments/" not in permalink: |
| 139 | continue |
| 140 | |
| 141 | score = int(post.get("score", 0) or 0) |
| 142 | num_comments = int(post.get("num_comments", 0) or 0) |
| 143 | selftext = str(post.get("selftext", "")) |
| 144 | author = str(post.get("author", "[deleted]")) |
| 145 | created_utc = post.get("created_utc") |
| 146 | |
| 147 | # Parse date |
| 148 | date_str = None |
| 149 | if created_utc: |
| 150 | try: |
| 151 | from datetime import datetime, timezone |
| 152 | dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc) |
| 153 | date_str = dt.strftime("%Y-%m-%d") |
| 154 | except (ValueError, TypeError, OSError): |
| 155 | pass |
| 156 | |
| 157 | posts.append({ |
| 158 | "id": "", # Will be assigned after dedup |
| 159 | "title": str(post.get("title", "")).strip(), |
| 160 | "url": f"https://www.reddit.com{permalink}", |
| 161 | "score": score, |
| 162 | "num_comments": num_comments, |
| 163 | "subreddit": str(post.get("subreddit", "")).strip(), |
| 164 | "created_utc": float(created_utc) if created_utc else None, |
| 165 | "author": author if author not in ("[deleted]", "[removed]") else "[deleted]", |
| 166 | "selftext": selftext[:500] if selftext else "", |
| 167 | # Normalized fields matching ScrapeCreators output |
| 168 | "date": date_str, |
| 169 | "engagement": { |
| 170 | "score": score, |
| 171 | "num_comments": num_comments, |
| 172 | "upvote_ratio": post.get("upvote_ratio"), |
| 173 | }, |
| 174 | "relevance": _compute_relevance(score, num_comments), |
| 175 | "why_relevant": "Reddit public search", |
| 176 | "metadata": {}, |
| 177 | }) |
| 178 | |
| 179 | return posts |
| 180 | |
| 181 | |
| 182 | def _compute_relevance(score: int, num_comments: int) -> float: |
| 183 | """Estimate relevance from engagement signals.""" |
| 184 | score_component = min(1.0, max(0.0, score / 500.0)) |
| 185 | comments_component = min(1.0, max(0.0, num_comments / 200.0)) |
| 186 | return round((score_component * 0.6) + (comments_component * 0.4), 3) |
| 187 | |
| 188 | |
| 189 | def search( |
| 190 | query: str, |
| 191 | depth: str = "default", |
| 192 | subreddit: Optional[str] = None, |
| 193 | timeout: int = 15, |
| 194 | ) -> List[Dict[str, Any]]: |
| 195 | """Search Reddit via the public JSON endpoint. |
| 196 | |
| 197 | Args: |
| 198 | query: Search query string |
| 199 | depth: 'quick', 'default', or 'deep' — controls result limit |
| 200 | subreddit: Optional subreddit name (without r/) for scoped search |
| 201 | timeout: HTTP timeout in seconds |
| 202 | |
| 203 | Returns: |
| 204 | List of normalized post dicts. Empty list on any failure. |
| 205 | """ |
| 206 | limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"]) |
| 207 | encoded_query = _url_encode(query) |
| 208 | |
| 209 | if subreddit: |
| 210 | sub = subreddit.removeprefix("r/").strip() |
| 211 | url = ( |
| 212 | f"https://www.reddit.com/r/{sub}/search.json" |
| 213 | f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1" |
| 214 | ) |
| 215 | else: |
| 216 | url = ( |
| 217 | f"https://www.reddit.com/search.json" |
| 218 | f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1" |
| 219 | ) |
| 220 | |
| 221 | data = _fetch_json(url, timeout=timeout) |
| 222 | posts = _parse_posts(data) |
| 223 | |
| 224 | # Dedupe by URL and assign IDs |
| 225 | seen_urls = set() |
| 226 | unique = [] |
| 227 | for post in posts: |
| 228 | if post["url"] not in seen_urls: |
| 229 | seen_urls.add(post["url"]) |
| 230 | unique.append(post) |
| 231 | |
| 232 | for i, post in enumerate(unique): |
| 233 | post["id"] = f"R{i + 1}" |
| 234 | |
| 235 | return unique[:limit] |
| 236 | |
| 237 | |
| 238 | def search_reddit_public( |
| 239 | topic: str, |
| 240 | from_date: str, |
| 241 | to_date: str, |
| 242 | depth: str = "default", |
| 243 | subreddits: Optional[List[str]] = None, |
| 244 | dedicated_subreddits: Optional[List[str]] = None, |
| 245 | ) -> List[Dict[str, Any]]: |
| 246 | """High-level free Reddit search + enrichment (keyless). |
| 247 | |
| 248 | Thin compatibility shim over the keyless pipeline: the legacy ``.json`` |
| 249 | search/enrichment endpoints now return HTTP 403, so this delegates to |
| 250 | ``reddit_keyless.search_and_enrich`` (dedicated-sub listings + RSS discovery |
| 251 | → shreddit comment enrichment; no ``.json`` search). The name and signature |
| 252 | are preserved so ``pipeline.py`` and other callers need no change and the |
| 253 | ScrapeCreators backup still engages when this returns empty. |
| 254 | |
| 255 | The module-level ``search`` / ``_parse_posts`` helpers remain as a |
| 256 | standalone ``.json`` search utility (own test coverage), no longer wired |
| 257 | into the keyless production path. |
| 258 | |
| 259 | Args: |
| 260 | topic: Search topic |
| 261 | from_date: Start date (YYYY-MM-DD) |
| 262 | to_date: End date (YYYY-MM-DD) |
| 263 | depth: 'quick', 'default', or 'deep' |
| 264 | subreddits: Optional list of subreddit names (without r/) for targeted search |
| 265 | |
| 266 | Returns: |
| 267 | List of normalized item dicts matching ScrapeCreators output format. |
| 268 | Empty list on total failure (so SC backup can engage). |
| 269 | """ |
| 270 | from . import reddit_keyless |
| 271 | return reddit_keyless.search_and_enrich( |
| 272 | topic, from_date, to_date, depth=depth, subreddits=subreddits, |
| 273 | dedicated_subreddits=dedicated_subreddits, |
| 274 | ) |
| 275 |