| 1 | """Reddit public ``.json`` search module (demoted to keyless Tier 0). |
| 2 | |
| 3 | Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts |
| 4 | (shreddit anti-bot), so this is no longer the primary free path. The keyless |
| 5 | pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot |
| 6 | Tier 0 attempt — a residential machine may occasionally get a 200 — before |
| 7 | falling through to RSS discovery (reddit_rss.py) and shreddit comment |
| 8 | enrichment (reddit_shreddit.py). |
| 9 | |
| 10 | ``search_reddit_public`` is retained as a compatibility shim that delegates to |
| 11 | the keyless pipeline, so existing callers (pipeline.py) need no change. |
| 12 | |
| 13 | Endpoints (Tier 0): |
| 14 | - Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit} |
| 15 | - Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month |
| 16 | |
| 17 | Handles 429 rate limits with exponential backoff, HTML anti-bot responses, |
| 18 | network timeouts, and missing subreddits. |
| 19 | """ |
| 20 | |
| 21 | import gzip |
| 22 | import json |
| 23 | import sys |
| 24 | import time |
| 25 | import urllib.error |
| 26 | import urllib.parse |
| 27 | import urllib.request |
| 28 | from typing import Any, Dict, List, Optional |
| 29 | |
| 30 | |
| 31 | USER_AGENT = ( |
| 32 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " |
| 33 | "AppleWebKit/537.36 (KHTML, like Gecko) " |
| 34 | "Chrome/124.0.0.0 Safari/537.36" |
| 35 | ) |
| 36 | |
| 37 | # Depth-aware limits for thread counts |
| 38 | DEPTH_LIMITS = { |
| 39 | "quick": 10, |
| 40 | "default": 25, |
| 41 | "deep": 50, |
| 42 | } |
| 43 | |
| 44 | MAX_RETRIES = 3 |
| 45 | BASE_BACKOFF = 2.0 # seconds |
| 46 | |
| 47 | |
| 48 | def _log(msg: str): |
| 49 | """Log to stderr.""" |
| 50 | sys.stderr.write(f"[RedditPublic] {msg}\n") |
| 51 | sys.stderr.flush() |
| 52 | |
| 53 | |
| 54 | def _url_encode(text: str) -> str: |
| 55 | """URL-encode a query string.""" |
| 56 | return urllib.parse.quote_plus(text) |
| 57 | |
| 58 | |
| 59 | def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]: |
| 60 | """Fetch JSON from a URL with retry on 429 and error handling. |
| 61 | |
| 62 | Returns parsed JSON dict, or None on unrecoverable failure. |
| 63 | """ |
| 64 | headers = { |
| 65 | "User-Agent": USER_AGENT, |
| 66 | "Accept": "application/json", |
| 67 | "Accept-Language": "en-US,en;q=0.9", |
| 68 | "Accept-Encoding": "gzip, deflate", |
| 69 | "Connection": "keep-alive", |
| 70 | } |
| 71 | req = urllib.request.Request(url, headers=headers) |
| 72 | |
| 73 | for attempt in range(MAX_RETRIES): |
| 74 | try: |
| 75 | with urllib.request.urlopen(req, timeout=timeout) as resp: |
| 76 | content_type = resp.headers.get("Content-Type", "") |
| 77 | if "json" not in content_type and "text/html" in content_type: |
| 78 | _log(f"Anti-bot HTML response (Content-Type: {content_type})") |
| 79 | return None |
| 80 | |
| 81 | raw = resp.read() |
| 82 | if resp.headers.get("Content-Encoding", "").lower() == "gzip": |
| 83 | raw = gzip.decompress(raw) |
| 84 | body = raw.decode("utf-8") |
| 85 | return json.loads(body) |
| 86 | |
| 87 | except urllib.error.HTTPError as e: |
| 88 | if e.code == 429: |
| 89 | delay = BASE_BACKOFF * (2 ** attempt) |
| 90 | retry_after = None |
| 91 | if hasattr(e, "headers"): |
| 92 | retry_after = e.headers.get("Retry-After") |
| 93 | if retry_after: |
| 94 | try: |
| 95 | delay = float(retry_after) |
| 96 | except ValueError: |
| 97 | pass |
| 98 | _log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s") |
| 99 | if attempt < MAX_RETRIES - 1: |
| 100 | time.sleep(delay) |
| 101 | continue |
| 102 | # Last attempt exhausted |
| 103 | _log("429 retries exhausted") |
| 104 | return None |
| 105 | elif e.code == 404: |
| 106 | _log(f"404 not found: {url}") |
| 107 | return None |
| 108 | elif e.code == 403: |
| 109 | _log(f"403 forbidden: {url}") |
| 110 | return None |
| 111 | else: |
| 112 | _log(f"HTTP {e.code}: {e.reason}") |
| 113 | return None |
| 114 | |
| 115 | except (urllib.error.URLError, OSError, TimeoutError) as e: |
| 116 | _log(f"Network error: {e}") |
| 117 | return None |
| 118 | |
| 119 | except json.JSONDecodeError as e: |
| 120 | _log(f"JSON decode error: {e}") |
| 121 | return None |
| 122 | |
| 123 | return None |
| 124 | |
| 125 | |
| 126 | def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: |
| 127 | """Parse Reddit listing JSON into normalized post dicts.""" |
| 128 | if not data: |
| 129 | return [] |
| 130 | |
| 131 | children = data.get("data", {}).get("children", []) |
| 132 | posts = [] |
| 133 | |
| 134 | for child in children: |
| 135 | if child.get("kind") != "t3": |
| 136 | continue |
| 137 | post = child.get("data", {}) |
| 138 | permalink = str(post.get("permalink", "")).strip() |
| 139 | if not permalink or "/comments/" not in permalink: |
| 140 | continue |
| 141 | |
| 142 | score = int(post.get("score", 0) or 0) |
| 143 | num_comments = int(post.get("num_comments", 0) or 0) |
| 144 | selftext = str(post.get("selftext", "")) |
| 145 | author = str(post.get("author", "[deleted]")) |
| 146 | created_utc = post.get("created_utc") |
| 147 | |
| 148 | # Parse date |
| 149 | date_str = None |
| 150 | if created_utc: |
| 151 | try: |
| 152 | from datetime import datetime, timezone |
| 153 | dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc) |
| 154 | date_str = dt.strftime("%Y-%m-%d") |
| 155 | except (ValueError, TypeError, OSError): |
| 156 | pass |
| 157 | |
| 158 | posts.append({ |
| 159 | "id": "", # Will be assigned after dedup |
| 160 | "title": str(post.get("title", "")).strip(), |
| 161 | "url": f"https://www.reddit.com{permalink}", |
| 162 | "score": score, |
| 163 | "num_comments": num_comments, |
| 164 | "subreddit": str(post.get("subreddit", "")).strip(), |
| 165 | "created_utc": float(created_utc) if created_utc else None, |
| 166 | "author": author if author not in ("[deleted]", "[removed]") else "[deleted]", |
| 167 | "selftext": selftext[:500] if selftext else "", |
| 168 | # Normalized fields matching ScrapeCreators output |
| 169 | "date": date_str, |
| 170 | "engagement": { |
| 171 | "score": score, |
| 172 | "num_comments": num_comments, |
| 173 | "upvote_ratio": post.get("upvote_ratio"), |
| 174 | }, |
| 175 | "relevance": _compute_relevance(score, num_comments), |
| 176 | "why_relevant": "Reddit public search", |
| 177 | "metadata": {}, |
| 178 | }) |
| 179 | |
| 180 | return posts |
| 181 | |
| 182 | |
| 183 | def _compute_relevance(score: int, num_comments: int) -> float: |
| 184 | """Estimate relevance from engagement signals.""" |
| 185 | score_component = min(1.0, max(0.0, score / 500.0)) |
| 186 | comments_component = min(1.0, max(0.0, num_comments / 200.0)) |
| 187 | return round((score_component * 0.6) + (comments_component * 0.4), 3) |
| 188 | |
| 189 | |
| 190 | def search( |
| 191 | query: str, |
| 192 | depth: str = "default", |
| 193 | subreddit: Optional[str] = None, |
| 194 | timeout: int = 15, |
| 195 | ) -> List[Dict[str, Any]]: |
| 196 | """Search Reddit via the public JSON endpoint. |
| 197 | |
| 198 | Args: |
| 199 | query: Search query string |
| 200 | depth: 'quick', 'default', or 'deep' — controls result limit |
| 201 | subreddit: Optional subreddit name (without r/) for scoped search |
| 202 | timeout: HTTP timeout in seconds |
| 203 | |
| 204 | Returns: |
| 205 | List of normalized post dicts. Empty list on any failure. |
| 206 | """ |
| 207 | limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"]) |
| 208 | encoded_query = _url_encode(query) |
| 209 | |
| 210 | if subreddit: |
| 211 | sub = subreddit.removeprefix("r/").strip() |
| 212 | url = ( |
| 213 | f"https://www.reddit.com/r/{sub}/search.json" |
| 214 | f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1" |
| 215 | ) |
| 216 | else: |
| 217 | url = ( |
| 218 | f"https://www.reddit.com/search.json" |
| 219 | f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1" |
| 220 | ) |
| 221 | |
| 222 | data = _fetch_json(url, timeout=timeout) |
| 223 | posts = _parse_posts(data) |
| 224 | |
| 225 | # Dedupe by URL and assign IDs |
| 226 | seen_urls = set() |
| 227 | unique = [] |
| 228 | for post in posts: |
| 229 | if post["url"] not in seen_urls: |
| 230 | seen_urls.add(post["url"]) |
| 231 | unique.append(post) |
| 232 | |
| 233 | for i, post in enumerate(unique): |
| 234 | post["id"] = f"R{i + 1}" |
| 235 | |
| 236 | return unique[:limit] |
| 237 | |
| 238 | |
| 239 | def search_reddit_public( |
| 240 | topic: str, |
| 241 | from_date: str, |
| 242 | to_date: str, |
| 243 | depth: str = "default", |
| 244 | subreddits: Optional[List[str]] = None, |
| 245 | dedicated_subreddits: Optional[List[str]] = None, |
| 246 | ) -> List[Dict[str, Any]]: |
| 247 | """High-level free Reddit search + enrichment (keyless). |
| 248 | |
| 249 | Thin compatibility shim over the keyless pipeline: the legacy ``.json`` |
| 250 | search/enrichment endpoints now return HTTP 403, so this delegates to |
| 251 | ``reddit_keyless.search_and_enrich`` (dedicated-sub listings + RSS discovery |
| 252 | → shreddit comment enrichment; no ``.json`` search). The name and signature |
| 253 | are preserved so ``pipeline.py`` and other callers need no change and the |
| 254 | ScrapeCreators backup still engages when this returns empty. |
| 255 | |
| 256 | The module-level ``search`` / ``_parse_posts`` helpers remain as a |
| 257 | standalone ``.json`` search utility (own test coverage), no longer wired |
| 258 | into the keyless production path. |
| 259 | |
| 260 | Args: |
| 261 | topic: Search topic |
| 262 | from_date: Start date (YYYY-MM-DD) |
| 263 | to_date: End date (YYYY-MM-DD) |
| 264 | depth: 'quick', 'default', or 'deep' |
| 265 | subreddits: Optional list of subreddit names (without r/) for targeted search |
| 266 | |
| 267 | Returns: |
| 268 | List of normalized item dicts matching ScrapeCreators output format. |
| 269 | Empty list on total failure (so SC backup can engage). |
| 270 | """ |
| 271 | from . import reddit_keyless |
| 272 | return reddit_keyless.search_and_enrich( |
| 273 | topic, from_date, to_date, depth=depth, subreddits=subreddits, |
| 274 | dedicated_subreddits=dedicated_subreddits, |
| 275 | ) |
| 276 |