| 1 | """Reddit thread enrichment with real engagement metrics. |
| 2 | |
| 3 | Supports two backends: |
| 4 | 1. ScrapeCreators API (preferred) - no rate limits, 1 credit/call |
| 5 | 2. reddit.com/.json (fallback) - free but 429-prone |
| 6 | """ |
| 7 | |
| 8 | import re |
| 9 | from typing import Any, Dict, List, Optional |
| 10 | from urllib.parse import urlparse |
| 11 | |
| 12 | from . import http, dates |
| 13 | |
| 14 | |
| 15 | def extract_reddit_path(url: str) -> Optional[str]: |
| 16 | """Extract the path from a Reddit URL. |
| 17 | |
| 18 | Args: |
| 19 | url: Reddit URL |
| 20 | |
| 21 | Returns: |
| 22 | Path component or None |
| 23 | """ |
| 24 | parsed = urlparse(url) |
| 25 | if "reddit.com" not in parsed.netloc: |
| 26 | return None |
| 27 | return parsed.path |
| 28 | |
| 29 | |
| 30 | class RedditRateLimitError(Exception): |
| 31 | """Raised when Reddit returns HTTP 429 (rate limited).""" |
| 32 | pass |
| 33 | |
| 34 | |
| 35 | def fetch_thread_data( |
| 36 | url: str, |
| 37 | mock_data: Optional[Dict] = None, |
| 38 | timeout: int = 30, |
| 39 | retries: int = 3, |
| 40 | ) -> Optional[Dict[str, Any]]: |
| 41 | """Fetch Reddit thread JSON data. |
| 42 | |
| 43 | Args: |
| 44 | url: Reddit thread URL |
| 45 | mock_data: Mock data for testing |
| 46 | timeout: HTTP timeout per attempt in seconds |
| 47 | retries: Number of retries on failure |
| 48 | |
| 49 | Returns: |
| 50 | Thread data dict or None on failure |
| 51 | |
| 52 | Raises: |
| 53 | RedditRateLimitError: When Reddit returns 429 (caller should bail) |
| 54 | """ |
| 55 | if mock_data is not None: |
| 56 | return mock_data |
| 57 | |
| 58 | path = extract_reddit_path(url) |
| 59 | if not path: |
| 60 | return None |
| 61 | |
| 62 | try: |
| 63 | data = http.get_reddit_json(path, timeout=timeout, retries=retries) |
| 64 | return data |
| 65 | except http.HTTPError as e: |
| 66 | if e.status_code == 429: |
| 67 | raise RedditRateLimitError(f"Reddit rate limited (429) fetching {url}") from e |
| 68 | return None |
| 69 | |
| 70 | |
| 71 | def parse_thread_data(data: Any) -> Dict[str, Any]: |
| 72 | """Parse Reddit thread JSON into structured data. |
| 73 | |
| 74 | Args: |
| 75 | data: Raw Reddit JSON response |
| 76 | |
| 77 | Returns: |
| 78 | Dict with submission and comments data |
| 79 | """ |
| 80 | result = { |
| 81 | "submission": None, |
| 82 | "comments": [], |
| 83 | } |
| 84 | |
| 85 | if not isinstance(data, list) or len(data) < 1: |
| 86 | return result |
| 87 | |
| 88 | # First element is submission listing |
| 89 | submission_listing = data[0] |
| 90 | if isinstance(submission_listing, dict): |
| 91 | children = submission_listing.get("data", {}).get("children", []) |
| 92 | if children: |
| 93 | sub_data = children[0].get("data", {}) |
| 94 | result["submission"] = { |
| 95 | "score": sub_data.get("score"), |
| 96 | "num_comments": sub_data.get("num_comments"), |
| 97 | "upvote_ratio": sub_data.get("upvote_ratio"), |
| 98 | "created_utc": sub_data.get("created_utc"), |
| 99 | "permalink": sub_data.get("permalink"), |
| 100 | "title": sub_data.get("title"), |
| 101 | "selftext": sub_data.get("selftext", "")[:500], # Truncate |
| 102 | } |
| 103 | |
| 104 | # Second element is comments listing |
| 105 | if len(data) >= 2: |
| 106 | comments_listing = data[1] |
| 107 | if isinstance(comments_listing, dict): |
| 108 | children = comments_listing.get("data", {}).get("children", []) |
| 109 | for child in children: |
| 110 | if child.get("kind") != "t1": # t1 = comment |
| 111 | continue |
| 112 | c_data = child.get("data", {}) |
| 113 | if not c_data.get("body"): |
| 114 | continue |
| 115 | |
| 116 | comment = { |
| 117 | "score": c_data.get("score", 0), |
| 118 | "created_utc": c_data.get("created_utc"), |
| 119 | "author": c_data.get("author", "[deleted]"), |
| 120 | "body": c_data.get("body", "")[:300], # Truncate |
| 121 | "permalink": c_data.get("permalink"), |
| 122 | } |
| 123 | result["comments"].append(comment) |
| 124 | |
| 125 | return result |
| 126 | |
| 127 | |
| 128 | def get_top_comments(comments: List[Dict], limit: int = 10) -> List[Dict[str, Any]]: |
| 129 | """Get top comments sorted by score. |
| 130 | |
| 131 | Args: |
| 132 | comments: List of comment dicts |
| 133 | limit: Maximum number to return |
| 134 | |
| 135 | Returns: |
| 136 | Top comments sorted by score |
| 137 | """ |
| 138 | # Filter out deleted/removed |
| 139 | valid = [c for c in comments if c.get("author") not in ("[deleted]", "[removed]")] |
| 140 | |
| 141 | # Sort by score descending |
| 142 | sorted_comments = sorted(valid, key=lambda c: c.get("score", 0), reverse=True) |
| 143 | |
| 144 | return sorted_comments[:limit] |
| 145 | |
| 146 | |
| 147 | def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]: |
| 148 | """Extract key insights from top comments. |
| 149 | |
| 150 | Uses simple heuristics to identify valuable comments: |
| 151 | - Has substantive text |
| 152 | - Contains actionable information |
| 153 | - Not just agreement/disagreement |
| 154 | |
| 155 | Args: |
| 156 | comments: Top comments |
| 157 | limit: Max insights to extract |
| 158 | |
| 159 | Returns: |
| 160 | List of insight strings |
| 161 | """ |
| 162 | insights = [] |
| 163 | |
| 164 | for comment in comments[:limit * 2]: # Look at more comments than we need |
| 165 | body = comment.get("body", "").strip() |
| 166 | if not body or len(body) < 30: |
| 167 | continue |
| 168 | |
| 169 | # Skip low-value patterns |
| 170 | skip_patterns = [ |
| 171 | r'^(this|same|agreed|exactly|yep|nope|yes|no|thanks|thank you)\.?$', |
| 172 | r'^lol|lmao|haha', |
| 173 | r'^\[deleted\]', |
| 174 | r'^\[removed\]', |
| 175 | ] |
| 176 | if any(re.match(p, body.lower()) for p in skip_patterns): |
| 177 | continue |
| 178 | |
| 179 | # Truncate to first meaningful sentence or ~150 chars |
| 180 | insight = body[:150] |
| 181 | if len(body) > 150: |
| 182 | # Try to find a sentence boundary |
| 183 | for i, char in enumerate(insight): |
| 184 | if char in '.!?' and i > 50: |
| 185 | insight = insight[:i+1] |
| 186 | break |
| 187 | else: |
| 188 | insight = insight.rstrip() + "..." |
| 189 | |
| 190 | insights.append(insight) |
| 191 | if len(insights) >= limit: |
| 192 | break |
| 193 | |
| 194 | return insights |
| 195 | |
| 196 | |
| 197 | def enrich_reddit_item( |
| 198 | item: Dict[str, Any], |
| 199 | mock_thread_data: Optional[Dict] = None, |
| 200 | timeout: int = 10, |
| 201 | retries: int = 1, |
| 202 | ) -> Dict[str, Any]: |
| 203 | """Enrich a Reddit item with real engagement data. |
| 204 | |
| 205 | Args: |
| 206 | item: Reddit item dict |
| 207 | mock_thread_data: Mock data for testing |
| 208 | timeout: HTTP timeout per attempt (default 10s for enrichment) |
| 209 | retries: Number of retries (default 1 — fail fast for enrichment) |
| 210 | |
| 211 | Returns: |
| 212 | Enriched item dict |
| 213 | |
| 214 | Raises: |
| 215 | RedditRateLimitError: Propagated so caller can bail on remaining items |
| 216 | """ |
| 217 | url = item.get("url", "") |
| 218 | |
| 219 | # Fetch thread data (RedditRateLimitError propagates to caller) |
| 220 | thread_data = fetch_thread_data(url, mock_thread_data, timeout=timeout, retries=retries) |
| 221 | if not thread_data: |
| 222 | return item |
| 223 | |
| 224 | parsed = parse_thread_data(thread_data) |
| 225 | submission = parsed.get("submission") |
| 226 | comments = parsed.get("comments", []) |
| 227 | |
| 228 | # Update engagement metrics |
| 229 | if submission: |
| 230 | item["engagement"] = { |
| 231 | "score": submission.get("score"), |
| 232 | "num_comments": submission.get("num_comments"), |
| 233 | "upvote_ratio": submission.get("upvote_ratio"), |
| 234 | } |
| 235 | |
| 236 | # Update date from actual data |
| 237 | created_utc = submission.get("created_utc") |
| 238 | if created_utc: |
| 239 | item["date"] = dates.timestamp_to_date(created_utc) |
| 240 | |
| 241 | # Get top comments |
| 242 | top_comments = get_top_comments(comments) |
| 243 | item["top_comments"] = [] |
| 244 | for c in top_comments: |
| 245 | permalink = c.get("permalink", "") |
| 246 | comment_url = f"https://reddit.com{permalink}" if permalink else "" |
| 247 | item["top_comments"].append({ |
| 248 | "score": c.get("score", 0), |
| 249 | "date": dates.timestamp_to_date(c.get("created_utc")), |
| 250 | "author": c.get("author", ""), |
| 251 | "excerpt": c.get("body", "")[:200], |
| 252 | "url": comment_url, |
| 253 | }) |
| 254 | |
| 255 | # Extract insights |
| 256 | item["comment_insights"] = extract_comment_insights(top_comments) |
| 257 | |
| 258 | return item |
| 259 | |
| 260 | |
| 261 | def enrich_reddit_item_sc( |
| 262 | item: Dict[str, Any], |
| 263 | token: str, |
| 264 | timeout: int = 30, |
| 265 | ) -> Dict[str, Any]: |
| 266 | """Enrich a Reddit item using ScrapeCreators comment API. |
| 267 | |
| 268 | No rate limit risk. Uses 1 credit per call. |
| 269 | |
| 270 | Args: |
| 271 | item: Reddit item dict (already has engagement from search) |
| 272 | token: ScrapeCreators API key |
| 273 | timeout: HTTP timeout |
| 274 | |
| 275 | Returns: |
| 276 | Enriched item with top_comments and comment_insights |
| 277 | """ |
| 278 | from . import reddit as reddit_mod |
| 279 | |
| 280 | url = item.get("url", "") |
| 281 | if not url: |
| 282 | return item |
| 283 | |
| 284 | raw_comments = reddit_mod.fetch_post_comments(url, token) |
| 285 | if not raw_comments: |
| 286 | return item |
| 287 | |
| 288 | top_comments = [] |
| 289 | for c in raw_comments[:10]: |
| 290 | body = c.get("body", "") |
| 291 | if not body or body in ("[deleted]", "[removed]"): |
| 292 | continue |
| 293 | |
| 294 | score = c.get("ups") or c.get("score", 0) |
| 295 | author = c.get("author", "[deleted]") |
| 296 | permalink = c.get("permalink", "") |
| 297 | comment_url = f"https://reddit.com{permalink}" if permalink else "" |
| 298 | |
| 299 | top_comments.append({ |
| 300 | "score": score, |
| 301 | "date": dates.timestamp_to_date(c.get("created_utc")) if c.get("created_utc") else None, |
| 302 | "author": author, |
| 303 | "body": body[:300], |
| 304 | "excerpt": body[:200], |
| 305 | "url": comment_url, |
| 306 | }) |
| 307 | |
| 308 | top_comments.sort(key=lambda c: c.get("score", 0), reverse=True) |
| 309 | |
| 310 | item["top_comments"] = [] |
| 311 | for c in top_comments: |
| 312 | item["top_comments"].append({ |
| 313 | "score": c.get("score", 0), |
| 314 | "date": c.get("date"), |
| 315 | "author": c.get("author", ""), |
| 316 | "excerpt": c.get("excerpt", ""), |
| 317 | "url": c.get("url", ""), |
| 318 | }) |
| 319 | |
| 320 | item["comment_insights"] = extract_comment_insights(top_comments) |
| 321 | |
| 322 | return item |
| 323 |