| 1 | """Keyless Reddit comment enrichment via shreddit /svc endpoints. |
| 2 | |
| 3 | Reddit's ``{thread}.json`` endpoint now returns HTTP 403. The shreddit partial |
| 4 | endpoint ``/svc/shreddit/comments/r/{sub}/t3_{id}`` still serves HTTP 200 HTML |
| 5 | with no API key, embedding each comment as a ``<shreddit-comment>`` custom |
| 6 | element whose start-tag attributes carry ``score`` / ``author`` / ``created`` / |
| 7 | ``permalink``, and whose body lives in a ``<div id="{thingId}-post-rtjson-content">`` |
| 8 | block. This module parses that markup into top comments, matching the |
| 9 | ``top_comments`` / ``comment_insights`` shape produced by ``reddit_enrich`` so |
| 10 | the renderer is unaffected. |
| 11 | |
| 12 | Limitation: the comments endpoint carries the real comment count |
| 13 | (``total-comments``) but not the post's upvote score, so post-level ``score`` |
| 14 | cannot be recovered keylessly here (ScrapeCreators backup still provides it). |
| 15 | """ |
| 16 | |
| 17 | import html as _html |
| 18 | import re |
| 19 | import sys |
| 20 | from datetime import datetime |
| 21 | from typing import Any, Dict, List, Optional |
| 22 | |
| 23 | from . import http |
| 24 | from . import reddit_enrich |
| 25 | |
| 26 | # Up to N posts enriched per subquery, by depth. Raised from 3/5/8 once the |
| 27 | # per-command memo (http.reddit_keyless_get_text) collapsed repeat shreddit |
| 28 | # fetches across subqueries and the 1 req/s bucket stopped the 429s: eight |
| 29 | # fetches fit inside ENRICH_BUDGET at four workers, and the comments are the |
| 30 | # lane's headline value. |
| 31 | ENRICH_LIMITS = { |
| 32 | "quick": 4, |
| 33 | "default": 8, |
| 34 | "deep": 12, |
| 35 | } |
| 36 | |
| 37 | # Max comments returned per post (independent of how many posts get enriched). |
| 38 | # Twelve so a thousand-comment thread feeds more than ten candidates into the |
| 39 | # cross-platform Top Community Comments block. |
| 40 | MAX_COMMENTS = 12 |
| 41 | |
| 42 | SVC_TIMEOUT = 12 |
| 43 | |
| 44 | # Known bots whose comments carry no community signal. |
| 45 | BOT_AUTHORS = frozenset({ |
| 46 | "automoderator", |
| 47 | "remindmebot", |
| 48 | "repostsleuthbot", |
| 49 | "sneakpeekbot", |
| 50 | "savevideo", |
| 51 | "videodownloadbot", |
| 52 | "totesmessenger", |
| 53 | "b0trank", |
| 54 | "amputatorbot", |
| 55 | "stabbot", |
| 56 | "gifreversingbot", |
| 57 | "haikubotinaction", |
| 58 | "imagesofnetwork", |
| 59 | "botdefense", |
| 60 | }) |
| 61 | |
| 62 | _BOT_SUFFIXES = ("-bot", "_bot") |
| 63 | |
| 64 | # CamelCase catches WikiTextBot without swallowing names such as Talbot. |
| 65 | _CAMEL_BOT = re.compile(r"[a-z0-9]Bot\d*$") |
| 66 | |
| 67 | # Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree> |
| 68 | # or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>'). |
| 69 | _COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>") |
| 70 | _TOTAL_COMMENTS = re.compile(r'total-comments="(\d+)"') |
| 71 | _PARA = re.compile(r"<p[^>]*>(.*?)</p>", re.S) |
| 72 | _TAG = re.compile(r"<[^>]+>") |
| 73 | _WS = re.compile(r"\s+") |
| 74 | _NEXT_RTJSON = re.compile(r'id="t1_[A-Za-z0-9]+-(?:comment|post)-rtjson-content"') |
| 75 | |
| 76 | |
| 77 | def _log(msg: str) -> None: |
| 78 | sys.stderr.write(f"[RedditShreddit] {msg}\n") |
| 79 | sys.stderr.flush() |
| 80 | |
| 81 | |
| 82 | def extract_post_ref(url: str) -> Optional[tuple]: |
| 83 | """Return (subreddit, post_id) from a Reddit thread URL, or None.""" |
| 84 | m = re.search(r"/r/([^/]+)/comments/([A-Za-z0-9]+)", url or "") |
| 85 | if not m: |
| 86 | return None |
| 87 | return m.group(1), m.group(2) |
| 88 | |
| 89 | |
| 90 | def _svc_url(subreddit: str, post_id: str) -> str: |
| 91 | # sort=top guarantees Reddit front-loads the highest-scored comments on the |
| 92 | # first page, so the true top comments are captured even on huge threads |
| 93 | # (we still re-sort by score locally as a backstop). |
| 94 | return ( |
| 95 | f"https://www.reddit.com/svc/shreddit/comments/r/{subreddit}/t3_{post_id}" |
| 96 | f"?sort=top" |
| 97 | ) |
| 98 | |
| 99 | |
| 100 | def _attr(tag: str, name: str) -> str: |
| 101 | m = re.search(rf'\b{name}="([^"]*)"', tag) |
| 102 | return _html.unescape(m.group(1)) if m else "" |
| 103 | |
| 104 | |
| 105 | def _iso_to_date(value: str) -> Optional[str]: |
| 106 | if not value: |
| 107 | return None |
| 108 | try: |
| 109 | return datetime.fromisoformat(value.strip()).date().isoformat() |
| 110 | except (ValueError, TypeError): |
| 111 | return None |
| 112 | |
| 113 | |
| 114 | def _body_for(html_text: str, thing_id: str) -> str: |
| 115 | """Extract a comment's text body, anchored on its unique thingId. |
| 116 | |
| 117 | The body div id embeds the comment's thingId, so this assigns body→comment |
| 118 | correctly even for nested replies. The slice is bounded by the next |
| 119 | comment's rtjson anchor to avoid swallowing child-comment text. |
| 120 | """ |
| 121 | if not thing_id: |
| 122 | return "" |
| 123 | anchor = f'id="{thing_id}-post-rtjson-content"' |
| 124 | idx = html_text.find(anchor) |
| 125 | if idx == -1: |
| 126 | return "" |
| 127 | window = html_text[idx + len(anchor): idx + len(anchor) + 8000] |
| 128 | nxt = _NEXT_RTJSON.search(window) |
| 129 | if nxt: |
| 130 | window = window[: nxt.start()] |
| 131 | paras = _PARA.findall(window) |
| 132 | if not paras: |
| 133 | return "" |
| 134 | text = " ".join(_TAG.sub("", p) for p in paras) |
| 135 | return _WS.sub(" ", _html.unescape(text)).strip() |
| 136 | |
| 137 | |
| 138 | def _is_bot_author(author: str) -> bool: |
| 139 | """Whether an author is a bot whose comments carry no community signal.""" |
| 140 | raw = (author or "").strip() |
| 141 | if not raw: |
| 142 | return False |
| 143 | name = raw.lower() |
| 144 | return (name in BOT_AUTHORS |
| 145 | or name.endswith(_BOT_SUFFIXES) |
| 146 | or bool(_CAMEL_BOT.search(raw))) |
| 147 | |
| 148 | |
| 149 | def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]: |
| 150 | """Parse <shreddit-comment> elements into scored comment dicts (sorted desc). |
| 151 | |
| 152 | Deleted, removed, and bot authors are dropped: they occupy top-comment slots |
| 153 | on high-traffic threads without saying anything about the topic. |
| 154 | """ |
| 155 | comments: List[Dict[str, Any]] = [] |
| 156 | for m in _COMMENT_START.finditer(html_text or ""): |
| 157 | tag = m.group(0) |
| 158 | author = _attr(tag, "author") or "[deleted]" |
| 159 | if author in ("[deleted]", "[removed]") or _is_bot_author(author): |
| 160 | continue |
| 161 | thing_id = _attr(tag, "thingId") |
| 162 | body = _body_for(html_text, thing_id) |
| 163 | if not body or body in ("[deleted]", "[removed]"): |
| 164 | continue |
| 165 | try: |
| 166 | score = int(_attr(tag, "score") or 0) |
| 167 | except ValueError: |
| 168 | score = 0 |
| 169 | permalink = _attr(tag, "permalink") |
| 170 | comments.append({ |
| 171 | "score": score, |
| 172 | "author": author, |
| 173 | "body": body[:300], |
| 174 | "excerpt": body[:200], |
| 175 | "permalink": permalink, |
| 176 | "date": _iso_to_date(_attr(tag, "created")), |
| 177 | "url": f"https://reddit.com{permalink}" if permalink else "", |
| 178 | }) |
| 179 | |
| 180 | comments.sort(key=lambda c: c.get("score", 0), reverse=True) |
| 181 | return comments[:limit] |
| 182 | |
| 183 | |
| 184 | def _total_comments(html_text: str) -> Optional[int]: |
| 185 | m = _TOTAL_COMMENTS.search(html_text or "") |
| 186 | return int(m.group(1)) if m else None |
| 187 | |
| 188 | |
| 189 | def fetch_comments( |
| 190 | post_url: str, |
| 191 | timeout: int = SVC_TIMEOUT, |
| 192 | ) -> Dict[str, Any]: |
| 193 | """Fetch and parse top comments for a Reddit post via the shreddit endpoint. |
| 194 | |
| 195 | Args: |
| 196 | post_url: Reddit thread URL (…/r/{sub}/comments/{id}/…) |
| 197 | timeout: HTTP timeout in seconds |
| 198 | |
| 199 | Returns: |
| 200 | Dict with 'top_comments' (list, reddit_enrich shape), 'comment_insights' |
| 201 | (list[str]), and 'num_comments' (int or None). Empty/None on any |
| 202 | failure — never raises, so the caller can fall through to SC backup. |
| 203 | """ |
| 204 | ref = extract_post_ref(post_url) |
| 205 | if not ref: |
| 206 | return {"top_comments": [], "comment_insights": [], "num_comments": None} |
| 207 | sub, post_id = ref |
| 208 | |
| 209 | html_text = http.reddit_keyless_get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html") |
| 210 | if not html_text: |
| 211 | return {"top_comments": [], "comment_insights": [], "num_comments": None} |
| 212 | |
| 213 | comments = parse_comments(html_text, limit=MAX_COMMENTS) |
| 214 | insights = reddit_enrich.extract_comment_insights(comments) |
| 215 | return { |
| 216 | "top_comments": [ |
| 217 | { |
| 218 | "score": c["score"], |
| 219 | "date": c["date"], |
| 220 | "author": c["author"], |
| 221 | "excerpt": c["excerpt"], |
| 222 | "url": c["url"], |
| 223 | } |
| 224 | for c in comments |
| 225 | ], |
| 226 | "comment_insights": insights, |
| 227 | "num_comments": _total_comments(html_text), |
| 228 | } |
| 229 |