| 1 | """Arctic-shift score resolver — post upvote counts by id, keyless and free. |
| 2 | |
| 3 | ``search.json`` and ``/comments/{id}.json`` are 403 keyless, and ``search.rss`` |
| 4 | (used for discovery) carries titles but NO score; the shreddit listing partials |
| 5 | score only posts that appear in a pulled listing. For a thread found only via |
| 6 | global RSS search in a broad sub, the free score comes from arctic-shift |
| 7 | (https://arctic-shift.photon-reddit.com), a public Reddit archive whose |
| 8 | ``/api/posts/ids`` returns the post object (score, num_comments, title) for a |
| 9 | batch of base36 post ids. Scores are point-in-time snapshots — slightly stale vs |
| 10 | live, which is fine for ranking and display. |
| 11 | |
| 12 | Best-effort, never raises. On rate-limit (HTTP 422 "slow down"), error, or an |
| 13 | unreachable host it returns ``{}`` so the caller shows the thread without a point |
| 14 | count rather than failing the Reddit source. |
| 15 | """ |
| 16 | |
| 17 | import sys |
| 18 | import time |
| 19 | from datetime import datetime, timezone |
| 20 | from typing import Any, Dict, List, Optional |
| 21 | |
| 22 | from . import http |
| 23 | |
| 24 | API = "https://arctic-shift.photon-reddit.com/api/posts/ids" |
| 25 | SEARCH_API = "https://arctic-shift.photon-reddit.com/api/posts/search" |
| 26 | BATCH = 50 # ids per request |
| 27 | TIMEOUT = 15 |
| 28 | MAX_BATCHES = 3 # cap total requests per run (bounds latency + rate-limit risk) |
| 29 | PACE_SECONDS = 0.4 # gap between batches; arctic-shift answers 422 "slow down" |
| 30 | CACHE_MAX = 4096 # hard size bound so the in-run memo can never grow unbounded |
| 31 | # Listing-lane knobs. Base limits mirror reddit_listing's DEPTH_LIMITS so callers |
| 32 | # get the same per-depth volume. The supplement multiplier is applied when the |
| 33 | # caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes, |
| 34 | # so we fetch more posts to increase the chance of covering what the failed |
| 35 | # shreddit lanes would have returned. |
| 36 | # |
| 37 | # KNOWN LIMITATION: Arctic-shift is recency-only (sort=desc). It has no top/hot/ |
| 38 | # new/rising lanes — failed shreddit sort lanes are supplemented with recent |
| 39 | # posts, not lane-specific results. This is a fundamental backend constraint. |
| 40 | _LISTING_DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50} |
| 41 | _LISTING_SUPPLEMENT_MULTIPLIER = 2 # fetch 2x posts when supplementing multi-sort requests |
| 42 | # Total deadline for listing fetches to prevent unbounded stalls when many |
| 43 | # subreddits are requested and arctic is slow/unreachable. |
| 44 | _LISTING_DEADLINE_SECONDS = 45 # ~3 subs at 15s timeout each |
| 45 | # In-run memo: base36 id -> {score, num_comments}. Module-level so repeated |
| 46 | # fetch_scores calls within one `/last30days` run (e.g. across subqueries) reuse |
| 47 | # results, but capped at CACHE_MAX entries (never reached in a normal CLI run). |
| 48 | # Tests clear it via reddit_arctic._cache.clear(). |
| 49 | _cache: Dict[str, Dict[str, int]] = {} |
| 50 | |
| 51 | |
| 52 | def _log(msg: str) -> None: |
| 53 | sys.stderr.write(f"[ArcticShift] {msg}\n") |
| 54 | sys.stderr.flush() |
| 55 | |
| 56 | |
| 57 | def fetch_scores(post_ids: List[str]) -> Dict[str, Dict[str, int]]: |
| 58 | """Return ``{base36_post_id: {"score", "num_comments"}}`` for the given ids. |
| 59 | |
| 60 | Batched, paced, in-run cached, and never raises. Ids that fail or are absent |
| 61 | from the archive are simply missing from the result (caller degrades to no |
| 62 | point count for those threads). |
| 63 | """ |
| 64 | out: Dict[str, Dict[str, int]] = {} |
| 65 | todo: List[str] = [] |
| 66 | for pid in post_ids: |
| 67 | if not pid: |
| 68 | continue |
| 69 | if pid in _cache: |
| 70 | out[pid] = _cache[pid] |
| 71 | elif pid not in todo: |
| 72 | todo.append(pid) |
| 73 | |
| 74 | batches = [todo[i:i + BATCH] for i in range(0, len(todo), BATCH)][:MAX_BATCHES] |
| 75 | for n, batch in enumerate(batches): |
| 76 | if n: |
| 77 | time.sleep(PACE_SECONDS) |
| 78 | try: |
| 79 | data = http.get( |
| 80 | f"{API}?ids={','.join(batch)}", |
| 81 | headers={"User-Agent": http.BROWSER_USER_AGENT}, |
| 82 | timeout=TIMEOUT, |
| 83 | ) |
| 84 | except Exception as e: # network error / non-200 — degrade, never raise |
| 85 | _log(f"lookup failed ({e}); {len(batch)} ids left unscored") |
| 86 | break |
| 87 | rows = (data or {}).get("data") |
| 88 | if not isinstance(rows, list): |
| 89 | # arctic-shift returns {"error": "..."} on rate-limit / bad request. |
| 90 | _log(f"unexpected response (rate-limited?): {str(data)[:80]}") |
| 91 | break |
| 92 | for row in rows: |
| 93 | if not isinstance(row, dict): |
| 94 | continue |
| 95 | rid = str(row.get("id") or "").removeprefix("t3_") |
| 96 | if not rid: |
| 97 | continue |
| 98 | try: |
| 99 | entry = { |
| 100 | "score": int(row.get("score") or 0), |
| 101 | "num_comments": int(row.get("num_comments") or 0), |
| 102 | } |
| 103 | except (TypeError, ValueError): |
| 104 | continue |
| 105 | if len(_cache) < CACHE_MAX: |
| 106 | _cache[rid] = entry |
| 107 | out[rid] = entry |
| 108 | return out |
| 109 | |
| 110 | |
| 111 | def _epoch_to_date(value: Any) -> Optional[str]: |
| 112 | """Epoch seconds -> YYYY-MM-DD (UTC), or None on garbage.""" |
| 113 | try: |
| 114 | return datetime.fromtimestamp(int(value), tz=timezone.utc).date().isoformat() |
| 115 | except (TypeError, ValueError, OSError): |
| 116 | return None |
| 117 | |
| 118 | |
| 119 | def _normalize_listing_row(row: Dict[str, Any], query: str = "") -> Dict[str, Any]: |
| 120 | """Normalize an arctic-shift post row to reddit_listing.parse_cards shape. |
| 121 | |
| 122 | Mirrors the shreddit card schema (title/url/score/num_comments/subreddit/ |
| 123 | created_utc/author/selftext/date/engagement/relevance/metadata.post_id) so |
| 124 | reddit_keyless can consume either backend interchangeably. |
| 125 | """ |
| 126 | from .relevance import token_overlap_relevance |
| 127 | |
| 128 | pid = str(row.get("id") or "").removeprefix("t3_") |
| 129 | permalink = row.get("permalink") or "" |
| 130 | title = row.get("title") or "" |
| 131 | try: |
| 132 | score = int(row.get("score") or 0) |
| 133 | except (TypeError, ValueError): |
| 134 | score = 0 |
| 135 | try: |
| 136 | num_comments = int(row.get("num_comments") or 0) |
| 137 | except (TypeError, ValueError): |
| 138 | num_comments = 0 |
| 139 | author = row.get("author") or "[deleted]" |
| 140 | if author in ("[deleted]", "[removed]"): |
| 141 | author = "[deleted]" |
| 142 | url = f"https://www.reddit.com{permalink}" if permalink.startswith("/") else (permalink or "") |
| 143 | return { |
| 144 | "id": "", |
| 145 | "title": title, |
| 146 | "url": url, |
| 147 | "score": score, |
| 148 | "num_comments": num_comments, |
| 149 | "subreddit": row.get("subreddit") or "", |
| 150 | "created_utc": row.get("created_utc"), |
| 151 | "author": author, |
| 152 | "selftext": row.get("selftext") or "", |
| 153 | "date": _epoch_to_date(row.get("created_utc")), |
| 154 | "engagement": {"score": score, "num_comments": num_comments, "upvote_ratio": None}, |
| 155 | "relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0, |
| 156 | "why_relevant": "Reddit listing (arctic-shift)", |
| 157 | "metadata": {"post_id": pid}, |
| 158 | } |
| 159 | |
| 160 | |
| 161 | def fetch_listings( |
| 162 | subreddits: List[str], |
| 163 | depth: str = "default", |
| 164 | query: str = "", |
| 165 | sorts: Optional[List[str]] = None, |
| 166 | timeframe: str = "month", |
| 167 | limit: Optional[int] = None, |
| 168 | ) -> List[Dict[str, Any]]: |
| 169 | """Scored subreddit listings from the arctic-shift archive, keyless. |
| 170 | |
| 171 | Drop-in fallback/supplement for ``reddit_listing.fetch_listings`` (shreddit |
| 172 | partials), which datacenter IPs get HTTP 403 on. Arctic-shift serves recent |
| 173 | posts with real score/num_comments from any IP. |
| 174 | |
| 175 | Arctic-shift has no top/hot/new lanes, only recency. When ``sorts`` contains |
| 176 | multiple entries (e.g., dedicated lanes requesting top+hot+new), we fetch |
| 177 | more posts per subreddit to partially compensate for the missing lane |
| 178 | coverage — the caller's engagement ranking does the final sorting. |
| 179 | |
| 180 | Best-effort, never raises: returns ``[]`` on any failure. |
| 181 | """ |
| 182 | if not subreddits: |
| 183 | return [] |
| 184 | base = limit or _LISTING_DEPTH_LIMITS.get(depth, _LISTING_DEPTH_LIMITS["default"]) |
| 185 | # When multiple sorts were requested, fetch more posts to compensate for |
| 186 | # arctic-shift's lack of sort lanes. |
| 187 | n = base * _LISTING_SUPPLEMENT_MULTIPLIER if sorts and len(sorts) > 1 else base |
| 188 | out: List[Dict[str, Any]] = [] |
| 189 | # Process all requested subreddits with pacing and a total deadline to |
| 190 | # prevent unbounded stalls when arctic is slow or unreachable. |
| 191 | deadline = time.time() + _LISTING_DEADLINE_SECONDS |
| 192 | fetched_count = 0 |
| 193 | for sub in subreddits: |
| 194 | if time.time() >= deadline: |
| 195 | _log(f"listing deadline reached after {fetched_count} subs; skipping remaining") |
| 196 | break |
| 197 | sub = sub.removeprefix("r/").strip() |
| 198 | if not sub or sub.lower() == "all": |
| 199 | continue |
| 200 | if fetched_count: |
| 201 | time.sleep(PACE_SECONDS) |
| 202 | fetched_count += 1 |
| 203 | try: |
| 204 | # Use retries=1 (single attempt) so retries don't exceed our deadline. |
| 205 | # The deadline handles overall timing; per-request retries would |
| 206 | # multiply the delay unpredictably. |
| 207 | data = http.get( |
| 208 | f"{SEARCH_API}?subreddit={sub}&limit={n}&sort=desc", |
| 209 | headers={"User-Agent": http.BROWSER_USER_AGENT}, |
| 210 | timeout=TIMEOUT, |
| 211 | retries=1, |
| 212 | ) |
| 213 | except Exception as e: # network error / non-200 — degrade, never raise |
| 214 | _log(f"listing search failed r/{sub}: {e}") |
| 215 | continue |
| 216 | rows = (data or {}).get("data") |
| 217 | if not isinstance(rows, list): |
| 218 | _log(f"unexpected listing response for r/{sub}: {str(data)[:80]}") |
| 219 | continue |
| 220 | for row in rows: |
| 221 | if not isinstance(row, dict): |
| 222 | continue |
| 223 | post = _normalize_listing_row(row, query) |
| 224 | if post["url"]: |
| 225 | out.append(post) |
| 226 | |
| 227 | seen: set = set() |
| 228 | unique: List[Dict[str, Any]] = [] |
| 229 | for p in out: |
| 230 | if p["url"] not in seen: |
| 231 | seen.add(p["url"]) |
| 232 | unique.append(p) |
| 233 | return unique |
| 234 |