| 1 | """Arctic-shift score resolver — post upvote counts by id, keyless and free. |
| 2 | |
| 3 | ``search.json`` and ``/comments/{id}.json`` are 403 keyless, and ``search.rss`` |
| 4 | (used for discovery) carries titles but NO score; the shreddit listing partials |
| 5 | score only posts that appear in a pulled listing. For a thread found only via |
| 6 | global RSS search in a broad sub, the free score comes from arctic-shift |
| 7 | (https://arctic-shift.photon-reddit.com), a public Reddit archive whose |
| 8 | ``/api/posts/ids`` returns the post object (score, num_comments, title) for a |
| 9 | batch of base36 post ids. Scores are point-in-time snapshots — slightly stale vs |
| 10 | live, which is fine for ranking and display. |
| 11 | |
| 12 | Best-effort, never raises. On rate-limit (HTTP 422 "slow down"), error, or an |
| 13 | unreachable host it returns ``{}`` so the caller shows the thread without a point |
| 14 | count rather than failing the Reddit source. |
| 15 | """ |
| 16 | |
| 17 | import sys |
| 18 | import time |
| 19 | from typing import Dict, List |
| 20 | |
| 21 | from . import http |
| 22 | |
| 23 | API = "https://arctic-shift.photon-reddit.com/api/posts/ids" |
| 24 | BATCH = 50 # ids per request |
| 25 | TIMEOUT = 15 |
| 26 | MAX_BATCHES = 3 # cap total requests per run (bounds latency + rate-limit risk) |
| 27 | PACE_SECONDS = 0.4 # gap between batches; arctic-shift answers 422 "slow down" |
| 28 | CACHE_MAX = 4096 # hard size bound so the in-run memo can never grow unbounded |
| 29 | # In-run memo: base36 id -> {score, num_comments}. Module-level so repeated |
| 30 | # fetch_scores calls within one `/last30days` run (e.g. across subqueries) reuse |
| 31 | # results, but capped at CACHE_MAX entries (never reached in a normal CLI run). |
| 32 | # Tests clear it via reddit_arctic._cache.clear(). |
| 33 | _cache: Dict[str, Dict[str, int]] = {} |
| 34 | |
| 35 | |
| 36 | def _log(msg: str) -> None: |
| 37 | sys.stderr.write(f"[ArcticShift] {msg}\n") |
| 38 | sys.stderr.flush() |
| 39 | |
| 40 | |
| 41 | def fetch_scores(post_ids: List[str]) -> Dict[str, Dict[str, int]]: |
| 42 | """Return ``{base36_post_id: {"score", "num_comments"}}`` for the given ids. |
| 43 | |
| 44 | Batched, paced, in-run cached, and never raises. Ids that fail or are absent |
| 45 | from the archive are simply missing from the result (caller degrades to no |
| 46 | point count for those threads). |
| 47 | """ |
| 48 | out: Dict[str, Dict[str, int]] = {} |
| 49 | todo: List[str] = [] |
| 50 | for pid in post_ids: |
| 51 | if not pid: |
| 52 | continue |
| 53 | if pid in _cache: |
| 54 | out[pid] = _cache[pid] |
| 55 | elif pid not in todo: |
| 56 | todo.append(pid) |
| 57 | |
| 58 | batches = [todo[i:i + BATCH] for i in range(0, len(todo), BATCH)][:MAX_BATCHES] |
| 59 | for n, batch in enumerate(batches): |
| 60 | if n: |
| 61 | time.sleep(PACE_SECONDS) |
| 62 | try: |
| 63 | data = http.get( |
| 64 | f"{API}?ids={','.join(batch)}", |
| 65 | headers={"User-Agent": http.BROWSER_USER_AGENT}, |
| 66 | timeout=TIMEOUT, |
| 67 | ) |
| 68 | except Exception as e: # network error / non-200 — degrade, never raise |
| 69 | _log(f"lookup failed ({e}); {len(batch)} ids left unscored") |
| 70 | break |
| 71 | rows = (data or {}).get("data") |
| 72 | if not isinstance(rows, list): |
| 73 | # arctic-shift returns {"error": "..."} on rate-limit / bad request. |
| 74 | _log(f"unexpected response (rate-limited?): {str(data)[:80]}") |
| 75 | break |
| 76 | for row in rows: |
| 77 | if not isinstance(row, dict): |
| 78 | continue |
| 79 | rid = str(row.get("id") or "").removeprefix("t3_") |
| 80 | if not rid: |
| 81 | continue |
| 82 | try: |
| 83 | entry = { |
| 84 | "score": int(row.get("score") or 0), |
| 85 | "num_comments": int(row.get("num_comments") or 0), |
| 86 | } |
| 87 | except (TypeError, ValueError): |
| 88 | continue |
| 89 | if len(_cache) < CACHE_MAX: |
| 90 | _cache[rid] = entry |
| 91 | out[rid] = entry |
| 92 | return out |
| 93 |