| 1 | """arXiv research-paper source for last30days. |
| 2 | |
| 3 | Shells out to ``arxiv-pp-cli`` (open Atom API, no auth) to surface recent |
| 4 | research papers relevant to a topic. arXiv carries no engagement signal, so |
| 5 | ranking leans on relevance (the CLI's own relevance sort plus token overlap) |
| 6 | and recency. |
| 7 | |
| 8 | Activation gate: this source is only available when ``arxiv-pp-cli`` is on |
| 9 | PATH. ``pipeline.available_sources`` checks ``shutil.which`` before including |
| 10 | ``arxiv``. The functions below also detect the missing-binary case defensively. |
| 11 | |
| 12 | Default-on safety (two gates, both required): |
| 13 | 1. Query construction. arXiv is queried with a *quoted* phrase and |
| 14 | ``--sort-by relevance``. Sorting by submitted-date instead returns the |
| 15 | newest cs.* papers regardless of topic -- topic-blind noise. |
| 16 | 2. Recency cutoff. Entries older than ``RECENCY_DAYS`` are dropped. Research |
| 17 | does not trend on a 30-day clock, so this window is wider than the social |
| 18 | sources' 30 days; it keeps arXiv current while dropping stale keyword |
| 19 | matches (e.g. a 2017 sports-statistics paper that an off-topic query like |
| 20 | "Golden State Warriors" would otherwise surface). |
| 21 | """ |
| 22 | |
| 23 | from __future__ import annotations |
| 24 | |
| 25 | import json |
| 26 | import shutil |
| 27 | from datetime import datetime, timezone |
| 28 | from typing import Any, Dict, List, Optional |
| 29 | |
| 30 | from . import log, subproc |
| 31 | from .relevance import token_overlap_relevance |
| 32 | |
| 33 | |
| 34 | CLI_BIN = "arxiv-pp-cli" |
| 35 | |
| 36 | # Per-depth result counts. |
| 37 | DEPTH_CONFIG = { |
| 38 | "quick": 5, |
| 39 | "default": 10, |
| 40 | "deep": 20, |
| 41 | } |
| 42 | |
| 43 | # Recency window for arXiv specifically. Papers do not trend daily; a year keeps |
| 44 | # the source current (the off-topic 2017 paper still drops) without discarding |
| 45 | # the genuinely-relevant work from the last few months. |
| 46 | RECENCY_DAYS = 365 |
| 47 | |
| 48 | SEARCH_TIMEOUT = 30 |
| 49 | |
| 50 | |
| 51 | def _log(msg: str) -> None: |
| 52 | log.source_log("arXiv", msg, tty_only=False) |
| 53 | |
| 54 | |
| 55 | def _is_available() -> bool: |
| 56 | """True when the arxiv-pp-cli binary is on PATH.""" |
| 57 | return shutil.which(CLI_BIN) is not None |
| 58 | |
| 59 | |
| 60 | def _today() -> datetime: |
| 61 | return datetime.now(timezone.utc) |
| 62 | |
| 63 | |
| 64 | def _build_search_query(topic: str, *, quoted: bool = True) -> str: |
| 65 | """Build the arXiv search-query string for ``topic``. |
| 66 | |
| 67 | Quoted (default): phrase-scoped exact match across all fields. Precise |
| 68 | for topics that genuinely appear as a phrase in a title/abstract, but a |
| 69 | natural-language multi-word topic ("AI video generation advances") almost |
| 70 | never appears verbatim, so it returns zero results (#908). Unquoted uses |
| 71 | an AND-conjoined clause for every individual term as a fallback retry. |
| 72 | |
| 73 | Inner double-quotes are stripped (arXiv has no phrase-escaping) either way. |
| 74 | """ |
| 75 | phrase = _clean_phrase(topic) |
| 76 | if quoted: |
| 77 | return f'all:"{phrase}"' |
| 78 | return " AND ".join(f'all:"{term}"' for term in phrase.split()) |
| 79 | |
| 80 | |
| 81 | def _clean_phrase(topic: str) -> str: |
| 82 | """Strip quotes and collapse whitespace into a phrase for the query.""" |
| 83 | return " ".join(topic.replace('"', " ").split()) |
| 84 | |
| 85 | |
| 86 | def _build_search_args(topic: str, limit: int, *, quoted: bool = True) -> List[str]: |
| 87 | return [ |
| 88 | CLI_BIN, |
| 89 | "query", |
| 90 | "--search-query", |
| 91 | _build_search_query(topic, quoted=quoted), |
| 92 | "--sort-by", |
| 93 | "relevance", |
| 94 | "--max-results", |
| 95 | str(limit), |
| 96 | "--agent", |
| 97 | ] |
| 98 | |
| 99 | |
| 100 | def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]: |
| 101 | """Invoke arxiv-pp-cli and parse the JSON envelope. |
| 102 | |
| 103 | arXiv returns ``{"meta": ..., "results": {"entries": [...]}}``. This |
| 104 | normalizes to ``{"results": [...entries...]}`` so the parse step sees a |
| 105 | flat list, matching the other sources' shape. Never raises. |
| 106 | """ |
| 107 | if not _is_available(): |
| 108 | return {"results": [], "error": f"{CLI_BIN} not on PATH"} |
| 109 | try: |
| 110 | result = subproc.run_with_timeout(cmd, timeout=timeout) |
| 111 | except subproc.SubprocTimeout as exc: |
| 112 | _log(f"Timeout: {exc}") |
| 113 | return {"results": [], "error": str(exc)} |
| 114 | except FileNotFoundError as exc: |
| 115 | _log(f"Binary missing: {exc}") |
| 116 | return {"results": [], "error": str(exc)} |
| 117 | except OSError as exc: |
| 118 | _log(f"Spawn failed: {exc}") |
| 119 | return {"results": [], "error": str(exc)} |
| 120 | |
| 121 | if result.returncode != 0: |
| 122 | snippet = (result.stderr or "").strip().splitlines()[:1] |
| 123 | first = snippet[0] if snippet else f"exit {result.returncode}" |
| 124 | _log(f"CLI exit {result.returncode}: {first}") |
| 125 | return {"results": [], "error": first} |
| 126 | |
| 127 | stdout = result.stdout or "" |
| 128 | if not stdout.strip(): |
| 129 | _log("CLI returned empty stdout") |
| 130 | return {"results": [], "error": "empty stdout"} |
| 131 | try: |
| 132 | data = json.loads(stdout) |
| 133 | except json.JSONDecodeError as exc: |
| 134 | _log(f"JSON decode failed: {exc}") |
| 135 | return {"results": [], "error": f"json decode: {exc}"} |
| 136 | |
| 137 | if not _is_entry_envelope(data): |
| 138 | _log("CLI returned an unrecognized JSON response") |
| 139 | return {"results": [], "error": "unrecognized JSON response"} |
| 140 | |
| 141 | return {"results": _extract_entries(data)} |
| 142 | |
| 143 | |
| 144 | def _extract_entries(data: Any) -> List[Dict[str, Any]]: |
| 145 | """Pull the entries list out of arXiv's nested envelope. |
| 146 | |
| 147 | Tolerates ``{"results": {"entries": [...]}}`` (current shape), |
| 148 | ``{"entries": [...]}``, and a bare list. |
| 149 | """ |
| 150 | if isinstance(data, list): |
| 151 | return [e for e in data if isinstance(e, dict)] |
| 152 | if isinstance(data, dict): |
| 153 | results = data.get("results") |
| 154 | if isinstance(results, dict): |
| 155 | entries = results.get("entries") |
| 156 | if isinstance(entries, list): |
| 157 | return [e for e in entries if isinstance(e, dict)] |
| 158 | if isinstance(results, list): |
| 159 | return [e for e in results if isinstance(e, dict)] |
| 160 | entries = data.get("entries") |
| 161 | if isinstance(entries, list): |
| 162 | return [e for e in entries if isinstance(e, dict)] |
| 163 | return [] |
| 164 | |
| 165 | |
| 166 | def _is_entry_envelope(data: Any) -> bool: |
| 167 | """Return whether ``data`` has one of the supported entry-list shapes.""" |
| 168 | if isinstance(data, list): |
| 169 | return True |
| 170 | if not isinstance(data, dict): |
| 171 | return False |
| 172 | results = data.get("results") |
| 173 | return ( |
| 174 | isinstance(results, list) |
| 175 | or (isinstance(results, dict) and isinstance(results.get("entries"), list)) |
| 176 | or isinstance(data.get("entries"), list) |
| 177 | ) |
| 178 | |
| 179 | |
| 180 | def search_arxiv( |
| 181 | topic: str, |
| 182 | from_date: str, |
| 183 | to_date: str, |
| 184 | depth: str = "default", |
| 185 | ) -> Dict[str, Any]: |
| 186 | """Search arXiv via arxiv-pp-cli using a quoted, relevance-sorted query. |
| 187 | |
| 188 | Returns a dict with a flat ``results`` list of entry dicts. On failure, |
| 189 | ``results`` is empty and an ``error`` key carries a one-line description. |
| 190 | """ |
| 191 | if not topic or not topic.strip(): |
| 192 | return {"results": []} |
| 193 | # A topic of only quote characters cleans to an empty phrase (all:""), |
| 194 | # which is a topic-blind query; bail rather than search for nothing. |
| 195 | if not _clean_phrase(topic): |
| 196 | return {"results": []} |
| 197 | limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 198 | cmd = _build_search_args(topic, limit) |
| 199 | _log(f"query '{topic}' (relevance, max={limit})") |
| 200 | response = _run_cli(cmd, timeout=SEARCH_TIMEOUT) |
| 201 | _log(f"found {len(response.get('results') or [])} entries") |
| 202 | # Retry a clean zero-result phrase match with individually quoted AND terms. |
| 203 | # CLI failures, malformed responses, and missing binaries skip the retry. |
| 204 | if not response.get("error") and not response.get("results"): |
| 205 | retry_cmd = _build_search_args(topic, limit, quoted=False) |
| 206 | _log(f"quoted phrase matched nothing; retrying unquoted for '{topic}'") |
| 207 | response = _run_cli(retry_cmd, timeout=SEARCH_TIMEOUT) |
| 208 | _log(f"unquoted retry found {len(response.get('results') or [])} entries") |
| 209 | return response |
| 210 | |
| 211 | |
| 212 | def _parse_published(published: Optional[str]) -> Optional[datetime]: |
| 213 | """Parse an arXiv ``published`` timestamp (ISO 8601, e.g. |
| 214 | '2026-06-25T17:59:48Z') into an aware datetime. Returns None on failure.""" |
| 215 | if not published or not isinstance(published, str): |
| 216 | return None |
| 217 | text = published.strip().replace("Z", "+00:00") |
| 218 | try: |
| 219 | dt = datetime.fromisoformat(text) |
| 220 | except ValueError: |
| 221 | return None |
| 222 | if dt.tzinfo is None: |
| 223 | dt = dt.replace(tzinfo=timezone.utc) |
| 224 | return dt |
| 225 | |
| 226 | |
| 227 | def _alternate_url(entry: Dict[str, Any]) -> str: |
| 228 | """Return the human-facing abstract URL (rel=alternate), not the PDF.""" |
| 229 | links = entry.get("links") |
| 230 | if isinstance(links, list): |
| 231 | for link in links: |
| 232 | if isinstance(link, dict) and link.get("rel") == "alternate": |
| 233 | href = str(link.get("href") or "").strip() |
| 234 | if href: |
| 235 | return href |
| 236 | # Fall back to the abstract URL derived from the entry id. |
| 237 | entry_id = str(entry.get("id") or "").strip() |
| 238 | if entry_id.startswith("http"): |
| 239 | return entry_id |
| 240 | return "" |
| 241 | |
| 242 | |
| 243 | def _author_names(entry: Dict[str, Any]) -> List[str]: |
| 244 | authors = entry.get("authors") |
| 245 | out: List[str] = [] |
| 246 | if isinstance(authors, list): |
| 247 | for a in authors: |
| 248 | if isinstance(a, dict): |
| 249 | name = str(a.get("name") or "").strip() |
| 250 | if name: |
| 251 | out.append(name) |
| 252 | return out |
| 253 | |
| 254 | |
| 255 | def parse_arxiv_response( |
| 256 | response: Dict[str, Any], |
| 257 | query: str = "", |
| 258 | today: Optional[datetime] = None, |
| 259 | ) -> List[Dict[str, Any]]: |
| 260 | """Parse an arXiv envelope into normalized item dicts. |
| 261 | |
| 262 | Applies the recency cutoff (drops entries older than ``RECENCY_DAYS`` and |
| 263 | entries with an unparseable date) and computes a token-overlap relevance |
| 264 | hint. Returns dicts ready for ``normalize._normalize_arxiv``. |
| 265 | """ |
| 266 | raw = response.get("results") if isinstance(response, dict) else None |
| 267 | if not isinstance(raw, list): |
| 268 | return [] |
| 269 | |
| 270 | now = today or _today() |
| 271 | items: List[Dict[str, Any]] = [] |
| 272 | for i, entry in enumerate(raw): |
| 273 | if not isinstance(entry, dict): |
| 274 | continue |
| 275 | title = " ".join(str(entry.get("title") or "").split()).strip() |
| 276 | if not title: |
| 277 | continue |
| 278 | published = _parse_published(entry.get("published") or entry.get("updated")) |
| 279 | if published is None: |
| 280 | # No usable date -> cannot honor the recency contract; drop. |
| 281 | continue |
| 282 | age_days = (now - published).days |
| 283 | # Allow a one-day grace on the future side: a paper announced later in |
| 284 | # the same UTC day yields age_days == -1 (timedelta.days floors toward |
| 285 | # negative); dropping it as "future" would discard the freshest work. |
| 286 | if age_days > RECENCY_DAYS or age_days < -1: |
| 287 | continue |
| 288 | |
| 289 | summary = " ".join(str(entry.get("summary") or "").split()).strip() |
| 290 | authors = _author_names(entry) |
| 291 | url = _alternate_url(entry) |
| 292 | |
| 293 | rank_decay = max(0.3, 1.0 - (i * 0.03)) |
| 294 | if query: |
| 295 | content_score = token_overlap_relevance(query, f"{title} {summary}".strip()) |
| 296 | else: |
| 297 | content_score = 0.5 |
| 298 | relevance = min(1.0, 0.6 * rank_decay + 0.4 * content_score) |
| 299 | |
| 300 | primary_author = authors[0] if authors else "" |
| 301 | author_label = primary_author |
| 302 | if len(authors) > 1: |
| 303 | author_label = f"{primary_author} et al." |
| 304 | |
| 305 | items.append( |
| 306 | { |
| 307 | "id": str(entry.get("id") or url or f"AX{i + 1}"), |
| 308 | "title": title, |
| 309 | "url": url, |
| 310 | "summary": summary, |
| 311 | "author": author_label, |
| 312 | "authors": authors, |
| 313 | "date": published.date().isoformat(), |
| 314 | "engagement": {}, |
| 315 | "relevance": round(relevance, 2), |
| 316 | "why_relevant": ( |
| 317 | f"arXiv paper ({primary_author}, {published.date().isoformat()})" |
| 318 | if primary_author |
| 319 | else f"arXiv paper ({published.date().isoformat()})" |
| 320 | ), |
| 321 | } |
| 322 | ) |
| 323 | |
| 324 | return items |
| 325 |