| 1 | """Keyless URL-to-markdown fetch (floor tier for engine-side page reads). |
| 2 | |
| 3 | Turns any URL into clean, JS-rendered markdown via Jina Reader's free hosted |
| 4 | endpoint (``https://r.jina.ai/{url}``) with no API key. This is a *fallback* |
| 5 | tier, never the primary firehose: |
| 6 | |
| 7 | - On agent hosts with a native fetch tool, prefer that (this module exists for |
| 8 | headless/cron and hosts without one). |
| 9 | - The free tier is rate-limited and returns cached snapshots (staleness), so |
| 10 | callers should treat results as best-effort and record when it was used. |
| 11 | - The target URL is sent to a third party; only use for public-research fetches. |
| 12 | |
| 13 | Never raises. Returns a typed :class:`KeylessFetchResult` carrying the failure |
| 14 | reason on any error, so tiered callers (and the source-health layer) can fall |
| 15 | through or report degradation instead of seeing a bare empty string. |
| 16 | """ |
| 17 | |
| 18 | from __future__ import annotations |
| 19 | |
| 20 | from dataclasses import dataclass |
| 21 | from typing import Optional |
| 22 | from urllib.parse import urlparse |
| 23 | |
| 24 | from . import http |
| 25 | |
| 26 | JINA_READER_PREFIX = "https://r.jina.ai/" |
| 27 | |
| 28 | # Conservative timeout: Jina renders the page server-side, so it is slower than |
| 29 | # a plain GET, but we are the floor tier and must not stall the pipeline. |
| 30 | DEFAULT_FETCH_TIMEOUT = 30 |
| 31 | |
| 32 | |
| 33 | @dataclass |
| 34 | class KeylessFetchResult: |
| 35 | """Result of a keyless page fetch. |
| 36 | |
| 37 | ``ok`` is True only when markdown was retrieved. On failure, ``markdown`` is |
| 38 | empty and ``reason`` explains why (consumed by the source-health layer). |
| 39 | """ |
| 40 | |
| 41 | url: str |
| 42 | ok: bool |
| 43 | markdown: str = "" |
| 44 | reason: str = "" |
| 45 | cached_snapshot: bool = False |
| 46 | |
| 47 | |
| 48 | def _looks_like_http_url(url: str) -> bool: |
| 49 | try: |
| 50 | parsed = urlparse(url) |
| 51 | except (ValueError, AttributeError): |
| 52 | return False |
| 53 | return parsed.scheme in ("http", "https") and bool(parsed.netloc) |
| 54 | |
| 55 | |
| 56 | def _detect_cached_snapshot(markdown: str) -> bool: |
| 57 | """Best-effort detection of Jina's cached-snapshot warning. |
| 58 | |
| 59 | Jina prepends a small metadata/warning header to the text response; when it |
| 60 | serves a cached copy it says so. We scan only the leading window to avoid |
| 61 | false positives from article bodies that happen to mention caching. |
| 62 | """ |
| 63 | head = markdown[:600].lower() |
| 64 | return "cached" in head and "snapshot" in head |
| 65 | |
| 66 | |
| 67 | def fetch_markdown( |
| 68 | url: str, |
| 69 | timeout: int = DEFAULT_FETCH_TIMEOUT, |
| 70 | retries: int = 2, |
| 71 | ) -> KeylessFetchResult: |
| 72 | """Fetch ``url`` as clean markdown via the keyless reader endpoint. |
| 73 | |
| 74 | Args: |
| 75 | url: The http(s) URL to fetch. |
| 76 | timeout: Per-attempt HTTP timeout in seconds. |
| 77 | retries: Retry budget (kept low; this is a fail-fast floor tier). |
| 78 | |
| 79 | Returns: |
| 80 | A :class:`KeylessFetchResult`. ``ok`` is False with a populated |
| 81 | ``reason`` on invalid input, network failure, or empty body. |
| 82 | """ |
| 83 | if not url or not _looks_like_http_url(url): |
| 84 | return KeylessFetchResult(url=url or "", ok=False, reason="invalid-url") |
| 85 | |
| 86 | reader_url = f"{JINA_READER_PREFIX}{url}" |
| 87 | text = http.get_text( |
| 88 | reader_url, |
| 89 | timeout=timeout, |
| 90 | retries=retries, |
| 91 | accept="text/plain", |
| 92 | ) |
| 93 | |
| 94 | if text is None: |
| 95 | return KeylessFetchResult(url=url, ok=False, reason="fetch-failed") |
| 96 | |
| 97 | if not text.strip(): |
| 98 | return KeylessFetchResult(url=url, ok=False, reason="empty-body") |
| 99 | |
| 100 | return KeylessFetchResult( |
| 101 | url=url, |
| 102 | ok=True, |
| 103 | markdown=text, |
| 104 | cached_snapshot=_detect_cached_snapshot(text), |
| 105 | ) |
| 106 |