| 1 | """Keyless web search (floor tier for engine-side general web). |
| 2 | |
| 3 | Returns ranked web results for a query with no API key. This is strictly the |
| 4 | FLOOR of the search-source ladder: |
| 5 | |
| 6 | host-native search > paid engine backend > keyless engine search |
| 7 | |
| 8 | It must never run on a host that has native search (the model does it better |
| 9 | there) or preempt a configured paid backend. The pipeline/grounding layer owns |
| 10 | that gating; this module just performs the search when asked. |
| 11 | |
| 12 | Three vendor-neutral rungs, all stdlib-only via :mod:`http`: |
| 13 | 1. DuckDuckGo HTML endpoint (no key, no instance to maintain). |
| 14 | 2. Startpage HTML results, tried when DuckDuckGo yields nothing — notably |
| 15 | when DuckDuckGo anomaly-blocks a datacenter IP with a 202 challenge page. |
| 16 | 3. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``), |
| 17 | tried when the HTML rungs yield nothing. |
| 18 | |
| 19 | Never raises. Returns results in the same dict shape as the paid backends in |
| 20 | :mod:`grounding` so they flow through normalize/score/dedupe unchanged. On total |
| 21 | failure returns ``([], artifact)`` with a degraded reason in the artifact, so the |
| 22 | source-health layer can report it. |
| 23 | """ |
| 24 | |
| 25 | from __future__ import annotations |
| 26 | |
| 27 | import html |
| 28 | import re |
| 29 | from urllib.parse import parse_qs, urlencode, urlparse |
| 30 | |
| 31 | from . import http |
| 32 | |
| 33 | KEYLESS_BACKEND = "keyless" |
| 34 | |
| 35 | _DDG_HTML_URL = "https://html.duckduckgo.com/html/" |
| 36 | |
| 37 | # Floor-tier relevance: below the paid backends' 0.8 so fusion prefers paid/native |
| 38 | # results when both are present. |
| 39 | _KEYLESS_RELEVANCE = 0.6 |
| 40 | |
| 41 | _TAG_RE = re.compile(r"<[^>]+>") |
| 42 | # Strip <style>/<script> blocks *including their contents* before dropping tags, |
| 43 | # so inline CSS/JS text (e.g. Startpage's emotion styles) never leaks into a |
| 44 | # title or snippet. |
| 45 | _STYLE_SCRIPT_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL) |
| 46 | _RESULT_A_RE = re.compile( |
| 47 | r'class="result__a"[^>]*href="(?P<href>[^"]+)"[^>]*>(?P<title>.*?)</a>', |
| 48 | re.IGNORECASE | re.DOTALL, |
| 49 | ) |
| 50 | _SNIPPET_RE = re.compile( |
| 51 | r'class="result__snippet"[^>]*>(?P<snippet>.*?)</a>', |
| 52 | re.IGNORECASE | re.DOTALL, |
| 53 | ) |
| 54 | |
| 55 | _STARTPAGE_HTML_URL = "https://www.startpage.com/sp/search" |
| 56 | # Startpage marks each organic hit with an <a class="result-title result-link …" |
| 57 | # href="<target>">…<h2 …>title</h2></a>, and the description in a following |
| 58 | # <p class="…description…">. Class names carry hashed emotion suffixes, so match |
| 59 | # on the stable "result-title" / "description" substrings. |
| 60 | _SP_RESULT_RE = re.compile( |
| 61 | r'<a\b[^>]*class="[^"]*result-title[^"]*"[^>]*href="(?P<href>https?://[^"]+)"[^>]*>(?P<inner>.*?)</a>', |
| 62 | re.IGNORECASE | re.DOTALL, |
| 63 | ) |
| 64 | _SP_H2_RE = re.compile(r"<h2\b[^>]*>(?P<title>.*?)</h2>", re.IGNORECASE | re.DOTALL) |
| 65 | _SP_DESC_RE = re.compile( |
| 66 | r'<p\b[^>]*class="[^"]*description[^"]*"[^>]*>(?P<snippet>.*?)</p>', |
| 67 | re.IGNORECASE | re.DOTALL, |
| 68 | ) |
| 69 | |
| 70 | |
| 71 | def _domain(url: str) -> str: |
| 72 | # Normalize identically to grounding._domain (strip + lowercase) so keyless |
| 73 | # and paid results dedupe/group consistently by source_domain. |
| 74 | try: |
| 75 | return urlparse(url).netloc.strip().lower() |
| 76 | except (ValueError, AttributeError): |
| 77 | return "" |
| 78 | |
| 79 | |
| 80 | def _strip_html(fragment: str) -> str: |
| 81 | without_blocks = _STYLE_SCRIPT_RE.sub("", fragment or "") |
| 82 | return html.unescape(_TAG_RE.sub("", without_blocks)).strip() |
| 83 | |
| 84 | |
| 85 | def _unwrap_ddg_redirect(href: str) -> str: |
| 86 | """DuckDuckGo wraps result links as //duckduckgo.com/l/?uddg=<encoded>.""" |
| 87 | if "uddg=" not in href: |
| 88 | return href if href.startswith("http") else f"https:{href}" if href.startswith("//") else href |
| 89 | try: |
| 90 | query = urlparse(href if href.startswith("http") else f"https:{href}").query |
| 91 | target = parse_qs(query).get("uddg", [""])[0] |
| 92 | return target or href |
| 93 | except (ValueError, AttributeError): |
| 94 | return href |
| 95 | |
| 96 | |
| 97 | def keyless_search( |
| 98 | query: str, |
| 99 | date_range: tuple[str, str], |
| 100 | config: dict, |
| 101 | count: int = 5, |
| 102 | ) -> tuple[list[dict], dict]: |
| 103 | """Run keyless web search; returns (items, artifact). Never raises.""" |
| 104 | items = _search_ddg(query, count) |
| 105 | used = "ddg" |
| 106 | if not items: |
| 107 | # DuckDuckGo anomaly-blocks datacenter IPs (202 challenge page); fall |
| 108 | # back to Startpage, which still serves organic results there. |
| 109 | items = _search_startpage(query, count) |
| 110 | used = "startpage" |
| 111 | if not items: |
| 112 | searxng_url = (config.get("LAST30DAYS_SEARXNG_URL") or "").strip() |
| 113 | if searxng_url: |
| 114 | items = _search_searxng(query, count, searxng_url) |
| 115 | used = "searxng" |
| 116 | artifact = { |
| 117 | "label": "keyless", |
| 118 | "webSearchQueries": [query], |
| 119 | "resultCount": len(items), |
| 120 | "keyless_backend": used, |
| 121 | } |
| 122 | if not items: |
| 123 | artifact["reason"] = "keyless-search-unavailable" |
| 124 | return items, artifact |
| 125 | |
| 126 | |
| 127 | def _search_ddg(query: str, count: int) -> list[dict]: |
| 128 | url = f"{_DDG_HTML_URL}?{urlencode({'q': query})}" |
| 129 | text = http.get_text(url, accept="text/html", retries=2) |
| 130 | if not text: |
| 131 | return [] |
| 132 | items: list[dict] = [] |
| 133 | # Associate each result's snippet by position, not by a parallel index: |
| 134 | # some result anchors (video/news modules) have no snippet, so a global |
| 135 | # zip would shift every later snippet onto the wrong result. Take the first |
| 136 | # snippet that falls between this anchor and the next one. |
| 137 | matches = list(_RESULT_A_RE.finditer(text)) |
| 138 | for idx, match in enumerate(matches): |
| 139 | if len(items) >= count: |
| 140 | break |
| 141 | target = _unwrap_ddg_redirect(match.group("href")) |
| 142 | if not target.startswith("http"): |
| 143 | continue |
| 144 | next_start = matches[idx + 1].start() if idx + 1 < len(matches) else len(text) |
| 145 | window = text[match.end():next_start] |
| 146 | snippet_match = _SNIPPET_RE.search(window) |
| 147 | snippet = _strip_html(snippet_match.group("snippet")) if snippet_match else "" |
| 148 | title = _strip_html(match.group("title")) |
| 149 | items.append(_to_item(len(items), title, target, snippet)) |
| 150 | return items |
| 151 | |
| 152 | |
| 153 | def _search_startpage(query: str, count: int) -> list[dict]: |
| 154 | """Keyless rung 2: Startpage's HTML results page. Unlike DuckDuckGo's HTML |
| 155 | endpoint (which anomaly-blocks datacenter IPs with a 202 challenge page), |
| 156 | Startpage returns organic results to a plain browser-UA GET, making it the |
| 157 | working floor on hosts DuckDuckGo refuses. Never raises.""" |
| 158 | url = f"{_STARTPAGE_HTML_URL}?{urlencode({'query': query})}" |
| 159 | text = http.get_text(url, accept="text/html", retries=2) |
| 160 | if not text: |
| 161 | return [] |
| 162 | items: list[dict] = [] |
| 163 | result_matches = list(_SP_RESULT_RE.finditer(text)) |
| 164 | desc_matches = list(_SP_DESC_RE.finditer(text)) |
| 165 | for match in result_matches: |
| 166 | if len(items) >= count: |
| 167 | break |
| 168 | target = html.unescape(match.group("href")) |
| 169 | if not target.startswith("http"): |
| 170 | continue |
| 171 | h2 = _SP_H2_RE.search(match.group("inner")) |
| 172 | title = _strip_html(h2.group("title") if h2 else match.group("inner")) |
| 173 | if not title: |
| 174 | continue |
| 175 | # First description block that appears after this result's title anchor. |
| 176 | snippet = "" |
| 177 | for desc in desc_matches: |
| 178 | if desc.start() > match.end(): |
| 179 | snippet = _strip_html(desc.group("snippet")) |
| 180 | break |
| 181 | items.append(_to_item(len(items), title, target, snippet)) |
| 182 | return items |
| 183 | |
| 184 | |
| 185 | def _search_searxng(query: str, count: int, instance_url: str) -> list[dict]: |
| 186 | base = instance_url.rstrip("/") |
| 187 | url = f"{base}/search?{urlencode({'q': query, 'format': 'json'})}" |
| 188 | try: |
| 189 | data = http.get(url, headers={"Accept": "application/json"}, timeout=15, retries=2) |
| 190 | except http.HTTPError: |
| 191 | return [] |
| 192 | if not isinstance(data, dict): |
| 193 | return [] |
| 194 | items: list[dict] = [] |
| 195 | for i, r in enumerate(data.get("results", [])): |
| 196 | if len(items) >= count: |
| 197 | break |
| 198 | if not isinstance(r, dict): |
| 199 | continue |
| 200 | target = r.get("url", "") |
| 201 | if not target.startswith("http"): |
| 202 | continue |
| 203 | items.append(_to_item(i, r.get("title", ""), target, r.get("content", ""))) |
| 204 | return items |
| 205 | |
| 206 | |
| 207 | def _to_item(index: int, title: str, url: str, snippet: str) -> dict: |
| 208 | return { |
| 209 | "id": f"WK{index + 1}", |
| 210 | "title": title, |
| 211 | "url": url, |
| 212 | "source_domain": _domain(url), |
| 213 | "snippet": (snippet or "")[:500], |
| 214 | "date": None, |
| 215 | "relevance": _KEYLESS_RELEVANCE, |
| 216 | "why_relevant": "Keyless web search", |
| 217 | } |
| 218 |