| 1 | """Backend-chain descriptors with predicted selection (doctor, R4). |
| 2 | |
| 3 | Chained sources declare their routing here ONCE — imported from the |
| 4 | definitions ``lib/env.py`` already owns (chain order, pin var names) — and |
| 5 | ``resolve()`` turns side-effect-free probes into a truthful prediction of |
| 6 | what the next run will do. |
| 7 | |
| 8 | Two resolution modes: |
| 9 | |
| 10 | - ``alternative`` (X, YouTube, web search): the pipeline tries genuinely |
| 11 | interchangeable backends in a declared order. Resolution probes ALL |
| 12 | candidates first, then picks (collect-then-pick): the first fully-usable |
| 13 | backend wins the "will use" prediction; otherwise the best degraded |
| 14 | candidate resolves with a warn tier; otherwise the source is an error |
| 15 | carrying the highest-priority backend's prescription. Collecting before |
| 16 | picking prevents an installed-but-unauthenticated preferred backend from |
| 17 | shadowing a fully working fallback. |
| 18 | |
| 19 | - ``conditional`` (Reddit): routing is per-query and outcome-dependent — |
| 20 | public keyless composite by default, ScrapeCreators backfill only when |
| 21 | results fall below the configured thinness floor (see the gating in |
| 22 | ``lib/pipeline.py``). No probe can pick one winner, so resolution renders |
| 23 | honest conditional wording instead of an ``active_backend``. Reddit's |
| 24 | internal keyless lanes (rss/listing/arctic/shreddit) are sub-probe detail |
| 25 | inside the public composite, never chain entries. |
| 26 | |
| 27 | ``active_backend`` semantics: a PREDICTION — "the first backend the probes |
| 28 | say the next run will try" — rendered as "will use". It is not an |
| 29 | observation of what served a past run, and runtime failover can still |
| 30 | diverge mid-run (a present-but-expired paid key passes a presence probe). |
| 31 | |
| 32 | Paid lanes (xai, xapi, xquik, serper, and every other API-key backend, |
| 33 | including ScrapeCreators) probe KEY PRESENCE ONLY: a dict lookup, never a |
| 34 | network call or credential spend. Binary-backed lanes reuse the U1 dependency |
| 35 | probe layer (``health.probe_dependency``) so a stale shim reads as BROKEN, |
| 36 | not available (#692). |
| 37 | |
| 38 | This module observes and predicts only. It must never alter which backend |
| 39 | the pipeline actually uses; parity with the pipeline's pre-failover |
| 40 | selection is asserted in ``tests/test_backend_descriptors.py``. The X chain |
| 41 | is shaped by ``env.x_policy``: on an official-only host the findings and |
| 42 | the chain carry only the policy's backends (plus a pinned one), so doctor |
| 43 | JSON never names a non-official backend there unless it is pinned. |
| 44 | """ |
| 45 | |
| 46 | from __future__ import annotations |
| 47 | |
| 48 | from dataclasses import dataclass |
| 49 | from shutil import which |
| 50 | from typing import Any, Callable, Dict, List, Optional, Tuple |
| 51 | |
| 52 | from . import env, health, prescriptions |
| 53 | |
| 54 | # Resolution modes. |
| 55 | MODE_ALTERNATIVE = "alternative" # probe-ordered chain, first-usable wins |
| 56 | MODE_CONDITIONAL = "conditional" # per-query routing; wording, never a winner |
| 57 | |
| 58 | # Rollup tiers for a resolved chain (doctor maps these into its R1 table). |
| 59 | TIER_OK = "ok" |
| 60 | TIER_WARN = "warn" |
| 61 | TIER_ERROR = "error" |
| 62 | |
| 63 | # Web search backend order. grounding.web_search's auto branch owns the |
| 64 | # runtime behavior (brave -> exa -> serper -> parallel -> keyless floor); |
| 65 | # there is no importable constant there, so this declaration is guarded by |
| 66 | # the grounding-auto parity test rather than an import. |
| 67 | WEB_BACKEND_ORDER: Tuple[str, ...] = ("brave", "exa", "serper", "parallel", "keyless") |
| 68 | |
| 69 | # YouTube backend order (pipeline: yt-dlp first, ScrapeCreators search |
| 70 | # fallback when yt-dlp is absent or fails — see lib/pipeline.py). |
| 71 | YOUTUBE_BACKEND_ORDER: Tuple[str, ...] = ("yt-dlp", "scrapecreators") |
| 72 | |
| 73 | # Chain-failure fixes embed the registry's CLI forms (KTD 7): the command a |
| 74 | # backend finding prescribes and the one doctor/quality-nudge render for the |
| 75 | # same failure mode come from one entry and cannot drift. |
| 76 | _SC_PRESCRIPTION = ( |
| 77 | "set SCRAPECREATORS_API_KEY (free 10,000-call signup: " |
| 78 | f"{prescriptions.get('scrapecreators', 'key_missing').fix_cli})" |
| 79 | ) |
| 80 | |
| 81 | |
| 82 | def _x_cookies_prescription(config: Dict[str, Any]) -> str: |
| 83 | """Bird's unconfigured fix, routed through the X policy. |
| 84 | |
| 85 | Off an official-only host this is the cookie-consent command; on one |
| 86 | the same lookup yields the official-path entry (connector lane, bearer, |
| 87 | xAI key), so a pinned-but-unconfigured scraper never prescribes a |
| 88 | cookie read there. |
| 89 | """ |
| 90 | entry = prescriptions.for_x(config, "cookies_missing") |
| 91 | if entry.failure == "cookies_missing": |
| 92 | return f"run setup with browser-cookie consent: {entry.fix_cli}" |
| 93 | return f"{entry.fix_nl} (cli: {entry.fix_cli})" |
| 94 | |
| 95 | |
| 96 | @dataclass |
| 97 | class BackendFinding: |
| 98 | """Side-effect-free probe outcome for one backend of a chained source. |
| 99 | |
| 100 | ``status`` uses the ``lib.health`` vocabulary (OK/DEGRADED/MISSING/ |
| 101 | BROKEN/TIMEOUT/ERROR). ``prescription`` is the fix when non-OK. |
| 102 | ``requires`` is the backend's requirement note for report rendering. |
| 103 | """ |
| 104 | |
| 105 | name: str |
| 106 | status: str |
| 107 | detail: str = "" |
| 108 | prescription: str = "" |
| 109 | requires: str = "" |
| 110 | |
| 111 | @property |
| 112 | def usable(self) -> bool: |
| 113 | """Fully or partially usable (OK/DEGRADED) — eligible for selection.""" |
| 114 | return self.status in (health.OK, health.DEGRADED) |
| 115 | |
| 116 | |
| 117 | @dataclass(frozen=True) |
| 118 | class BackendSpec: |
| 119 | """One backend in a chain: name, probe, requirement note, paid flag. |
| 120 | |
| 121 | ``probe`` must be side-effect-free. When ``paid`` is True the probe is |
| 122 | key-presence only: no subprocess, no network, no credential spend. |
| 123 | ``opt_in`` marks backends that are never auto-selected and require an |
| 124 | explicit pin (grok). |
| 125 | """ |
| 126 | |
| 127 | name: str |
| 128 | requires: str |
| 129 | probe: Callable[[Dict[str, Any]], "BackendFinding"] |
| 130 | paid: bool = False |
| 131 | opt_in: bool = False |
| 132 | |
| 133 | |
| 134 | @dataclass(frozen=True) |
| 135 | class ChainDescriptor: |
| 136 | """A chained source's declared routing: backends, mode, and pin knob.""" |
| 137 | |
| 138 | source: str |
| 139 | mode: str |
| 140 | backends: Tuple[BackendSpec, ...] |
| 141 | pin_var: Optional[str] = None # env var pin (X, Reddit) |
| 142 | pin_flag: Optional[str] = None # CLI flag pin (web: --web-backend) |
| 143 | |
| 144 | |
| 145 | @dataclass |
| 146 | class BackendResolution: |
| 147 | """Resolved routing for one chained source. |
| 148 | |
| 149 | ``active_backend`` is the will-use PREDICTION for alternative chains |
| 150 | and always None for conditional mode (Reddit never gets a computed |
| 151 | winner — ``conditional`` carries the honest wording instead). |
| 152 | """ |
| 153 | |
| 154 | source: str |
| 155 | mode: str |
| 156 | chain: List[str] |
| 157 | findings: List[BackendFinding] |
| 158 | active_backend: Optional[str] = None |
| 159 | tier: str = TIER_OK |
| 160 | pinned: bool = False |
| 161 | pin: Optional[str] = None |
| 162 | prescription: str = "" |
| 163 | conditional: str = "" |
| 164 | |
| 165 | @property |
| 166 | def summary(self) -> str: |
| 167 | """One-line rendering: will-use prediction or conditional wording.""" |
| 168 | if self.mode == MODE_CONDITIONAL: |
| 169 | return self.conditional |
| 170 | if self.active_backend is None: |
| 171 | line = f"no usable backend (chain: {' -> '.join(self.chain)})" |
| 172 | if self.prescription: |
| 173 | line += f"; fix: {self.prescription}" |
| 174 | return line |
| 175 | line = f"will use: {self.active_backend}" |
| 176 | if self.pinned: |
| 177 | line += f" (pinned via {self._pin_origin()})" |
| 178 | return line |
| 179 | |
| 180 | def _pin_origin(self) -> str: |
| 181 | d = DESCRIPTORS.get(self.source) |
| 182 | if d is None: |
| 183 | return "pin" |
| 184 | return d.pin_var or d.pin_flag or "pin" |
| 185 | |
| 186 | |
| 187 | # --------------------------------------------------------------------------- |
| 188 | # Probes. All side-effect-free; paid lanes are pure dict lookups. |
| 189 | # --------------------------------------------------------------------------- |
| 190 | |
| 191 | def _key_probe(name: str, key_var: str, requires: str, note: str = "") -> Callable: |
| 192 | """Key-presence probe for a paid API lane. Never touches the network.""" |
| 193 | |
| 194 | def probe(config: Dict[str, Any]) -> BackendFinding: |
| 195 | if config.get(key_var): |
| 196 | return BackendFinding( |
| 197 | name=name, |
| 198 | status=health.OK, |
| 199 | detail=f"{key_var} present", |
| 200 | requires=requires, |
| 201 | ) |
| 202 | prescription = note or f"set {key_var} in ~/.config/last30days/.env" |
| 203 | return BackendFinding( |
| 204 | name=name, |
| 205 | status=health.MISSING, |
| 206 | detail=f"{key_var} not set", |
| 207 | prescription=prescription, |
| 208 | requires=requires, |
| 209 | ) |
| 210 | |
| 211 | return probe |
| 212 | |
| 213 | |
| 214 | def _probe_bird(config: Dict[str, Any]) -> BackendFinding: |
| 215 | """Bird = vendored X GraphQL client (node script) + browser-cookie creds. |
| 216 | |
| 217 | Cookie presence is checked FIRST, mirroring ``env._x_backend_available``'s |
| 218 | gating (``has_bird_creds and is_bird_installed()``): without cookies bird |
| 219 | is unconfigured regardless of node/script state, and the fix is the |
| 220 | cookie-consent flow — a broken node runtime must not turn an unconfigured |
| 221 | backend into an error carrying a node prescription. |
| 222 | """ |
| 223 | from . import bird_x |
| 224 | |
| 225 | requires = "X browser cookies (AUTH_TOKEN/CT0) + node" |
| 226 | if not (config.get("AUTH_TOKEN") and config.get("CT0")): |
| 227 | return BackendFinding( |
| 228 | name="bird", |
| 229 | status=health.MISSING, |
| 230 | detail="X browser cookies (AUTH_TOKEN/CT0) not configured", |
| 231 | prescription=_x_cookies_prescription(config), |
| 232 | requires=requires, |
| 233 | ) |
| 234 | if not bird_x.is_bird_installed(): |
| 235 | # Distinguish a missing/broken node runtime from a missing script. |
| 236 | node = health.probe_dependency("node") |
| 237 | if node.status != health.OK: |
| 238 | return BackendFinding( |
| 239 | name="bird", |
| 240 | status=node.status, |
| 241 | detail=node.detail, |
| 242 | prescription=node.prescription, |
| 243 | requires=requires, |
| 244 | ) |
| 245 | return BackendFinding( |
| 246 | name="bird", |
| 247 | status=health.MISSING, |
| 248 | detail="vendored bird-search client not found", |
| 249 | prescription="reinstall the skill (npx skills add . -g -y) to restore lib/vendor/bird-search", |
| 250 | requires=requires, |
| 251 | ) |
| 252 | node = health.probe_dependency("node") |
| 253 | if node.status != health.OK: |
| 254 | # Resolvable-but-broken node (stale shim) must not read as usable. |
| 255 | return BackendFinding( |
| 256 | name="bird", |
| 257 | status=node.status, |
| 258 | detail=node.detail, |
| 259 | prescription=node.prescription, |
| 260 | requires=requires, |
| 261 | ) |
| 262 | return BackendFinding( |
| 263 | name="bird", |
| 264 | status=health.OK, |
| 265 | detail="browser-cookie auth (AUTH_TOKEN/CT0) configured", |
| 266 | requires=requires, |
| 267 | ) |
| 268 | |
| 269 | |
| 270 | def _probe_grok(config: Dict[str, Any]) -> BackendFinding: |
| 271 | """grok CLI = keyless X. LOCAL-ONLY probe, like _probe_xurl. |
| 272 | |
| 273 | Deliberately does NOT call ``health.probe_dependency``: that helper runs |
| 274 | ``subprocess.run([name, "--version"])``, and the whole-doctor-path test |
| 275 | patches ``subprocess.run`` to raise. |
| 276 | |
| 277 | Consequence to be honest about: a grok binary that resolves on PATH but |
| 278 | will not execute (the stale-shim class) reports OK here and fails only when |
| 279 | a real run shells out. ``grok_x.is_available`` does not close that gap |
| 280 | either -- it is also filesystem-only. ``health.probe_dependency("grok")`` |
| 281 | is the executing probe, and it runs in doctor's CLI-health block rather |
| 282 | than on this no-subprocess path. |
| 283 | """ |
| 284 | from . import grok_x |
| 285 | |
| 286 | requires = "grok CLI installed + signed in (no X credential)" |
| 287 | if which("grok") is None: |
| 288 | off_path = health._off_path_binary("grok") |
| 289 | if off_path is not None: |
| 290 | return BackendFinding( |
| 291 | name="grok", |
| 292 | status=health.MISSING, |
| 293 | requires=requires, |
| 294 | detail=f"grok is installed at {off_path} but that directory is not on this process's PATH", |
| 295 | prescription=f'add {off_path.parent} to PATH (e.g. export PATH="{off_path.parent}:$PATH")', |
| 296 | ) |
| 297 | return BackendFinding( |
| 298 | name="grok", |
| 299 | status=health.MISSING, |
| 300 | requires=requires, |
| 301 | detail="grok CLI not found on PATH", |
| 302 | prescription=( |
| 303 | "install the Grok CLI: curl -fsSL https://x.ai/cli/install.sh | bash, " |
| 304 | "then run `grok login`" |
| 305 | ), |
| 306 | ) |
| 307 | store_status, store_detail, expires_at = grok_x.stored_auth_status() |
| 308 | if store_status == grok_x.AUTH_OK: |
| 309 | return BackendFinding( |
| 310 | name="grok", |
| 311 | status=health.OK, |
| 312 | requires=requires, |
| 313 | detail=f"{store_detail} (not live-verified until a run)", |
| 314 | ) |
| 315 | if store_status == grok_x.AUTH_EXPIRED: |
| 316 | expiry_str = expires_at.isoformat() if expires_at else "unknown" |
| 317 | return BackendFinding( |
| 318 | name="grok", |
| 319 | status=health.DEGRADED, |
| 320 | requires=requires, |
| 321 | detail=( |
| 322 | f"Grok session expired at {expiry_str}; " |
| 323 | "refresh happens at run time (if revoked, run `grok login --device-auth`)" |
| 324 | ), |
| 325 | prescription="grok login --device-auth", |
| 326 | ) |
| 327 | if store_status == grok_x.AUTH_ERROR: |
| 328 | return BackendFinding( |
| 329 | name="grok", |
| 330 | status=health.ERROR, |
| 331 | requires=requires, |
| 332 | detail=store_detail, |
| 333 | prescription="grok login", |
| 334 | ) |
| 335 | return BackendFinding( |
| 336 | name="grok", |
| 337 | status=health.MISSING, |
| 338 | requires=requires, |
| 339 | detail="grok CLI installed but not signed in", |
| 340 | prescription="grok login", |
| 341 | ) |
| 342 | |
| 343 | |
| 344 | def _probe_xapi(config: Dict[str, Any]) -> BackendFinding: |
| 345 | """xapi = direct X API v2 with an app-only bearer. KEY PRESENCE ONLY. |
| 346 | |
| 347 | Never a network call. The unconfigured fix is the official-path |
| 348 | prescription on an official-only host (connector lane, bearer, xAI key, |
| 349 | with the about-a-week caveat); elsewhere the plain key hint, |
| 350 | since xapi runs there only under an explicit pin. |
| 351 | """ |
| 352 | requires = "X_BEARER_TOKEN (X API v2)" |
| 353 | if config.get("X_BEARER_TOKEN"): |
| 354 | return BackendFinding( |
| 355 | name="xapi", |
| 356 | status=health.OK, |
| 357 | detail="X_BEARER_TOKEN present", |
| 358 | requires=requires, |
| 359 | ) |
| 360 | if env.x_policy(config).official_only: |
| 361 | entry = prescriptions.get("x", "bearer_missing") |
| 362 | prescription = f"{entry.fix_nl} (cli: {entry.fix_cli})" |
| 363 | else: |
| 364 | prescription = "set X_BEARER_TOKEN in ~/.config/last30days/.env" |
| 365 | return BackendFinding( |
| 366 | name="xapi", |
| 367 | status=health.MISSING, |
| 368 | detail="X_BEARER_TOKEN not set", |
| 369 | prescription=prescription, |
| 370 | requires=requires, |
| 371 | ) |
| 372 | |
| 373 | |
| 374 | def _probe_xurl(config: Dict[str, Any]) -> BackendFinding: |
| 375 | """xurl = official X API v2 CLI (OAuth2). Free lane; LOCAL-ONLY probe. |
| 376 | |
| 377 | Doctor's no-network guarantee forbids the live ``xurl whoami`` check |
| 378 | (``xurl_x.is_available()`` — an authenticated X API call, reserved for |
| 379 | research time). This probe keys on local evidence instead: the binary |
| 380 | on PATH plus xurl's on-disk token store (~/.xurl). Stored credentials |
| 381 | read as OK with an explicit "not live-verified" caveat; an unreadable |
| 382 | token store is a typed ERROR (broken, not unconfigured). |
| 383 | """ |
| 384 | from . import xurl_x |
| 385 | |
| 386 | requires = "xurl CLI installed + OAuth2 login" |
| 387 | if which("xurl") is None: |
| 388 | return BackendFinding( |
| 389 | name="xurl", |
| 390 | status=health.MISSING, |
| 391 | detail="xurl CLI not found on PATH", |
| 392 | prescription="npm install -g xurl && xurl auth oauth2 login", |
| 393 | requires=requires, |
| 394 | ) |
| 395 | store_status, store_detail = xurl_x.stored_auth_status() |
| 396 | if store_status == xurl_x.AUTH_OK: |
| 397 | return BackendFinding( |
| 398 | name="xurl", |
| 399 | status=health.OK, |
| 400 | detail=( |
| 401 | "installed; stored OAuth2 credentials present; " |
| 402 | "auth not live-verified (no network)" |
| 403 | ), |
| 404 | requires=requires, |
| 405 | ) |
| 406 | if store_status == xurl_x.AUTH_ERROR: |
| 407 | return BackendFinding( |
| 408 | name="xurl", |
| 409 | status=health.ERROR, |
| 410 | detail=store_detail, |
| 411 | prescription="xurl auth oauth2 login", |
| 412 | requires=requires, |
| 413 | ) |
| 414 | return BackendFinding( |
| 415 | name="xurl", |
| 416 | status=health.MISSING, |
| 417 | detail="xurl installed but not authenticated", |
| 418 | prescription="xurl auth oauth2 login", |
| 419 | requires=requires, |
| 420 | ) |
| 421 | |
| 422 | |
| 423 | def _probe_ytdlp(config: Dict[str, Any]) -> BackendFinding: |
| 424 | """yt-dlp via the U1 dependency-probe layer (missing/broken/timeout).""" |
| 425 | dep = health.probe_dependency("yt-dlp") |
| 426 | return BackendFinding( |
| 427 | name="yt-dlp", |
| 428 | status=dep.status, |
| 429 | detail=dep.detail, |
| 430 | prescription=dep.prescription, |
| 431 | requires="yt-dlp on the agent-subprocess PATH", |
| 432 | ) |
| 433 | |
| 434 | |
| 435 | def _probe_web_keyless(config: Dict[str, Any]) -> BackendFinding: |
| 436 | """The keyless web-search floor: works keyless, but degraded quality.""" |
| 437 | requires = "no key; suppressed on native-search hosts" |
| 438 | if env.keyless_web_allowed(config): |
| 439 | return BackendFinding( |
| 440 | name="keyless", |
| 441 | status=health.DEGRADED, |
| 442 | detail="keyless search floor (no paid key; lower quality)", |
| 443 | requires=requires, |
| 444 | ) |
| 445 | return BackendFinding( |
| 446 | name="keyless", |
| 447 | status=health.MISSING, |
| 448 | detail="keyless floor suppressed: host has native web search", |
| 449 | prescription="", |
| 450 | requires=requires, |
| 451 | ) |
| 452 | |
| 453 | |
| 454 | def _probe_reddit_public(config: Dict[str, Any]) -> BackendFinding: |
| 455 | """Public keyless Reddit composite; internal lanes are sub-probe detail.""" |
| 456 | return BackendFinding( |
| 457 | name="public", |
| 458 | status=health.OK, |
| 459 | detail="public keyless composite (lanes: rss, listing, arctic, shreddit)", |
| 460 | requires="none (public endpoints)", |
| 461 | ) |
| 462 | |
| 463 | |
| 464 | # --------------------------------------------------------------------------- |
| 465 | # Registry: routing declared once, from env.py's definitions where they exist. |
| 466 | # --------------------------------------------------------------------------- |
| 467 | |
| 468 | _X_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = { |
| 469 | "xai": _key_probe("xai", "XAI_API_KEY", "XAI_API_KEY (xAI/Grok live search)"), |
| 470 | "grok": _probe_grok, |
| 471 | "bird": _probe_bird, |
| 472 | "xurl": _probe_xurl, |
| 473 | "xquik": _key_probe("xquik", "XQUIK_API_KEY", "XQUIK_API_KEY (xquik.com)"), |
| 474 | # Direct X API v2 with an app-only bearer: key presence only, no network. |
| 475 | "xapi": _probe_xapi, |
| 476 | } |
| 477 | _X_PAID = {"xai", "xquik", "xapi"} |
| 478 | # Opt-in backends: never auto-selected; require explicit pin. |
| 479 | _X_OPT_IN = set(env.X_BACKEND_OPT_IN) |
| 480 | |
| 481 | _WEB_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = { |
| 482 | "brave": _key_probe("brave", "BRAVE_API_KEY", "BRAVE_API_KEY"), |
| 483 | "exa": _key_probe("exa", "EXA_API_KEY", "EXA_API_KEY"), |
| 484 | "serper": _key_probe("serper", "SERPER_API_KEY", "SERPER_API_KEY"), |
| 485 | "parallel": _key_probe("parallel", "PARALLEL_API_KEY", "PARALLEL_API_KEY"), |
| 486 | "keyless": _probe_web_keyless, |
| 487 | } |
| 488 | _WEB_KEYED = {"brave", "exa", "serper", "parallel"} |
| 489 | |
| 490 | _SC_SPEC = BackendSpec( |
| 491 | name="scrapecreators", |
| 492 | requires="SCRAPECREATORS_API_KEY", |
| 493 | probe=_key_probe( |
| 494 | "scrapecreators", "SCRAPECREATORS_API_KEY", "SCRAPECREATORS_API_KEY", |
| 495 | note=_SC_PRESCRIPTION, |
| 496 | ), |
| 497 | paid=True, |
| 498 | ) |
| 499 | |
| 500 | # X backend requirements, keyed by name. |
| 501 | _X_REQUIRES: Dict[str, str] = { |
| 502 | "xai": "XAI_API_KEY (xAI/Grok live search)", |
| 503 | "grok": "grok CLI installed + signed in (opt-in only; pin to enable)", |
| 504 | "bird": "X browser cookies (AUTH_TOKEN/CT0) + node", |
| 505 | "xurl": "xurl CLI installed + OAuth2 login", |
| 506 | "xquik": "XQUIK_API_KEY (xquik.com)", |
| 507 | "xapi": "X_BEARER_TOKEN (X API v2)", |
| 508 | } |
| 509 | |
| 510 | DESCRIPTORS: Dict[str, ChainDescriptor] = { |
| 511 | # X: chain order and pin var imported from env.py (single source of truth). |
| 512 | # Backends include the auto chain (X_BACKEND_ORDER) plus opt-in entries |
| 513 | # (X_BACKEND_OPT_IN) for doctor visibility. Opt-in backends like grok |
| 514 | # appear in findings but are never auto-selected; pin to enable. |
| 515 | "x": ChainDescriptor( |
| 516 | source="x", |
| 517 | mode=MODE_ALTERNATIVE, |
| 518 | backends=tuple( |
| 519 | BackendSpec( |
| 520 | name=name, |
| 521 | requires=_X_REQUIRES[name], |
| 522 | probe=_X_PROBES[name], |
| 523 | paid=name in _X_PAID, |
| 524 | opt_in=name in _X_OPT_IN, |
| 525 | ) |
| 526 | for name in env.X_BACKEND_ORDER + env.X_BACKEND_OPT_IN |
| 527 | ), |
| 528 | pin_var=env.X_BACKEND_PIN_VAR, |
| 529 | ), |
| 530 | "youtube": ChainDescriptor( |
| 531 | source="youtube", |
| 532 | mode=MODE_ALTERNATIVE, |
| 533 | backends=( |
| 534 | BackendSpec( |
| 535 | name="yt-dlp", |
| 536 | requires="yt-dlp on the agent-subprocess PATH", |
| 537 | probe=_probe_ytdlp, |
| 538 | ), |
| 539 | _SC_SPEC, |
| 540 | ), |
| 541 | pin_var=None, # no YouTube pin knob exists |
| 542 | ), |
| 543 | "web": ChainDescriptor( |
| 544 | source="web", |
| 545 | mode=MODE_ALTERNATIVE, |
| 546 | backends=tuple( |
| 547 | BackendSpec( |
| 548 | name=name, |
| 549 | requires=(f"{name.upper()}_API_KEY" if name in _WEB_KEYED |
| 550 | else "no key; suppressed on native-search hosts"), |
| 551 | probe=_WEB_PROBES[name], |
| 552 | paid=name in _WEB_KEYED, |
| 553 | ) |
| 554 | for name in WEB_BACKEND_ORDER |
| 555 | ), |
| 556 | pin_var=None, # pinned per-run via --web-backend, not an env var |
| 557 | pin_flag="--web-backend", |
| 558 | ), |
| 559 | "reddit": ChainDescriptor( |
| 560 | source="reddit", |
| 561 | mode=MODE_CONDITIONAL, |
| 562 | backends=( |
| 563 | BackendSpec( |
| 564 | name="public", |
| 565 | requires="none (public endpoints)", |
| 566 | probe=_probe_reddit_public, |
| 567 | ), |
| 568 | _SC_SPEC, |
| 569 | ), |
| 570 | pin_var=env.REDDIT_BACKEND_PIN_VAR, |
| 571 | ), |
| 572 | } |
| 573 | |
| 574 | |
| 575 | def get_descriptor(source: str) -> ChainDescriptor: |
| 576 | """Return the declared routing descriptor for ``source`` (KeyError if none).""" |
| 577 | return DESCRIPTORS[source] |
| 578 | |
| 579 | |
| 580 | # --------------------------------------------------------------------------- |
| 581 | # Resolution |
| 582 | # --------------------------------------------------------------------------- |
| 583 | |
| 584 | def resolve( |
| 585 | source: str, |
| 586 | config: Dict[str, Any], |
| 587 | pin: Optional[str] = None, |
| 588 | ) -> BackendResolution: |
| 589 | """Resolve a chained source's routing into a truthful prediction. |
| 590 | |
| 591 | ``pin`` is an explicit per-run pin (the ``--web-backend`` flag); it |
| 592 | takes precedence over the descriptor's env pin var. ``"auto"``/None |
| 593 | mean unpinned. Probing is side-effect-free and collect-then-pick. |
| 594 | |
| 595 | Time budget: backends are probed sequentially, so a chain's budget is |
| 596 | ADDITIVE across its backends — each binary-backed probe is bounded by |
| 597 | ``health.PROBE_TIMEOUT`` and paid/key lanes are dict lookups that cost |
| 598 | nothing, giving a worst case of roughly (binary probes in the chain) x |
| 599 | ``health.PROBE_TIMEOUT``. Deliberately no intra-chain concurrency: |
| 600 | probes are memoized per process and the worst case only occurs when |
| 601 | multiple binaries are simultaneously hung. |
| 602 | """ |
| 603 | descriptor = get_descriptor(source) |
| 604 | specs, auto_names = _specs_for_policy(descriptor, config) |
| 605 | findings = [_run_probe(spec, config) for spec in specs] |
| 606 | if descriptor.mode == MODE_CONDITIONAL: |
| 607 | return _resolve_conditional(descriptor, config, findings) |
| 608 | return _resolve_alternative(descriptor, config, findings, pin, auto_names) |
| 609 | |
| 610 | |
| 611 | def _specs_for_policy( |
| 612 | descriptor: ChainDescriptor, |
| 613 | config: Dict[str, Any], |
| 614 | ) -> Tuple[List[BackendSpec], set]: |
| 615 | """The backends to probe for this host, and which of them auto-select. |
| 616 | |
| 617 | Every source except X keeps its declared backends; auto-selection is the |
| 618 | non-opt-in set. For X the answer comes from ``env.x_policy``: on a |
| 619 | default host the declared chain (auto order plus opt-in entries for |
| 620 | doctor visibility) is unchanged; on an official-only host the findings |
| 621 | are the policy's chain in its order, plus the pinned backend when the |
| 622 | pin names something outside it, so neither doctor JSON nor the chain |
| 623 | string carries a non-official backend unless it is pinned. Observation |
| 624 | only: this mirrors ``env.x_backend_chain``, it never alters it. |
| 625 | """ |
| 626 | specs = list(descriptor.backends) |
| 627 | if descriptor.source != "x": |
| 628 | return specs, {spec.name for spec in specs if not spec.opt_in} |
| 629 | policy = env.x_policy(config) |
| 630 | auto_names = set(policy.auto_chain) |
| 631 | if not policy.official_only: |
| 632 | return specs, auto_names |
| 633 | by_name = {spec.name: spec for spec in specs} |
| 634 | ordered = [by_name[name] for name in policy.auto_chain if name in by_name] |
| 635 | pin = env.x_backend_pin(config) |
| 636 | if pin in by_name and pin not in auto_names: |
| 637 | ordered.append(by_name[pin]) |
| 638 | return ordered, auto_names |
| 639 | |
| 640 | |
| 641 | def _run_probe(spec: BackendSpec, config: Dict[str, Any]) -> BackendFinding: |
| 642 | """Run one probe, isolating failures so one bad probe can't blank a chain.""" |
| 643 | try: |
| 644 | finding = spec.probe(config) |
| 645 | except Exception as exc: # a probe bug must not take the report down |
| 646 | finding = BackendFinding( |
| 647 | name=spec.name, |
| 648 | status=health.ERROR, |
| 649 | detail=f"probe failed: {type(exc).__name__}: {exc}", |
| 650 | requires=spec.requires, |
| 651 | ) |
| 652 | if not finding.requires: |
| 653 | finding.requires = spec.requires |
| 654 | return finding |
| 655 | |
| 656 | |
| 657 | def _resolve_alternative( |
| 658 | descriptor: ChainDescriptor, |
| 659 | config: Dict[str, Any], |
| 660 | findings: List[BackendFinding], |
| 661 | pin: Optional[str], |
| 662 | auto_names: Optional[set] = None, |
| 663 | ) -> BackendResolution: |
| 664 | names = [f.name for f in findings] |
| 665 | by_name = {f.name: f for f in findings} |
| 666 | if auto_names is None: |
| 667 | auto_names = {spec.name for spec in descriptor.backends if not spec.opt_in} |
| 668 | # Backends outside the auto set are opt-in here (never auto-selected). |
| 669 | opt_in_names = {name for name in names if name not in auto_names} |
| 670 | res = BackendResolution( |
| 671 | source=descriptor.source, |
| 672 | mode=MODE_ALTERNATIVE, |
| 673 | chain=list(names), |
| 674 | findings=findings, |
| 675 | ) |
| 676 | |
| 677 | pin_name: Optional[str] = None |
| 678 | if pin and pin not in ("auto", "none") and pin in by_name: |
| 679 | pin_name = pin |
| 680 | elif descriptor.pin_var: |
| 681 | raw = (config.get(descriptor.pin_var) or "").lower() |
| 682 | if raw in by_name: |
| 683 | pin_name = raw |
| 684 | |
| 685 | if pin_name: |
| 686 | # A pin forces a single backend (no failover) — mirror |
| 687 | # env.x_backend_chain's pin semantics exactly. |
| 688 | res.pinned = True |
| 689 | res.pin = pin_name |
| 690 | finding = by_name[pin_name] |
| 691 | if finding.status == health.OK: |
| 692 | res.active_backend = pin_name |
| 693 | res.tier = TIER_OK |
| 694 | elif finding.status == health.DEGRADED: |
| 695 | res.active_backend = pin_name |
| 696 | res.tier = TIER_WARN |
| 697 | else: |
| 698 | res.tier = TIER_ERROR |
| 699 | res.prescription = finding.prescription or ( |
| 700 | f"unpin {descriptor.pin_var or descriptor.pin_flag} or fix {pin_name}" |
| 701 | ) |
| 702 | return res |
| 703 | |
| 704 | # Collect-then-pick: first fully-usable wins; else best degraded; else |
| 705 | # error carrying the highest-priority backend's prescription. |
| 706 | # Opt-in backends are NEVER auto-selected; skip them entirely. |
| 707 | auto_findings = [f for f in findings if f.name not in opt_in_names] |
| 708 | for finding in auto_findings: |
| 709 | if finding.status == health.OK: |
| 710 | res.active_backend = finding.name |
| 711 | res.tier = TIER_OK |
| 712 | return res |
| 713 | for finding in auto_findings: |
| 714 | if finding.status == health.DEGRADED: |
| 715 | res.active_backend = finding.name |
| 716 | res.tier = TIER_WARN |
| 717 | return res |
| 718 | res.tier = TIER_ERROR |
| 719 | # Prescription comes from the first auto-chain backend, not opt-in. |
| 720 | res.prescription = auto_findings[0].prescription if auto_findings else "" |
| 721 | return res |
| 722 | |
| 723 | |
| 724 | def _reddit_sc_min_items(config: Dict[str, Any]) -> int: |
| 725 | """The thinness floor, parsed exactly as the pipeline parses it |
| 726 | (lib/pipeline.py reddit fetch: int(... or 0), malformed -> 0).""" |
| 727 | try: |
| 728 | return int(config.get(env.REDDIT_SC_MIN_ITEMS_VAR) or 0) |
| 729 | except (TypeError, ValueError): |
| 730 | return 0 |
| 731 | |
| 732 | |
| 733 | def _resolve_conditional( |
| 734 | descriptor: ChainDescriptor, |
| 735 | config: Dict[str, Any], |
| 736 | findings: List[BackendFinding], |
| 737 | ) -> BackendResolution: |
| 738 | """Reddit: render the real per-query semantics, never a computed winner.""" |
| 739 | res = BackendResolution( |
| 740 | source=descriptor.source, |
| 741 | mode=MODE_CONDITIONAL, |
| 742 | chain=[spec.name for spec in descriptor.backends], |
| 743 | findings=findings, |
| 744 | active_backend=None, # conditional mode never picks a winner |
| 745 | tier=TIER_OK, # the public keyless composite is always reachable |
| 746 | ) |
| 747 | has_key = bool(config.get("SCRAPECREATORS_API_KEY")) |
| 748 | raw_pin = (config.get(descriptor.pin_var) or "").lower() if descriptor.pin_var else "" |
| 749 | pinned_sc = has_key and raw_pin == "scrapecreators" |
| 750 | floor = _reddit_sc_min_items(config) |
| 751 | |
| 752 | if pinned_sc: |
| 753 | res.pinned = True |
| 754 | res.pin = "scrapecreators" |
| 755 | res.conditional = ( |
| 756 | f"ScrapeCreators primary (pinned via {descriptor.pin_var}); " |
| 757 | "public keyless composite fallback" |
| 758 | ) |
| 759 | return res |
| 760 | |
| 761 | if has_key: |
| 762 | if floor > 0: |
| 763 | backfill = ( |
| 764 | f"ScrapeCreators backfill when results fall below the " |
| 765 | f"{floor}-item floor" |
| 766 | ) |
| 767 | else: |
| 768 | backfill = "ScrapeCreators backfill when the free path returns nothing" |
| 769 | res.conditional = f"public keyless composite (default); {backfill}" |
| 770 | return res |
| 771 | |
| 772 | res.conditional = "public keyless composite (default); no ScrapeCreators key for backfill" |
| 773 | if raw_pin == "scrapecreators": |
| 774 | # The pipeline ignores the pin without a key; say so honestly. |
| 775 | res.conditional += ( |
| 776 | f" ({descriptor.pin_var} pin ignored: SCRAPECREATORS_API_KEY not set)" |
| 777 | ) |
| 778 | return res |
| 779 |