| 1 | """X (Twitter) search via the Grok CLI — no X credential of any kind. |
| 2 | |
| 3 | The `grok` CLI (https://x.ai/cli) exposes X search tools natively |
| 4 | (`x_keyword_search`, `x_semantic_search`, `x_thread_fetch`, `x_user_search`). |
| 5 | Reaching X through it needs no X account, no browser cookies, and no |
| 6 | `XAI_API_KEY` — only an installed and signed-in `grok`. |
| 7 | |
| 8 | Install: curl -fsSL https://x.ai/cli/install.sh | bash (or npm i -g @xai-official/grok) |
| 9 | Auth: grok login |
| 10 | |
| 11 | Two invocation constraints, both measured, both load-bearing: |
| 12 | |
| 13 | * **Never pass `--json-schema`.** Constrained decoding competes with tool use: |
| 14 | the search silently does not run and the model fills the schema's required |
| 15 | fields from training data instead. Measured with an interleaved A/B |
| 16 | controlling for time: plain output returned verified in-window posts on 4 of |
| 17 | 4 calls, `--json-schema` on 1 of 4. |
| 18 | * **Never pass `--tools`.** Two runs produced no output in 7 minutes and were |
| 19 | killed; the identical prompts without it completed normally. |
| 20 | * **Do pass `--output-format json`.** Grok CLI 1.0.5 narrates tool use and |
| 21 | then fences a JSON array; the field-block parser treats that as empty |
| 22 | (``no items parsed``). JSON stdout is the CLI's supported way to skip the |
| 23 | preamble without constrained decoding. See #1051. |
| 24 | |
| 25 | Because retrieval is performed by a language model rather than an API client, |
| 26 | its output can be *confidently wrong* in a way no other backend's can. Every |
| 27 | returned post is therefore validated against the requested window via its |
| 28 | snowflake timestamp before it is allowed into the item flow — see |
| 29 | `_validate_items`. Author matching and schema shape are not sufficient: a |
| 30 | fabricated post carries a plausible handle and a numeric id by construction. |
| 31 | """ |
| 32 | |
| 33 | import json |
| 34 | import os |
| 35 | import re |
| 36 | import shutil |
| 37 | import subprocess |
| 38 | import tempfile |
| 39 | import time |
| 40 | from datetime import datetime, timezone |
| 41 | from pathlib import Path |
| 42 | from typing import Any, Dict, List, Optional, Tuple |
| 43 | |
| 44 | from . import log |
| 45 | from .relevance import token_overlap_relevance as _compute_relevance |
| 46 | # One copy of the snowflake, handle-grammar, and generated-sequence helpers |
| 47 | # lives in x_api; grok_x keeps its private names for its callers and |
| 48 | # tests. Model-reported handles are interpolated into post URLs and into the |
| 49 | # NEXT child prompt, so anything outside the X handle charset is rejected |
| 50 | # rather than passed through (entity_extract applies the same rule). |
| 51 | from .x_api import ( # noqa: F401 - re-exported under the same names |
| 52 | _HANDLE_RE, |
| 53 | _SNOWFLAKE_EPOCH_MS, |
| 54 | _clean_handle, |
| 55 | _decode_snowflake, |
| 56 | _looks_generated, |
| 57 | ) |
| 58 | |
| 59 | |
| 60 | def _log(msg: str) -> None: |
| 61 | log.source_log("Grok", msg, tty_only=False) |
| 62 | |
| 63 | |
| 64 | # Posts requested per call. The tool caps `limit` at 10, so depth is achieved |
| 65 | # by fanning out across queries rather than by raising a single call's limit. |
| 66 | _MAX_LIMIT_PER_CALL = 10 |
| 67 | |
| 68 | # Upper bound on calls per topic search. Each call is an LLM subprocess of |
| 69 | # roughly 15-45s, so depth must not translate into unbounded wall time. |
| 70 | _MAX_FANOUT_CALLS = 4 |
| 71 | |
| 72 | # Wall-clock ceiling for ALL grok Phase 2 lanes combined. Without it, three |
| 73 | # lanes over three handles is up to 14 sequential LLM subprocess calls bounded |
| 74 | # only by per-call timeouts -- tens of minutes of foreground time for a source |
| 75 | # that now runs by default. Lanes stop issuing queries once this passes and |
| 76 | # return whatever they have. |
| 77 | LANE_BUDGET_SECONDS = 150.0 |
| 78 | |
| 79 | # Below this, a call cannot plausibly complete (measured calls run 15-45s), so |
| 80 | # the budget is spent rather than overrun. Skipping is strictly better than |
| 81 | # starting a call guaranteed to be killed mid-flight. |
| 82 | _MIN_USEFUL_CALL_SECONDS = 15 |
| 83 | |
| 84 | |
| 85 | def _is_proper_name(topic: str) -> bool: |
| 86 | """True when topic looks like a title-cased proper name (person/product). |
| 87 | |
| 88 | "Peter Steinberger" → True (phrase-quote in fanout) |
| 89 | "Rome Italy" → False (no phrase-quote; place/disambiguation string) |
| 90 | """ |
| 91 | words = topic.split() |
| 92 | if len(words) < 2: |
| 93 | return False |
| 94 | # Title-cased: each word starts uppercase, rest lowercase |
| 95 | # Place names like "Rome Italy" are title-cased but are NOT proper names |
| 96 | # for phrase-quoting purposes. Heuristic: if ALL words are common place/ |
| 97 | # disambiguation words OR all-caps acronyms, don't phrase-quote. |
| 98 | place_words = { |
| 99 | "italy", "rome", "paris", "london", "berlin", "tokyo", "new", "york", |
| 100 | "los", "angeles", "san", "francisco", "city", "country", "state", |
| 101 | "north", "south", "east", "west", "united", "states", "kingdom", |
| 102 | } |
| 103 | lower_words = [w.lower() for w in words] |
| 104 | if all(w in place_words or w.isupper() for w in lower_words): |
| 105 | return False |
| 106 | # Check for title case pattern (First Last, First Middle Last) |
| 107 | return all( |
| 108 | w[0].isupper() and (len(w) == 1 or w[1:].islower()) |
| 109 | for w in words |
| 110 | if w.isalpha() |
| 111 | ) |
| 112 | |
| 113 | |
| 114 | def _fanout_queries(topic: str, from_date: str, to_date: str, calls: int) -> List[str]: |
| 115 | """Distinct query formulations for one topic, widest signal first. |
| 116 | |
| 117 | Each returns at most 10 posts, and the formulations surface different |
| 118 | sets -- Top vs Latest ordering, and an engagement-floored variant -- so |
| 119 | fanning out adds coverage rather than repeating one result set. |
| 120 | |
| 121 | Multi-word topics are NOT phrase-quoted unless they look like proper names |
| 122 | (person/product). "Rome Italy" → no phrase-quote (place/disambiguation). |
| 123 | "Peter Steinberger" → phrase-quote in one variant (proper name). |
| 124 | """ |
| 125 | window = f"since:{from_date} until:{to_date}" |
| 126 | # First variant: unquoted AND (multi-word topics naturally AND their terms) |
| 127 | variants = [ |
| 128 | f"{topic} {window}", |
| 129 | f"{topic} {window} min_faves:5", |
| 130 | ] |
| 131 | # Third variant: phrase-quote only for proper names, else filter:links |
| 132 | if " " in topic and _is_proper_name(topic): |
| 133 | variants.append(f'"{topic}" {window}') |
| 134 | else: |
| 135 | variants.append(f"{topic} {window} filter:links") |
| 136 | variants.append(f"{topic} {window} -filter:replies") |
| 137 | return variants[:calls] |
| 138 | |
| 139 | DEPTH_CONFIG = { |
| 140 | "quick": 10, |
| 141 | "default": 30, |
| 142 | "deep": 60, |
| 143 | } |
| 144 | |
| 145 | # Wall-clock ceiling for one `grok` invocation. A run that blocks on an |
| 146 | # unexpected interactive prompt would otherwise hang indefinitely, and a |
| 147 | # non-daemon worker can outlive a wall-clock budget. |
| 148 | _TIMEOUT_SECONDS = {"quick": 120, "default": 240, "deep": 360} |
| 149 | |
| 150 | _AUTH_STORE = Path.home() / ".grok" / "auth.json" |
| 151 | |
| 152 | # Substrings that indicate stored credentials. Deliberately format-agnostic: |
| 153 | # the observed store is a JSON object keyed by issuer and principal, but the |
| 154 | # shape is the vendor's to change. Mirrors xurl_x's marker scan. |
| 155 | _TOKEN_STORE_MARKERS = ("refresh_token", "access_token", "auth_mode", '"key"') |
| 156 | |
| 157 | AUTH_OK = "ok" # token store present with non-expired credentials |
| 158 | AUTH_EXPIRED = "expired" # credentials present but access_token expires_at is past |
| 159 | AUTH_MISSING = "missing" # no token store, or no credentials stored in it |
| 160 | AUTH_ERROR = "error" # token store exists but could not be read |
| 161 | |
| 162 | # Markers that indicate the Grok session was revoked mid-run (refresh failed). |
| 163 | # When these appear in grok CLI stderr/stdout, the run should fall back once |
| 164 | # and not retry grok in that run. Distinct from "never signed in" since a prior |
| 165 | # run may have succeeded with the same auth.json. |
| 166 | _AUTH_REVOKED_MARKERS = ( |
| 167 | "not signed in", |
| 168 | "not logged in", |
| 169 | "invalid_grant", |
| 170 | "refresh token has been revoked", |
| 171 | "session expired", |
| 172 | "authentication failed", |
| 173 | "unauthorized", |
| 174 | ) |
| 175 | |
| 176 | _availability_cache: Optional[bool] = None |
| 177 | |
| 178 | |
| 179 | def clear_availability_cache() -> None: |
| 180 | """Reset the memoized is_available() result (tests, or a re-check after login).""" |
| 181 | global _availability_cache |
| 182 | _availability_cache = None |
| 183 | |
| 184 | |
| 185 | def binary_path() -> Optional[str]: |
| 186 | """Resolved `grok` path, or None when it is not on PATH. |
| 187 | |
| 188 | PATH resolution is the gate, not file existence: a binary present on disk |
| 189 | but off the agent subprocess PATH is not installed as far as the engine is |
| 190 | concerned. |
| 191 | """ |
| 192 | return shutil.which("grok") |
| 193 | |
| 194 | |
| 195 | def token_store_path() -> Path: |
| 196 | return _AUTH_STORE |
| 197 | |
| 198 | |
| 199 | def _find_expires_at(obj: Any) -> Optional[str]: |
| 200 | """Recursively find expires_at in a nested dict/list structure. |
| 201 | |
| 202 | The Grok auth.json is keyed by issuer and principal; this finds expires_at |
| 203 | anywhere in the tree without assuming the structure. |
| 204 | """ |
| 205 | if isinstance(obj, dict): |
| 206 | if "expires_at" in obj: |
| 207 | return obj["expires_at"] |
| 208 | for v in obj.values(): |
| 209 | found = _find_expires_at(v) |
| 210 | if found is not None: |
| 211 | return found |
| 212 | elif isinstance(obj, list): |
| 213 | for item in obj: |
| 214 | found = _find_expires_at(item) |
| 215 | if found is not None: |
| 216 | return found |
| 217 | return None |
| 218 | |
| 219 | |
| 220 | def _parse_expires_at(raw: str) -> Optional[datetime]: |
| 221 | """Parse an ISO 8601 expires_at timestamp.""" |
| 222 | if not raw: |
| 223 | return None |
| 224 | try: |
| 225 | normalized = raw.replace("Z", "+00:00") |
| 226 | return datetime.fromisoformat(normalized) |
| 227 | except (TypeError, ValueError): |
| 228 | return None |
| 229 | |
| 230 | |
| 231 | def stored_auth_status() -> Tuple[str, str, Optional[datetime]]: |
| 232 | """Local-only auth check: filesystem read, no subprocess, no network. |
| 233 | |
| 234 | This is the doctor / --diagnose / --preflight surface. It must never spawn |
| 235 | a process: the whole-doctor-path test patches ``subprocess.run`` to raise, |
| 236 | and shelling out to `grok` here would fail it. |
| 237 | |
| 238 | Returns (status, detail, expires_at). The expires_at datetime is None when |
| 239 | not parseable or not present. Status is: |
| 240 | - AUTH_OK: credentials present and not expired (or no expires_at to check) |
| 241 | - AUTH_EXPIRED: credentials present but expires_at is in the past |
| 242 | - AUTH_MISSING: no token store or no credential markers |
| 243 | - AUTH_ERROR: token store exists but could not be read |
| 244 | """ |
| 245 | path = token_store_path() |
| 246 | try: |
| 247 | if not path.exists(): |
| 248 | return AUTH_MISSING, f"no Grok credential store at {path}", None |
| 249 | raw = path.read_text(encoding="utf-8", errors="replace") |
| 250 | except OSError as exc: |
| 251 | return AUTH_ERROR, f"{type(exc).__name__}: {exc}", None |
| 252 | |
| 253 | if not any(marker in raw for marker in _TOKEN_STORE_MARKERS): |
| 254 | return AUTH_MISSING, f"Grok credential store at {path} has no stored credentials", None |
| 255 | |
| 256 | expires_at: Optional[datetime] = None |
| 257 | try: |
| 258 | data = json.loads(raw) |
| 259 | expires_str = _find_expires_at(data) |
| 260 | expires_at = _parse_expires_at(expires_str) if expires_str else None |
| 261 | except (json.JSONDecodeError, TypeError): |
| 262 | pass |
| 263 | |
| 264 | if expires_at is not None: |
| 265 | now = datetime.now(timezone.utc) |
| 266 | if expires_at.tzinfo is None: |
| 267 | expires_at = expires_at.replace(tzinfo=timezone.utc) |
| 268 | if expires_at < now: |
| 269 | return ( |
| 270 | AUTH_EXPIRED, |
| 271 | f"Grok session expired at {expires_at.isoformat()} " |
| 272 | f"(refresh may restore it; if revoked, run `grok login --device-auth`)", |
| 273 | expires_at, |
| 274 | ) |
| 275 | |
| 276 | return AUTH_OK, f"stored Grok credentials found in {path}", expires_at |
| 277 | |
| 278 | |
| 279 | def has_stored_auth() -> bool: |
| 280 | """True when grok binary is on PATH and credentials are stored. |
| 281 | |
| 282 | NOTE: This returns True even when AUTH_EXPIRED, because the refresh_token |
| 283 | might still work. The caller (is_available) decides whether to attempt |
| 284 | grok anyway. Doctor uses the status directly to show the degraded state. |
| 285 | """ |
| 286 | if binary_path() is None: |
| 287 | return False |
| 288 | status = stored_auth_status()[0] |
| 289 | return status in (AUTH_OK, AUTH_EXPIRED) |
| 290 | |
| 291 | |
| 292 | def is_available() -> bool: |
| 293 | """Research-time availability. May spawn a subprocess; memoized per process.""" |
| 294 | global _availability_cache |
| 295 | if _availability_cache is None: |
| 296 | _availability_cache = _is_available_uncached() |
| 297 | return _availability_cache |
| 298 | |
| 299 | |
| 300 | def _is_available_uncached() -> bool: |
| 301 | """Research-time availability check. |
| 302 | |
| 303 | Returns True when grok is on PATH and credentials exist, even if |
| 304 | AUTH_EXPIRED. Rationale: expires_at being in the past does not prove the |
| 305 | refresh_token is dead. The CLI will attempt OIDC refresh at run time and |
| 306 | might succeed. Only a runtime failure ("Not signed in", invalid_grant) |
| 307 | proves the session is truly revoked. |
| 308 | """ |
| 309 | if binary_path() is None: |
| 310 | return False |
| 311 | status = stored_auth_status()[0] |
| 312 | return status in (AUTH_OK, AUTH_EXPIRED) |
| 313 | |
| 314 | |
| 315 | def _subprocess_env(home: str) -> Dict[str, str]: |
| 316 | """Minimal environment for the `grok` child process, rooted at a throwaway HOME. |
| 317 | |
| 318 | The child runs with tool permissions bypassed (non-interactivity requires |
| 319 | it) while its context is filled with retrieved X post text, which is |
| 320 | attacker-controlled. Stripping credential env vars is necessary but not |
| 321 | sufficient: this engine writes XAI_API_KEY / AUTH_TOKEN / CT0 / |
| 322 | SCRAPECREATORS_API_KEY to ``$HOME/.config/last30days/.env``, and ``~/.ssh`` |
| 323 | and ``~/.aws`` sit alongside it. An empty cwd is not a boundary for a |
| 324 | filesystem-capable agent -- cwd constrains relative paths, not ``$HOME/...`` |
| 325 | reads -- so the child gets its own HOME containing only the Grok credential |
| 326 | store it actually needs. |
| 327 | """ |
| 328 | keep = ("PATH", "LANG", "LC_ALL", "TMPDIR", "SystemRoot") |
| 329 | env = {k: os.environ[k] for k in keep if k in os.environ} |
| 330 | env.setdefault("PATH", os.defpath) |
| 331 | env["HOME"] = home |
| 332 | if os.name == "nt": |
| 333 | env["USERPROFILE"] = home |
| 334 | return env |
| 335 | |
| 336 | |
| 337 | def _stage_child_home(workdir: str) -> str: |
| 338 | """Create the child's throwaway HOME holding only the credential file. |
| 339 | |
| 340 | Copies ``auth.json`` alone, never the ``~/.grok`` tree: that directory is |
| 341 | ~1.6 GB (marketplace cache, bundled runtime, session history), and copying |
| 342 | it per invocation made a single search take minutes. The child needs the |
| 343 | credential to authenticate and nothing else -- session history and caches |
| 344 | are state we specifically do not want a permission-bypassed child to read |
| 345 | or mutate. |
| 346 | |
| 347 | Copied rather than symlinked so the child cannot follow a link back to the |
| 348 | real store, and copied rather than shared so it cannot rewrite the user's |
| 349 | credentials. |
| 350 | """ |
| 351 | home = os.path.join(workdir, "home") |
| 352 | store = token_store_path() |
| 353 | child_store_dir = os.path.join(home, store.parent.name) |
| 354 | os.makedirs(child_store_dir, mode=0o700, exist_ok=True) |
| 355 | try: |
| 356 | if store.is_file(): |
| 357 | shutil.copyfile(store, os.path.join(child_store_dir, store.name)) |
| 358 | os.chmod(os.path.join(child_store_dir, store.name), 0o600) |
| 359 | except OSError as exc: |
| 360 | _log(f"could not stage Grok credentials for the child: {exc}") |
| 361 | return home |
| 362 | |
| 363 | |
| 364 | # Why the most recent parse returned nothing. Lets _run_query distinguish a |
| 365 | # clean empty window (common, and not worth a second LLM call) from a suspect |
| 366 | # response (fabricated ids, a generated sequence, a self-reported |
| 367 | # non-execution), which is the only case retrying can actually fix. |
| 368 | _LAST_REJECTION = {"reason": ""} |
| 369 | |
| 370 | _RETRYABLE_REJECTIONS = ( |
| 371 | "failed provenance validation", |
| 372 | "near-uniform sequence", |
| 373 | "unparsable date window", |
| 374 | ) |
| 375 | |
| 376 | _PLACEHOLDER_HANDLES = {"unknown", "n/a", "none", "null", "example", "user", ""} |
| 377 | |
| 378 | _NON_EXECUTION_MARKERS = ( |
| 379 | "was not executed", |
| 380 | "not executed in this turn", |
| 381 | "unable to search", |
| 382 | "could not search", |
| 383 | "no tool call", |
| 384 | "tool not available", |
| 385 | ) |
| 386 | |
| 387 | |
| 388 | def _validate_items( |
| 389 | items: List[Dict[str, Any]], |
| 390 | from_date: str, |
| 391 | to_date: str, |
| 392 | ) -> Tuple[List[Dict[str, Any]], str]: |
| 393 | """Drop anything that did not come from a real in-window post. |
| 394 | |
| 395 | Returns (kept, reason). A non-empty reason means the response should be |
| 396 | treated as a non-execution to retry rather than as a thin result. |
| 397 | """ |
| 398 | if not items: |
| 399 | return [], "no items parsed" |
| 400 | |
| 401 | try: |
| 402 | lo = datetime.strptime(from_date, "%Y-%m-%d").replace(tzinfo=timezone.utc) |
| 403 | hi = datetime.strptime(to_date, "%Y-%m-%d").replace(tzinfo=timezone.utc) |
| 404 | except (TypeError, ValueError): |
| 405 | # Fail closed. Skipping the window check would silently disable the |
| 406 | # module's central provenance guarantee for the whole response. |
| 407 | return [], "unparsable date window" |
| 408 | |
| 409 | kept: List[Dict[str, Any]] = [] |
| 410 | for item in items: |
| 411 | text = str(item.get("text") or "").lower() |
| 412 | if any(marker in text for marker in _NON_EXECUTION_MARKERS): |
| 413 | continue |
| 414 | handle = str(item.get("author_handle") or "").strip().lstrip("@").lower() |
| 415 | if handle in _PLACEHOLDER_HANDLES: |
| 416 | continue |
| 417 | created = _decode_snowflake(item.get("post_id")) |
| 418 | if created is None: |
| 419 | continue |
| 420 | if not (lo <= created <= hi.replace(hour=23, minute=59, second=59)): |
| 421 | continue |
| 422 | kept.append(item) |
| 423 | |
| 424 | if not kept: |
| 425 | return [], "every item failed provenance validation (window/handle/id)" |
| 426 | if _looks_generated([str(i.get("post_id")) for i in kept]): |
| 427 | return [], "post ids form a near-uniform sequence (generated, not retrieved)" |
| 428 | return kept, "" |
| 429 | |
| 430 | |
| 431 | # --- prose parsing --------------------------------------------------------- |
| 432 | |
| 433 | _FIELD_ALIASES = { |
| 434 | "id": "post_id", |
| 435 | "post id": "post_id", |
| 436 | "conversation id": "conversation_id", |
| 437 | "author": "author", |
| 438 | "handle": "author_handle", |
| 439 | "text": "text", |
| 440 | "content": "text", |
| 441 | "created_at": "created_at", |
| 442 | "timestamp": "created_at", |
| 443 | "likes": "likes", |
| 444 | "reposts": "reposts", |
| 445 | "retweets": "reposts", |
| 446 | "replies": "replies", |
| 447 | "quotes": "quotes", |
| 448 | "bookmarks": "bookmarks", |
| 449 | "views": "views", |
| 450 | } |
| 451 | |
| 452 | _FIELD_LINE = re.compile( |
| 453 | r"^[\s\-*>]*\**\s*([A-Za-z][A-Za-z _]{1,20}?)\**\s*[:=]\s*(.+?)\s*$" |
| 454 | ) |
| 455 | # Engagement counts arrive either literal ("1,462") or display-abbreviated |
| 456 | # ("39K", "1.2M"). Parsing only the leading digits turns 1.2M into 1, which |
| 457 | # does not merely lose precision -- it inverts ranking, placing a viral post |
| 458 | # below one with 500 literal likes. |
| 459 | _INT_RE = re.compile(r"(-?\d[\d,]*(?:\.\d+)?)\s*([KMB])?", re.I) |
| 460 | _SUFFIX_MULTIPLIER = {"k": 1_000, "m": 1_000_000, "b": 1_000_000_000} |
| 461 | _STATUS_URL_RE = re.compile( |
| 462 | r"(?:x\.com|twitter\.com)/([A-Za-z0-9_]{1,15})/status/(\d+)", |
| 463 | re.I, |
| 464 | ) |
| 465 | _JSON_FENCE_RE = re.compile(r"```(?:json)?\s*\n?(.*?)\n?```", re.I | re.S) |
| 466 | |
| 467 | |
| 468 | def _as_int(value: str) -> Optional[int]: |
| 469 | match = _INT_RE.search(value or "") |
| 470 | if not match: |
| 471 | return None |
| 472 | number, suffix = match.group(1), match.group(2) |
| 473 | try: |
| 474 | parsed = float(number.replace(",", "")) |
| 475 | except ValueError: |
| 476 | return None |
| 477 | if suffix: |
| 478 | parsed *= _SUFFIX_MULTIPLIER[suffix.lower()] |
| 479 | return int(parsed) |
| 480 | |
| 481 | |
| 482 | def _parse_date(value: str) -> Optional[str]: |
| 483 | value = (value or "").strip() |
| 484 | for fmt in ("%a, %d %b %Y %H:%M:%S %Z", "%a %b %d %H:%M:%S %z %Y"): |
| 485 | try: |
| 486 | return datetime.strptime(value, fmt).strftime("%Y-%m-%d") |
| 487 | except (TypeError, ValueError): |
| 488 | continue |
| 489 | try: |
| 490 | return datetime.fromisoformat(value.replace("Z", "+00:00")).strftime("%Y-%m-%d") |
| 491 | except (TypeError, ValueError): |
| 492 | return None |
| 493 | |
| 494 | |
| 495 | def _fields_from_json_obj(obj: Any) -> Optional[Dict[str, Any]]: |
| 496 | """Map one JSON object onto the field-block shape, or None if unusable.""" |
| 497 | if not isinstance(obj, dict): |
| 498 | return None |
| 499 | url = str(obj.get("url") or obj.get("link") or "") |
| 500 | post_id = obj.get("id") or obj.get("post_id") or obj.get("status_id") or obj.get("tweet_id") |
| 501 | handle: Any = ( |
| 502 | obj.get("handle") |
| 503 | or obj.get("author_handle") |
| 504 | or obj.get("username") |
| 505 | or obj.get("author") |
| 506 | or obj.get("user") |
| 507 | ) |
| 508 | if isinstance(handle, dict): |
| 509 | handle = ( |
| 510 | handle.get("username") |
| 511 | or handle.get("screen_name") |
| 512 | or handle.get("handle") |
| 513 | or "" |
| 514 | ) |
| 515 | match = _STATUS_URL_RE.search(url) |
| 516 | if match: |
| 517 | handle = handle or match.group(1) |
| 518 | post_id = post_id or match.group(2) |
| 519 | post_id_s = str(post_id or "").strip() |
| 520 | if not post_id_s.isdigit(): |
| 521 | return None |
| 522 | text = obj.get("text") or obj.get("full_text") or "" |
| 523 | # `content` is often the CLI wrapper payload, not post text. |
| 524 | if not text and "content" in obj and not isinstance(obj.get("content"), (dict, list)): |
| 525 | text = obj.get("content") or "" |
| 526 | return { |
| 527 | "post_id": post_id_s, |
| 528 | "author_handle": str(handle or "").strip().lstrip("@"), |
| 529 | "text": str(text or ""), |
| 530 | "likes": obj.get("likes") if obj.get("likes") is not None else obj.get("favorite_count"), |
| 531 | "reposts": obj.get("reposts") if obj.get("reposts") is not None else obj.get("retweets"), |
| 532 | "replies": obj.get("replies"), |
| 533 | "quotes": obj.get("quotes"), |
| 534 | "created_at": obj.get("created_at") or obj.get("timestamp") or "", |
| 535 | } |
| 536 | |
| 537 | |
| 538 | def _extract_json_values(text: str) -> List[Any]: |
| 539 | """Yield JSON values from raw stdout, fenced blocks, or embedded payloads.""" |
| 540 | values: List[Any] = [] |
| 541 | text = text or "" |
| 542 | try: |
| 543 | values.append(json.loads(text)) |
| 544 | except (json.JSONDecodeError, TypeError): |
| 545 | pass |
| 546 | for match in _JSON_FENCE_RE.finditer(text): |
| 547 | blob = (match.group(1) or "").strip() |
| 548 | try: |
| 549 | values.append(json.loads(blob)) |
| 550 | except (json.JSONDecodeError, TypeError): |
| 551 | continue |
| 552 | if values: |
| 553 | return values |
| 554 | decoder = json.JSONDecoder() |
| 555 | idx = 0 |
| 556 | while idx < len(text): |
| 557 | starts = [i for i in (text.find("[", idx), text.find("{", idx)) if i >= 0] |
| 558 | if not starts: |
| 559 | break |
| 560 | start = min(starts) |
| 561 | try: |
| 562 | value, end = decoder.raw_decode(text, start) |
| 563 | values.append(value) |
| 564 | idx = end |
| 565 | except json.JSONDecodeError: |
| 566 | idx = start + 1 |
| 567 | if len(values) >= 8: |
| 568 | break |
| 569 | return values |
| 570 | |
| 571 | |
| 572 | def _posts_from_json_value(value: Any, *, depth: int = 0) -> List[Dict[str, Any]]: |
| 573 | if depth > 4: |
| 574 | return [] |
| 575 | if isinstance(value, list): |
| 576 | posts = [] |
| 577 | for item in value: |
| 578 | fields = _fields_from_json_obj(item) |
| 579 | if fields: |
| 580 | posts.append(fields) |
| 581 | return posts |
| 582 | if isinstance(value, str): |
| 583 | for nested in _extract_json_values(value): |
| 584 | posts = _posts_from_json_value(nested, depth=depth + 1) |
| 585 | if posts: |
| 586 | return posts |
| 587 | return [] |
| 588 | if not isinstance(value, dict): |
| 589 | return [] |
| 590 | direct = _fields_from_json_obj(value) |
| 591 | if direct: |
| 592 | return [direct] |
| 593 | for key in ("posts", "items", "results", "data", "tweets"): |
| 594 | if key in value: |
| 595 | posts = _posts_from_json_value(value[key], depth=depth + 1) |
| 596 | if posts: |
| 597 | return posts |
| 598 | for key in ("content", "message", "result", "text", "output"): |
| 599 | inner = value.get(key) |
| 600 | if isinstance(inner, (str, list, dict)): |
| 601 | posts = _posts_from_json_value(inner, depth=depth + 1) |
| 602 | if posts: |
| 603 | return posts |
| 604 | return [] |
| 605 | |
| 606 | |
| 607 | def _fields_from_json_text(text: str) -> List[Dict[str, Any]]: |
| 608 | for value in _extract_json_values(text): |
| 609 | posts = _posts_from_json_value(value) |
| 610 | if posts: |
| 611 | return posts |
| 612 | return [] |
| 613 | |
| 614 | |
| 615 | def _fields_from_blocks(text: str) -> List[Dict[str, Any]]: |
| 616 | raw: List[Dict[str, Any]] = [] |
| 617 | for block in _split_blocks(text): |
| 618 | fields: Dict[str, Any] = {} |
| 619 | for line in block.splitlines(): |
| 620 | match = _FIELD_LINE.match(line) |
| 621 | if not match: |
| 622 | continue |
| 623 | key = _FIELD_ALIASES.get(match.group(1).strip().lower()) |
| 624 | if key and key not in fields: |
| 625 | value = match.group(2).strip() |
| 626 | # Field values arrive with varying markdown decoration |
| 627 | # (`- **id:** 123`), so strip emphasis and code marks. |
| 628 | value = value.strip("*").strip().strip("`").strip() |
| 629 | fields[key] = value |
| 630 | if fields.get("post_id"): |
| 631 | raw.append(fields) |
| 632 | return raw |
| 633 | |
| 634 | |
| 635 | def _split_blocks(text: str) -> List[str]: |
| 636 | """Split the model's prose into per-post blocks. |
| 637 | |
| 638 | Keyed on the post-id field starting a new record rather than on any |
| 639 | heading style, because the narration around the blocks varies run to run. |
| 640 | """ |
| 641 | blocks: List[str] = [] |
| 642 | current: List[str] = [] |
| 643 | for line in (text or "").splitlines(): |
| 644 | match = _FIELD_LINE.match(line) |
| 645 | key = _FIELD_ALIASES.get(match.group(1).strip().lower()) if match else None |
| 646 | if key == "post_id" and current: |
| 647 | blocks.append("\n".join(current)) |
| 648 | current = [] |
| 649 | if match or current: |
| 650 | current.append(line) |
| 651 | if current: |
| 652 | blocks.append("\n".join(current)) |
| 653 | return blocks |
| 654 | |
| 655 | |
| 656 | def parse_x_response( |
| 657 | response: Dict[str, Any], |
| 658 | topic: str = "", |
| 659 | from_date: str = "", |
| 660 | to_date: str = "", |
| 661 | ) -> List[Dict[str, Any]]: |
| 662 | """Parse a grok response into normalized X item dicts. |
| 663 | |
| 664 | Total: returns [] on error rather than raising. |
| 665 | """ |
| 666 | if not isinstance(response, dict): |
| 667 | return [] |
| 668 | if response.get("error"): |
| 669 | _log(f"error: {response['error']}") |
| 670 | return [] |
| 671 | |
| 672 | text = response.get("text") or "" |
| 673 | raw = _fields_from_json_text(text) |
| 674 | if not raw: |
| 675 | probe = text |
| 676 | try: |
| 677 | loaded = json.loads(text) |
| 678 | except (json.JSONDecodeError, TypeError): |
| 679 | loaded = None |
| 680 | if isinstance(loaded, dict): |
| 681 | for key in ("content", "message", "result", "text", "output"): |
| 682 | inner = loaded.get(key) |
| 683 | if isinstance(inner, str) and inner.strip(): |
| 684 | probe = inner |
| 685 | raw = _fields_from_json_text(probe) |
| 686 | break |
| 687 | if not raw: |
| 688 | raw = _fields_from_blocks(probe) |
| 689 | |
| 690 | kept, reason = _validate_items(raw, from_date, to_date) if from_date else (raw, "") |
| 691 | if reason: |
| 692 | _log(f"rejected response: {reason}") |
| 693 | _LAST_REJECTION["reason"] = reason |
| 694 | return [] |
| 695 | _LAST_REJECTION["reason"] = "" |
| 696 | |
| 697 | items: List[Dict[str, Any]] = [] |
| 698 | seen_ids = set() |
| 699 | for index, fields in enumerate(kept, start=1): |
| 700 | post_id = str(fields.get("post_id") or "").strip() |
| 701 | if post_id in seen_ids: |
| 702 | continue |
| 703 | seen_ids.add(post_id) |
| 704 | handle = _clean_handle(fields.get("author_handle")) |
| 705 | if not handle: |
| 706 | author = str(fields.get("author") or "") |
| 707 | match = re.search(r"@([A-Za-z0-9_]{1,15})", author) |
| 708 | handle = match.group(1) if match else "" |
| 709 | if not handle: |
| 710 | continue |
| 711 | text = str(fields.get("text") or "").strip()[:500] |
| 712 | engagement = { |
| 713 | "likes": _as_int(str(fields.get("likes", ""))), |
| 714 | "reposts": _as_int(str(fields.get("reposts", ""))), |
| 715 | "replies": _as_int(str(fields.get("replies", ""))), |
| 716 | "quotes": _as_int(str(fields.get("quotes", ""))), |
| 717 | } |
| 718 | items.append({ |
| 719 | "id": f"GK{index}", |
| 720 | "text": text, |
| 721 | "url": f"https://x.com/{handle}/status/{post_id}", |
| 722 | "author_handle": handle, |
| 723 | "date": _parse_date(str(fields.get("created_at", ""))), |
| 724 | "engagement": engagement if any(v is not None for v in engagement.values()) else None, |
| 725 | "why_relevant": "", |
| 726 | "relevance": _compute_relevance(topic, text) if topic else 0.7, |
| 727 | }) |
| 728 | return items |
| 729 | |
| 730 | |
| 731 | # --- invocation ------------------------------------------------------------ |
| 732 | |
| 733 | _PROMPT = """Use {tool} with query '{query}', mode Top, limit {limit}. |
| 734 | |
| 735 | Report every post the tool returned, one block per post, using exactly these |
| 736 | field labels on their own lines: |
| 737 | |
| 738 | id: <numeric post id> |
| 739 | handle: <author handle without @> |
| 740 | created_at: <post timestamp> |
| 741 | likes: <number> |
| 742 | reposts: <number> |
| 743 | replies: <number> |
| 744 | quotes: <number> |
| 745 | text: <full post text on one line> |
| 746 | |
| 747 | Report only posts the tool actually returned. If the tool returned nothing or |
| 748 | could not run, say so plainly and report no post blocks. Do not supply posts |
| 749 | from your own knowledge.""" |
| 750 | |
| 751 | |
| 752 | def is_auth_revoked_error(error: str) -> bool: |
| 753 | """True when the error indicates the Grok session was revoked mid-run. |
| 754 | |
| 755 | Distinct from "never signed in": the user may have had a working session |
| 756 | that expired or was revoked (e.g., OIDC refresh returned invalid_grant). |
| 757 | """ |
| 758 | if not error: |
| 759 | return False |
| 760 | text = error.lower() |
| 761 | return any(marker in text for marker in _AUTH_REVOKED_MARKERS) |
| 762 | |
| 763 | |
| 764 | def classify_run_failure(detail: str) -> str: |
| 765 | """Classify a grok run failure into a health state. |
| 766 | |
| 767 | Used by the pipeline to report typed outcomes (AUTH_FAILED vs generic |
| 768 | ERROR) so doctor and the host can surface the right fix. |
| 769 | """ |
| 770 | from . import health |
| 771 | |
| 772 | if not detail: |
| 773 | return health.ERROR |
| 774 | text = detail.lower() |
| 775 | if any(marker in text for marker in _AUTH_REVOKED_MARKERS): |
| 776 | return health.AUTH_FAILED |
| 777 | if "timed out" in text or "timeout" in text: |
| 778 | return health.TIMEOUT |
| 779 | return health.ERROR |
| 780 | |
| 781 | |
| 782 | def _invoke(prompt: str, timeout: int) -> Dict[str, Any]: |
| 783 | """Run `grok` once. Never raises; every failure returns {'error': str}. |
| 784 | |
| 785 | When the error indicates auth revocation (refresh token rejected, not |
| 786 | signed in, etc.), the response also carries 'auth_revoked': True so |
| 787 | callers can fall back without retrying grok. |
| 788 | """ |
| 789 | binary = binary_path() |
| 790 | if binary is None: |
| 791 | return {"error": "grok CLI not found on PATH"} |
| 792 | try: |
| 793 | with tempfile.TemporaryDirectory(prefix="last30days-grok-") as workdir: |
| 794 | child_home = _stage_child_home(workdir) |
| 795 | result = subprocess.run( |
| 796 | [ |
| 797 | binary, |
| 798 | "-p", |
| 799 | prompt, |
| 800 | "--permission-mode", |
| 801 | "bypassPermissions", |
| 802 | "--output-format", |
| 803 | "json", |
| 804 | ], |
| 805 | capture_output=True, |
| 806 | text=True, |
| 807 | # X posts are not cp1252. Without this, text=True decodes the CLI's |
| 808 | # stdout with the locale codec and a reader thread dies on the first |
| 809 | # emoji or smart quote, which surfaces as "no items parsed" rather |
| 810 | # than an error. Matches the auth-store read above. |
| 811 | encoding="utf-8", |
| 812 | errors="replace", |
| 813 | timeout=timeout, |
| 814 | cwd=workdir, |
| 815 | env=_subprocess_env(child_home), |
| 816 | ) |
| 817 | except FileNotFoundError: |
| 818 | return {"error": "grok CLI not found on PATH"} |
| 819 | except subprocess.TimeoutExpired: |
| 820 | return {"error": f"grok CLI timed out after {timeout}s"} |
| 821 | except OSError as exc: |
| 822 | return {"error": f"{type(exc).__name__}: {exc}"} |
| 823 | except Exception as exc: # noqa: BLE001 - search_x must never raise |
| 824 | return {"error": f"{type(exc).__name__}: {exc}"} |
| 825 | |
| 826 | if result.returncode != 0: |
| 827 | detail = (result.stderr or result.stdout or "").strip()[:300] |
| 828 | error_msg = f"grok CLI exited {result.returncode}: {detail}" |
| 829 | response: Dict[str, Any] = {"error": error_msg} |
| 830 | if is_auth_revoked_error(detail): |
| 831 | response["auth_revoked"] = True |
| 832 | return response |
| 833 | return {"text": result.stdout or ""} |
| 834 | |
| 835 | |
| 836 | def _run_query( |
| 837 | query: str, |
| 838 | from_date: str, |
| 839 | to_date: str, |
| 840 | *, |
| 841 | tool: str = "x_keyword_search", |
| 842 | limit: int = _MAX_LIMIT_PER_CALL, |
| 843 | depth: str = "default", |
| 844 | attempts: int = 2, |
| 845 | relevance_topic: str = "", |
| 846 | deadline: Optional[float] = None, |
| 847 | ) -> Tuple[List[Dict[str, Any]], str, bool]: |
| 848 | """Run one query, retrying only when the response looks fabricated. |
| 849 | |
| 850 | A clean empty result is NOT retried: an empty window is a common, correct |
| 851 | outcome (especially for the mention lane on a low-profile handle and for |
| 852 | the name lane's engagement floor), and re-issuing a byte-identical prompt |
| 853 | doubles latency and Grok-plan spend to get the same answer. |
| 854 | |
| 855 | Returns (items, error, auth_revoked). When auth_revoked is True, the caller |
| 856 | should not retry grok in this run. |
| 857 | """ |
| 858 | timeout = _TIMEOUT_SECONDS.get(depth, _TIMEOUT_SECONDS["default"]) |
| 859 | prompt = _PROMPT.format(tool=tool, query=query, limit=min(limit, _MAX_LIMIT_PER_CALL)) |
| 860 | last_error = "" |
| 861 | for attempt in range(1, attempts + 1): |
| 862 | if deadline is not None: |
| 863 | remaining = deadline - time.monotonic() |
| 864 | if remaining < _MIN_USEFUL_CALL_SECONDS: |
| 865 | return [], last_error or "X lane budget exhausted", False |
| 866 | timeout = min(timeout, int(remaining)) |
| 867 | _log(f"searching: {query}" + (f" (attempt {attempt})" if attempt > 1 else "")) |
| 868 | response = _invoke(prompt, timeout) |
| 869 | if response.get("error"): |
| 870 | last_error = response["error"] |
| 871 | if response.get("auth_revoked"): |
| 872 | return [], last_error, True |
| 873 | continue |
| 874 | items = parse_x_response( |
| 875 | response, |
| 876 | topic=relevance_topic or query, |
| 877 | from_date=from_date, |
| 878 | to_date=to_date, |
| 879 | ) |
| 880 | if items: |
| 881 | return items, "", False |
| 882 | reason = _LAST_REJECTION.get("reason", "") |
| 883 | if not any(marker in reason for marker in _RETRYABLE_REJECTIONS): |
| 884 | return [], "", False |
| 885 | last_error = reason or "no verified in-window posts returned" |
| 886 | return [], last_error, False |
| 887 | |
| 888 | |
| 889 | def search_x( |
| 890 | topic: str, |
| 891 | from_date: str, |
| 892 | to_date: str, |
| 893 | depth: str = "default", |
| 894 | ) -> Dict[str, Any]: |
| 895 | """Search X for a topic, fanning out to reach the depth's target count. |
| 896 | |
| 897 | The underlying tool caps each call at 10 posts, so depth is achieved across |
| 898 | calls. Without this, grok returned 10 posts at every depth while sitting |
| 899 | ahead of bird in the chain -- silently downgrading a `--deep` run from 60 |
| 900 | posts to 10. |
| 901 | |
| 902 | Returns {'items': [...]}; 'error' is set only for an actual invocation |
| 903 | failure. A completed run that found nothing returns an empty list with no |
| 904 | error, matching bird and xquik -- reporting "no results" as a hard failure |
| 905 | would make an empty window look like a broken backend. |
| 906 | |
| 907 | When the Grok session is revoked mid-run (refresh token rejected), |
| 908 | 'auth_revoked': True is set so the pipeline can fall back without retrying |
| 909 | grok and can surface the correct fix to the user. |
| 910 | """ |
| 911 | target = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 912 | calls = max(1, min(_MAX_FANOUT_CALLS, -(-target // _MAX_LIMIT_PER_CALL))) |
| 913 | collected: List[Dict[str, Any]] = [] |
| 914 | seen: set = set() |
| 915 | last_error = "" |
| 916 | invocation_failed = False |
| 917 | auth_revoked = False |
| 918 | for mode_query in _fanout_queries(topic, from_date, to_date, calls): |
| 919 | items, error, revoked = _run_query(mode_query, from_date, to_date, depth=depth, |
| 920 | relevance_topic=topic) |
| 921 | if revoked: |
| 922 | auth_revoked = True |
| 923 | last_error = error or "Grok session expired or was revoked" |
| 924 | break |
| 925 | if error and not items: |
| 926 | last_error = error |
| 927 | if "not found" in error or "timed out" in error or "exited" in error: |
| 928 | invocation_failed = True |
| 929 | for item in items: |
| 930 | key = item["url"] |
| 931 | if key not in seen: |
| 932 | seen.add(key) |
| 933 | collected.append(item) |
| 934 | if len(collected) >= target: |
| 935 | break |
| 936 | for index, item in enumerate(collected, start=1): |
| 937 | item["id"] = f"GK{index}" |
| 938 | if collected: |
| 939 | result: Dict[str, Any] = {"items": collected[:target]} |
| 940 | if auth_revoked: |
| 941 | result["auth_revoked"] = True |
| 942 | return result |
| 943 | if auth_revoked: |
| 944 | return {"items": [], "error": last_error, "auth_revoked": True} |
| 945 | if invocation_failed: |
| 946 | return {"items": [], "error": last_error} |
| 947 | return {"items": []} |
| 948 | |
| 949 | |
| 950 | def search_handles( |
| 951 | handles: List[str], |
| 952 | topic: str, |
| 953 | from_date: str, |
| 954 | to_date: str, |
| 955 | *, |
| 956 | count_per: int = 8, |
| 957 | deadline: Optional[float] = None, |
| 958 | and_topic: bool = False, |
| 959 | ) -> Tuple[List[Dict[str, Any]], bool]: |
| 960 | """BY lane: posts authored by each handle. |
| 961 | |
| 962 | ``topic`` is used for relevance ranking only and is never ANDed into the |
| 963 | query by default -- doing so was a prior defect that emptied the lane |
| 964 | (person posts omit their own name). |
| 965 | |
| 966 | When ``and_topic=True``, the topic IS ANDed into the query (e.g., |
| 967 | ``from:handle Rome``) to ensure extracted handles demonstrate on-topic |
| 968 | content. This prevents off-topic timelines from filling the X budget. |
| 969 | |
| 970 | Returns (items, auth_revoked) so the pipeline can record AUTH_FAILED. |
| 971 | """ |
| 972 | collected: List[Dict[str, Any]] = [] |
| 973 | auth_revoked = False |
| 974 | for handle in handles: |
| 975 | if deadline is not None and time.monotonic() >= deadline: |
| 976 | _log("lane budget exhausted; skipping remaining handles") |
| 977 | break |
| 978 | clean = _clean_handle(handle) |
| 979 | if not clean: |
| 980 | continue |
| 981 | # AND topic only when explicitly requested (extracted handles) |
| 982 | if and_topic and topic: |
| 983 | query = f"from:{clean} {topic} since:{from_date} until:{to_date}" |
| 984 | else: |
| 985 | query = f"from:{clean} since:{from_date} until:{to_date}" |
| 986 | items, _, revoked = _run_query( |
| 987 | query, |
| 988 | from_date, to_date, limit=count_per, relevance_topic=topic, |
| 989 | attempts=1, deadline=deadline, |
| 990 | ) |
| 991 | if revoked: |
| 992 | _log("Grok session revoked; stopping lane") |
| 993 | auth_revoked = True |
| 994 | break |
| 995 | collected.extend( |
| 996 | i for i in items |
| 997 | if i["author_handle"].lower() == clean.lower() |
| 998 | ) |
| 999 | return collected, auth_revoked |
| 1000 | |
| 1001 | |
| 1002 | def search_mentions( |
| 1003 | handles: List[str], |
| 1004 | from_date: str, |
| 1005 | to_date: str, |
| 1006 | *, |
| 1007 | topic: str = "", |
| 1008 | count_per: int = 5, |
| 1009 | deadline: Optional[float] = None, |
| 1010 | ) -> Tuple[List[Dict[str, Any]], bool]: |
| 1011 | """ABOUT lane (mention form): posts @-mentioning each handle. |
| 1012 | |
| 1013 | Returns (items, auth_revoked) so the pipeline can record AUTH_FAILED. |
| 1014 | """ |
| 1015 | collected: List[Dict[str, Any]] = [] |
| 1016 | auth_revoked = False |
| 1017 | for handle in handles: |
| 1018 | if deadline is not None and time.monotonic() >= deadline: |
| 1019 | _log("lane budget exhausted; skipping remaining handles") |
| 1020 | break |
| 1021 | clean = _clean_handle(handle) |
| 1022 | if not clean: |
| 1023 | continue |
| 1024 | items, _, revoked = _run_query( |
| 1025 | f"@{clean} -from:{clean} since:{from_date} until:{to_date}", |
| 1026 | from_date, to_date, limit=count_per, relevance_topic=topic, |
| 1027 | attempts=1, deadline=deadline, |
| 1028 | ) |
| 1029 | if revoked: |
| 1030 | _log("Grok session revoked; stopping lane") |
| 1031 | auth_revoked = True |
| 1032 | break |
| 1033 | collected.extend( |
| 1034 | i for i in items |
| 1035 | if i["author_handle"].lower() != clean.lower() |
| 1036 | ) |
| 1037 | return collected, auth_revoked |
| 1038 | |
| 1039 | |
| 1040 | def search_name( |
| 1041 | name: str, |
| 1042 | from_date: str, |
| 1043 | to_date: str, |
| 1044 | *, |
| 1045 | exclude_handles: Optional[List[str]] = None, |
| 1046 | count_per: int = 8, |
| 1047 | min_faves: int = 2, |
| 1048 | deadline: Optional[float] = None, |
| 1049 | ) -> Tuple[List[Dict[str, Any]], bool]: |
| 1050 | """ABOUT lane (name form): posts naming the subject in plain text. |
| 1051 | |
| 1052 | Not redundant with the mention lane and not a fallback for it. Most talk |
| 1053 | about a person or company never @-mentions them -- people write "Bentgo |
| 1054 | lunch box from Costco", not "@Bentgo lunch box from Costco". A modest |
| 1055 | engagement floor applies here only, because a bare name query is the |
| 1056 | widest and noisiest of the three lanes. |
| 1057 | |
| 1058 | Returns (items, auth_revoked) so the pipeline can record AUTH_FAILED. |
| 1059 | """ |
| 1060 | name = (name or "").strip() |
| 1061 | if not name: |
| 1062 | return [], False |
| 1063 | if name.count('"') % 2: |
| 1064 | name = name.replace('"', " ").strip() |
| 1065 | phrase = f'"{name}"' if " " in name else name |
| 1066 | excludes = " ".join( |
| 1067 | f"-from:{clean}" |
| 1068 | for clean in (_clean_handle(h) for h in (exclude_handles or [])) |
| 1069 | if clean |
| 1070 | ) |
| 1071 | query = " ".join( |
| 1072 | part for part in |
| 1073 | [phrase, excludes, f"min_faves:{min_faves}", f"since:{from_date}", f"until:{to_date}"] |
| 1074 | if part |
| 1075 | ) |
| 1076 | items, _, revoked = _run_query( |
| 1077 | query, from_date, to_date, limit=count_per, attempts=1, deadline=deadline, |
| 1078 | ) |
| 1079 | if revoked: |
| 1080 | _log("Grok session revoked") |
| 1081 | blocked = {c.lower() for c in (_clean_handle(h) for h in (exclude_handles or [])) if c} |
| 1082 | return [i for i in items if i["author_handle"].lower() not in blocked], revoked |
| 1083 |