| 1 | """xAI API client for X (Twitter) discovery.""" |
| 2 | |
| 3 | import json |
| 4 | import re |
| 5 | import sys |
| 6 | from typing import Any, Dict, List, Optional |
| 7 | |
| 8 | from . import http, log |
| 9 | |
| 10 | |
| 11 | def _safe_text(val) -> str: |
| 12 | """Extract text from string or localized object.""" |
| 13 | if isinstance(val, str): |
| 14 | return val |
| 15 | if isinstance(val, dict): |
| 16 | return str(val.get("text", val.get("en", ""))) |
| 17 | return str(val) if val is not None else "" |
| 18 | |
| 19 | |
| 20 | def _log(msg: str): |
| 21 | log.source_log("xAI", msg, tty_only=False) |
| 22 | |
| 23 | |
| 24 | def _log_error(msg: str): |
| 25 | log.source_log("xAI ERROR", msg, tty_only=False) |
| 26 | |
| 27 | # xAI uses responses endpoint with Agent Tools API |
| 28 | XAI_RESPONSES_URL = "https://api.x.ai/v1/responses" |
| 29 | |
| 30 | # Depth configurations: (min, max) posts to request |
| 31 | DEPTH_CONFIG = { |
| 32 | "quick": (8, 12), |
| 33 | "default": (20, 30), |
| 34 | "deep": (40, 60), |
| 35 | } |
| 36 | |
| 37 | X_SEARCH_PROMPT = """You have access to real-time X (Twitter) data. Search for posts about: {topic} |
| 38 | |
| 39 | Focus on posts from {from_date} to {to_date}. Find {min_items}-{max_items} high-quality, relevant posts. |
| 40 | |
| 41 | IMPORTANT: Return ONLY valid JSON in this exact format, no other text: |
| 42 | {{ |
| 43 | "items": [ |
| 44 | {{ |
| 45 | "text": "Post text content (truncated if long)", |
| 46 | "url": "https://x.com/user/status/...", |
| 47 | "author_handle": "username", |
| 48 | "date": "YYYY-MM-DD or null if unknown", |
| 49 | "engagement": {{ |
| 50 | "likes": 100, |
| 51 | "reposts": 25, |
| 52 | "replies": 15, |
| 53 | "quotes": 5 |
| 54 | }}, |
| 55 | "why_relevant": "Brief explanation of relevance", |
| 56 | "relevance": 0.85 |
| 57 | }} |
| 58 | ] |
| 59 | }} |
| 60 | |
| 61 | Rules: |
| 62 | - relevance is 0.0 to 1.0 (1.0 = highly relevant) |
| 63 | - date must be YYYY-MM-DD format or null |
| 64 | - engagement can be null if unknown |
| 65 | - Include diverse voices/accounts if applicable |
| 66 | - Prefer posts with substantive content, not just links""" |
| 67 | |
| 68 | |
| 69 | def search_x( |
| 70 | api_key: str, |
| 71 | model: str, |
| 72 | topic: str, |
| 73 | from_date: str, |
| 74 | to_date: str, |
| 75 | depth: str = "default", |
| 76 | mock_response: Optional[Dict] = None, |
| 77 | ) -> Dict[str, Any]: |
| 78 | """Search X for relevant posts using xAI API with live search. |
| 79 | |
| 80 | Args: |
| 81 | api_key: xAI API key |
| 82 | model: Model to use |
| 83 | topic: Search topic |
| 84 | from_date: Start date (YYYY-MM-DD) |
| 85 | to_date: End date (YYYY-MM-DD) |
| 86 | depth: Research depth - "quick", "default", or "deep" |
| 87 | mock_response: Mock response for testing |
| 88 | |
| 89 | Returns: |
| 90 | Raw API response |
| 91 | """ |
| 92 | if mock_response is not None: |
| 93 | return mock_response |
| 94 | |
| 95 | min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 96 | |
| 97 | headers = { |
| 98 | "Authorization": f"Bearer {api_key}", |
| 99 | "Content-Type": "application/json", |
| 100 | } |
| 101 | |
| 102 | # Adjust timeout based on depth (generous for API response time) |
| 103 | timeout = 90 if depth == "quick" else 120 if depth == "default" else 180 |
| 104 | |
| 105 | # Use Agent Tools API with x_search tool (native date filtering) |
| 106 | payload = { |
| 107 | "model": model, |
| 108 | "tools": [ |
| 109 | {"type": "x_search", "from_date": from_date, "to_date": to_date} |
| 110 | ], |
| 111 | "input": [ |
| 112 | { |
| 113 | "role": "user", |
| 114 | "content": X_SEARCH_PROMPT.format( |
| 115 | topic=topic, |
| 116 | from_date=from_date, |
| 117 | to_date=to_date, |
| 118 | min_items=min_items, |
| 119 | max_items=max_items, |
| 120 | ), |
| 121 | } |
| 122 | ], |
| 123 | } |
| 124 | |
| 125 | return http.post(XAI_RESPONSES_URL, payload, headers=headers, timeout=timeout) |
| 126 | |
| 127 | |
| 128 | def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: |
| 129 | """Parse xAI response to extract X items. |
| 130 | |
| 131 | Args: |
| 132 | response: Raw API response |
| 133 | |
| 134 | Returns: |
| 135 | List of item dicts |
| 136 | """ |
| 137 | items = [] |
| 138 | |
| 139 | # Check for API errors first |
| 140 | if "error" in response and response["error"]: |
| 141 | error = response["error"] |
| 142 | err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error) |
| 143 | _log_error(f"xAI API error: {err_msg}") |
| 144 | if log.is_debug(): |
| 145 | _log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}") |
| 146 | return items |
| 147 | |
| 148 | # Try to find the output text |
| 149 | output_text = "" |
| 150 | if "output" in response: |
| 151 | output = response["output"] |
| 152 | if isinstance(output, str): |
| 153 | output_text = output |
| 154 | elif isinstance(output, list): |
| 155 | for item in output: |
| 156 | if isinstance(item, dict): |
| 157 | if item.get("type") == "message": |
| 158 | content = item.get("content", []) |
| 159 | for c in content: |
| 160 | if isinstance(c, dict) and c.get("type") == "output_text": |
| 161 | output_text = c.get("text", "") |
| 162 | break |
| 163 | elif "text" in item: |
| 164 | output_text = item["text"] |
| 165 | elif isinstance(item, str): |
| 166 | output_text = item |
| 167 | if output_text: |
| 168 | break |
| 169 | |
| 170 | # Also check for choices (older format) |
| 171 | if not output_text and "choices" in response: |
| 172 | for choice in response["choices"]: |
| 173 | if "message" in choice: |
| 174 | output_text = choice["message"].get("content", "") |
| 175 | break |
| 176 | |
| 177 | if not output_text: |
| 178 | response_preview = str(response)[:200] if response else "(empty)" |
| 179 | raise http.HTTPError( |
| 180 | f"xAI API returned empty response (no output text found; response preview: {response_preview})" |
| 181 | ) |
| 182 | |
| 183 | # Extract JSON from the response |
| 184 | json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text) |
| 185 | if not json_match: |
| 186 | raise http.HTTPError( |
| 187 | f"xAI API returned output without valid JSON items structure (output: {output_text[:200]})" |
| 188 | ) |
| 189 | try: |
| 190 | data = json.loads(json_match.group()) |
| 191 | items = data.get("items", []) |
| 192 | except json.JSONDecodeError: |
| 193 | raise http.HTTPError( |
| 194 | f"xAI API returned valid output but invalid JSON structure (output: {output_text[:200]})" |
| 195 | ) |
| 196 | |
| 197 | # Validate and clean items |
| 198 | clean_items = [] |
| 199 | for i, item in enumerate(items): |
| 200 | if not isinstance(item, dict): |
| 201 | continue |
| 202 | |
| 203 | url = item.get("url", "") |
| 204 | if not url: |
| 205 | continue |
| 206 | |
| 207 | # Parse engagement |
| 208 | engagement = None |
| 209 | eng_raw = item.get("engagement") |
| 210 | if isinstance(eng_raw, dict): |
| 211 | engagement = { |
| 212 | "likes": int(eng_raw["likes"]) if eng_raw.get("likes") is not None else None, |
| 213 | "reposts": int(eng_raw["reposts"]) if eng_raw.get("reposts") is not None else None, |
| 214 | "replies": int(eng_raw["replies"]) if eng_raw.get("replies") is not None else None, |
| 215 | "quotes": int(eng_raw["quotes"]) if eng_raw.get("quotes") is not None else None, |
| 216 | } |
| 217 | |
| 218 | clean_item = { |
| 219 | "id": f"X{i+1}", |
| 220 | "text": _safe_text(item.get("text", "")).strip()[:500], # Truncate long text |
| 221 | "url": url, |
| 222 | "author_handle": _safe_text(item.get("author_handle", "")).strip().lstrip("@"), |
| 223 | "date": item.get("date"), |
| 224 | "engagement": engagement, |
| 225 | "why_relevant": _safe_text(item.get("why_relevant", "")).strip(), |
| 226 | "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))), |
| 227 | } |
| 228 | |
| 229 | # Validate date format |
| 230 | if clean_item["date"]: |
| 231 | if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])): |
| 232 | clean_item["date"] = None |
| 233 | |
| 234 | clean_items.append(clean_item) |
| 235 | |
| 236 | return clean_items |
| 237 |