| 1 | """Threads keyword search via ScrapeCreators API for /last30days. |
| 2 | |
| 3 | Uses ScrapeCreators REST API to search Threads by keyword, extracting |
| 4 | engagement metrics (likes, replies) from short text posts. |
| 5 | |
| 6 | Requires SCRAPECREATORS_API_KEY in config. Opt-in source via INCLUDE_SOURCES. |
| 7 | API docs: https://scrapecreators.com/docs |
| 8 | """ |
| 9 | |
| 10 | import math |
| 11 | import re |
| 12 | from typing import Any, Dict, List, Optional |
| 13 | |
| 14 | from . import dates, http, log |
| 15 | from .relevance import token_overlap_relevance as _compute_relevance |
| 16 | |
| 17 | SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/threads" |
| 18 | |
| 19 | # Depth configurations: how many results to fetch |
| 20 | DEPTH_CONFIG = { |
| 21 | "quick": {"results": 10}, |
| 22 | "default": {"results": 20}, |
| 23 | "deep": {"results": 40}, |
| 24 | } |
| 25 | |
| 26 | |
| 27 | def _log(msg: str): |
| 28 | log.source_log("Threads", msg, tty_only=False) |
| 29 | |
| 30 | |
| 31 | def _extract_core_subject(topic: str) -> str: |
| 32 | """Extract core subject from verbose query for Threads search. |
| 33 | |
| 34 | The ScrapeCreators Threads keyword endpoint only returns hits for short |
| 35 | (1-2 word) queries; 3+ words or leaked boolean operators (the planner |
| 36 | emits "A OR B") return zero. Strip boolean operators and cap to the two |
| 37 | most salient words. |
| 38 | """ |
| 39 | from .query import SOCIAL_NOISE, extract_core_subject |
| 40 | core = extract_core_subject(topic, noise=SOCIAL_NOISE, max_words=2) |
| 41 | return " ".join(core.rstrip("?!.").split()[:2]) |
| 42 | |
| 43 | |
| 44 | |
| 45 | def _parse_date(item: Dict[str, Any]) -> Optional[str]: |
| 46 | """Parse date from Threads item to YYYY-MM-DD. |
| 47 | |
| 48 | Tries common timestamp fields in order: taken_at and create_time |
| 49 | (unix timestamps in Meta APIs), then created_at, published_at, and |
| 50 | date (ISO 8601 strings). dates.parse_date() handles both. |
| 51 | """ |
| 52 | for key in ("taken_at", "create_time", "created_at", "published_at", "date"): |
| 53 | val = item.get(key) |
| 54 | if val is None: |
| 55 | continue |
| 56 | dt = dates.parse_date(str(val)) |
| 57 | if dt: |
| 58 | return dt.strftime("%Y-%m-%d") |
| 59 | return None |
| 60 | |
| 61 | |
| 62 | def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]: |
| 63 | """Parse raw Threads items into normalized dicts.""" |
| 64 | items = [] |
| 65 | for i, raw in enumerate(raw_items): |
| 66 | post_id = str( |
| 67 | raw.get("id") |
| 68 | or raw.get("pk") |
| 69 | or raw.get("code") |
| 70 | or f"TH{i + 1}" |
| 71 | ) |
| 72 | text = raw.get("text") or raw.get("caption") or raw.get("content") or "" |
| 73 | if isinstance(text, dict): |
| 74 | text = text.get("text", "") |
| 75 | |
| 76 | # Author extraction |
| 77 | user = raw.get("user") or raw.get("author") or {} |
| 78 | if isinstance(user, dict): |
| 79 | handle = user.get("username") or user.get("handle") or "" |
| 80 | display_name = user.get("full_name") or user.get("displayName") or handle |
| 81 | elif isinstance(user, str): |
| 82 | handle = user |
| 83 | display_name = user |
| 84 | else: |
| 85 | handle = "" |
| 86 | display_name = "" |
| 87 | |
| 88 | # Engagement metrics |
| 89 | likes = raw.get("like_count") or raw.get("likes") or 0 |
| 90 | replies = raw.get("reply_count") or raw.get("replies") or 0 |
| 91 | reposts = raw.get("repost_count") or raw.get("reposts") or 0 |
| 92 | quotes = raw.get("quote_count") or raw.get("quotes") or 0 |
| 93 | |
| 94 | date_str = _parse_date(raw) |
| 95 | |
| 96 | # Build URL |
| 97 | code = raw.get("code") or raw.get("shortcode") or "" |
| 98 | url = raw.get("url") or raw.get("share_url") or "" |
| 99 | if not url and code: |
| 100 | url = f"https://www.threads.net/post/{code}" |
| 101 | elif not url and handle and post_id: |
| 102 | url = f"https://www.threads.net/@{handle}/post/{post_id}" |
| 103 | |
| 104 | # Relevance: position-based + engagement boost (similar to bluesky) |
| 105 | rank_score = max(0.3, 1.0 - (i * 0.02)) |
| 106 | engagement_boost = min(0.2, math.log1p(likes + reposts) / 40) |
| 107 | text_relevance = _compute_relevance(core_topic, text) |
| 108 | relevance = min(1.0, text_relevance * 0.5 + rank_score * 0.3 + engagement_boost + 0.1) |
| 109 | |
| 110 | items.append({ |
| 111 | "id": post_id, |
| 112 | "handle": handle, |
| 113 | "display_name": display_name, |
| 114 | "text": text, |
| 115 | "url": url, |
| 116 | "date": date_str, |
| 117 | "engagement": { |
| 118 | "likes": likes, |
| 119 | "replies": replies, |
| 120 | "reposts": reposts, |
| 121 | "quotes": quotes, |
| 122 | }, |
| 123 | "relevance": round(relevance, 2), |
| 124 | "why_relevant": f"Threads: @{handle}: {text[:60]}" if text else f"Threads: {handle}", |
| 125 | }) |
| 126 | return items |
| 127 | |
| 128 | |
| 129 | def search_threads( |
| 130 | topic: str, |
| 131 | from_date: str, |
| 132 | to_date: str, |
| 133 | depth: str = "default", |
| 134 | token: str = None, |
| 135 | ) -> Dict[str, Any]: |
| 136 | """Search Threads via ScrapeCreators API. |
| 137 | |
| 138 | Args: |
| 139 | topic: Search topic |
| 140 | from_date: Start date (YYYY-MM-DD) |
| 141 | to_date: End date (YYYY-MM-DD) |
| 142 | depth: 'quick', 'default', or 'deep' |
| 143 | token: ScrapeCreators API key |
| 144 | |
| 145 | Returns: |
| 146 | Dict with 'items' list and optional 'error'. |
| 147 | """ |
| 148 | if not token: |
| 149 | return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"} |
| 150 | |
| 151 | config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 152 | core_topic = _extract_core_subject(topic) |
| 153 | |
| 154 | _log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})") |
| 155 | |
| 156 | try: |
| 157 | data = http.get( |
| 158 | f"{SCRAPECREATORS_BASE}/search", |
| 159 | params={"query": core_topic}, |
| 160 | headers=http.scrapecreators_headers(token), |
| 161 | timeout=30, |
| 162 | retries=2, |
| 163 | ) |
| 164 | except Exception as e: |
| 165 | _log(f"ScrapeCreators error: {e}") |
| 166 | return {"items": [], "error": f"{type(e).__name__}: {e}"} |
| 167 | |
| 168 | # Extract items from response (try common SC response shapes) |
| 169 | raw_items = ( |
| 170 | data.get("items") |
| 171 | or data.get("data") |
| 172 | or data.get("threads") |
| 173 | or data.get("posts") |
| 174 | or data.get("search_results") |
| 175 | or [] |
| 176 | ) |
| 177 | |
| 178 | # Limit to configured count |
| 179 | raw_items = raw_items[:config["results"]] |
| 180 | |
| 181 | # Parse items |
| 182 | items = _parse_items(raw_items, core_topic) |
| 183 | |
| 184 | # Date filter |
| 185 | in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date] |
| 186 | out_of_range = len(items) - len(in_range) |
| 187 | if in_range: |
| 188 | items = in_range |
| 189 | if out_of_range: |
| 190 | _log(f"Filtered {out_of_range} posts outside date range") |
| 191 | else: |
| 192 | _log(f"No posts within date range, keeping all {len(items)}") |
| 193 | |
| 194 | # Sort by engagement (likes) descending |
| 195 | items.sort(key=lambda x: x["engagement"]["likes"], reverse=True) |
| 196 | |
| 197 | _log(f"Found {len(items)} Threads posts") |
| 198 | return {"items": items} |
| 199 | |
| 200 | |
| 201 | def parse_threads_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: |
| 202 | """Parse Threads search response to normalized format. |
| 203 | |
| 204 | Returns: |
| 205 | List of item dicts ready for normalization. |
| 206 | """ |
| 207 | return response.get("items", []) |
| 208 |