| 1 | """Digg AI 1000 source for last30days. |
| 2 | |
| 3 | Shells out to ``digg-pp-cli`` (read-only, no auth required) to surface |
| 4 | clustered stories curated from ~1000 high-signal AI accounts on X. Each |
| 5 | cluster carries a published TLDR, a curatorial rank, and a list of X |
| 6 | posts that can be fetched as inline quotes. |
| 7 | |
| 8 | Activation gate: this source is only available when ``digg-pp-cli`` is |
| 9 | on PATH. ``pipeline.available_sources`` checks ``shutil.which`` before |
| 10 | including ``digg`` in the source list. The functions below also detect |
| 11 | the missing-binary case as a defensive fallback. |
| 12 | |
| 13 | Primary path: ``digg-pp-cli search <topic> --since 30d --agent --limit N``. |
| 14 | Optional enrichment: ``digg-pp-cli posts <clusterUrlId> --agent --by rank |
| 15 | --limit M`` for the top K clusters in default/deep depth, attaching the |
| 16 | top-ranked X posts to each cluster's ``posts`` field. |
| 17 | """ |
| 18 | |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import json |
| 22 | import shutil |
| 23 | from datetime import datetime, timedelta, timezone |
| 24 | from typing import Any, Dict, List, Optional |
| 25 | from urllib.parse import urlparse |
| 26 | |
| 27 | from . import log, subproc |
| 28 | from .relevance import token_overlap_relevance |
| 29 | |
| 30 | |
| 31 | CLI_BIN = "digg-pp-cli" |
| 32 | |
| 33 | # Per-depth knobs. |
| 34 | DEPTH_CONFIG = { |
| 35 | "quick": 8, |
| 36 | "default": 20, |
| 37 | "deep": 40, |
| 38 | } |
| 39 | |
| 40 | # How many top-ranked clusters get post enrichment, per depth. Quick mode |
| 41 | # skips enrichment to keep latency low (clusters already carry a TLDR). |
| 42 | ENRICH_CONFIG = { |
| 43 | "quick": 0, |
| 44 | "default": 3, |
| 45 | "deep": 5, |
| 46 | } |
| 47 | |
| 48 | # X posts pulled per enriched cluster. Matches the 5-comment cap used by |
| 49 | # Reddit/HN/YouTube/TikTok/GitHub enrichment. |
| 50 | POSTS_PER_CLUSTER = 5 |
| 51 | |
| 52 | SEARCH_TIMEOUT = 30 |
| 53 | POSTS_TIMEOUT = 15 |
| 54 | |
| 55 | |
| 56 | def _log(msg: str) -> None: |
| 57 | log.source_log("Digg", msg, tty_only=False) |
| 58 | |
| 59 | |
| 60 | def _is_available() -> bool: |
| 61 | """True when the digg-pp-cli binary is on PATH.""" |
| 62 | return shutil.which(CLI_BIN) is not None |
| 63 | |
| 64 | |
| 65 | def _today() -> datetime: |
| 66 | return datetime.now(timezone.utc) |
| 67 | |
| 68 | |
| 69 | def _parse_first_post_age(age: Optional[str], today: Optional[datetime] = None) -> Optional[str]: |
| 70 | """Convert a digg firstPostAge token (e.g. '5d', '17d', '5h', '1w', '1m') |
| 71 | into a YYYY-MM-DD string. Returns None when the value is outside the |
| 72 | last-30-day window or cannot be parsed. |
| 73 | |
| 74 | Digg uses minutes-symbol-collision for 'months' (per agent-context: |
| 75 | 'Nh, Nd, Nw, Nm (e.g. 30d, 1w, 12h, 1m)'), so 'Nm' is months ~30 days. |
| 76 | """ |
| 77 | if not age or not isinstance(age, str): |
| 78 | return None |
| 79 | age = age.strip().lower() |
| 80 | if len(age) < 2: |
| 81 | return None |
| 82 | unit = age[-1] |
| 83 | try: |
| 84 | amount = int(age[:-1]) |
| 85 | except (ValueError, TypeError): |
| 86 | return None |
| 87 | if amount < 0: |
| 88 | return None |
| 89 | |
| 90 | base = today or _today() |
| 91 | |
| 92 | if unit == "h": |
| 93 | delta = timedelta(hours=amount) |
| 94 | elif unit == "d": |
| 95 | delta = timedelta(days=amount) |
| 96 | elif unit == "w": |
| 97 | delta = timedelta(weeks=amount) |
| 98 | elif unit == "m": |
| 99 | delta = timedelta(days=amount * 30) |
| 100 | else: |
| 101 | return None |
| 102 | |
| 103 | if delta > timedelta(days=30): |
| 104 | return None |
| 105 | |
| 106 | point = base - delta |
| 107 | return point.date().isoformat() |
| 108 | |
| 109 | |
| 110 | def _build_search_args(query: str, limit: int) -> List[str]: |
| 111 | return [ |
| 112 | CLI_BIN, |
| 113 | "search", |
| 114 | query, |
| 115 | "--since", |
| 116 | "30d", |
| 117 | "--agent", |
| 118 | "--limit", |
| 119 | str(limit), |
| 120 | ] |
| 121 | |
| 122 | |
| 123 | def _build_posts_args(cluster_url_id: str, posts_per: int) -> List[str]: |
| 124 | return [ |
| 125 | CLI_BIN, |
| 126 | "posts", |
| 127 | cluster_url_id, |
| 128 | "--agent", |
| 129 | "--by", |
| 130 | "rank", |
| 131 | "--limit", |
| 132 | str(posts_per), |
| 133 | ] |
| 134 | |
| 135 | |
| 136 | def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]: |
| 137 | """Invoke digg-pp-cli and parse the JSON envelope. |
| 138 | |
| 139 | Returns ``{"results": [...]}`` on success, ``{"results": [], "error": "..."}`` |
| 140 | on failure. Never raises; the pipeline relies on shape consistency. |
| 141 | """ |
| 142 | if not _is_available(): |
| 143 | return {"results": [], "error": f"{CLI_BIN} not on PATH"} |
| 144 | try: |
| 145 | result = subproc.run_with_timeout(cmd, timeout=timeout) |
| 146 | except subproc.SubprocTimeout as exc: |
| 147 | _log(f"Timeout: {exc}") |
| 148 | return {"results": [], "error": str(exc)} |
| 149 | except FileNotFoundError as exc: |
| 150 | _log(f"Binary missing: {exc}") |
| 151 | return {"results": [], "error": str(exc)} |
| 152 | except OSError as exc: |
| 153 | _log(f"Spawn failed: {exc}") |
| 154 | return {"results": [], "error": str(exc)} |
| 155 | |
| 156 | if result.returncode != 0: |
| 157 | snippet = (result.stderr or "").strip().splitlines()[:1] |
| 158 | first = snippet[0] if snippet else f"exit {result.returncode}" |
| 159 | _log(f"CLI exit {result.returncode}: {first}") |
| 160 | return {"results": [], "error": first} |
| 161 | |
| 162 | stdout = result.stdout or "" |
| 163 | if not stdout.strip(): |
| 164 | return {"results": []} |
| 165 | try: |
| 166 | data = json.loads(stdout) |
| 167 | except json.JSONDecodeError as exc: |
| 168 | _log(f"JSON decode failed: {exc}") |
| 169 | return {"results": [], "error": f"json decode: {exc}"} |
| 170 | |
| 171 | if not isinstance(data, dict): |
| 172 | return {"results": []} |
| 173 | results = data.get("results") |
| 174 | if not isinstance(results, list): |
| 175 | return {"results": []} |
| 176 | return data |
| 177 | |
| 178 | |
| 179 | def search_digg( |
| 180 | topic: str, |
| 181 | from_date: str, |
| 182 | to_date: str, |
| 183 | depth: str = "default", |
| 184 | ) -> Dict[str, Any]: |
| 185 | """Search Digg AI 1000 clusters via digg-pp-cli. |
| 186 | |
| 187 | Args: |
| 188 | topic: search query. |
| 189 | from_date: YYYY-MM-DD start (advisory; --since 30d is the actual filter). |
| 190 | to_date: YYYY-MM-DD end (advisory; same). |
| 191 | depth: 'quick' | 'default' | 'deep'. |
| 192 | |
| 193 | Returns: |
| 194 | Dict with ``results`` list. On failure, ``results`` is empty and an |
| 195 | ``error`` key carries a one-line description. |
| 196 | """ |
| 197 | limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 198 | if not topic or not topic.strip(): |
| 199 | return {"results": []} |
| 200 | cmd = _build_search_args(topic, limit) |
| 201 | _log(f"search '{topic}' (limit={limit}, since=30d)") |
| 202 | response = _run_cli(cmd, timeout=SEARCH_TIMEOUT) |
| 203 | n = len(response.get("results") or []) |
| 204 | _log(f"found {n} clusters") |
| 205 | return response |
| 206 | |
| 207 | |
| 208 | def _build_url(cluster_url_id: str) -> str: |
| 209 | return f"https://di.gg/ai/{cluster_url_id}" |
| 210 | |
| 211 | |
| 212 | def _rank_score(rank: Optional[int]) -> float: |
| 213 | """Convert Digg rank (lower is better, top 50 are notable) into a |
| 214 | positive engagement-style signal in [0, 50]. Anything off the top-50 |
| 215 | leaderboard contributes 0. |
| 216 | """ |
| 217 | if rank is None: |
| 218 | return 0.0 |
| 219 | try: |
| 220 | r = int(rank) |
| 221 | except (TypeError, ValueError): |
| 222 | return 0.0 |
| 223 | if r < 1 or r > 50: |
| 224 | return 0.0 |
| 225 | return float(51 - r) |
| 226 | |
| 227 | |
| 228 | def parse_digg_response( |
| 229 | response: Dict[str, Any], |
| 230 | query: str = "", |
| 231 | ) -> List[Dict[str, Any]]: |
| 232 | """Parse a digg search envelope into normalized item dicts. |
| 233 | |
| 234 | Args: |
| 235 | response: payload from ``search_digg``. |
| 236 | query: original search query, used for token-overlap relevance. |
| 237 | |
| 238 | Returns: |
| 239 | List of dicts ready for ``normalize._normalize_digg``. |
| 240 | """ |
| 241 | raw = response.get("results") if isinstance(response, dict) else None |
| 242 | if not isinstance(raw, list): |
| 243 | return [] |
| 244 | |
| 245 | items: List[Dict[str, Any]] = [] |
| 246 | for i, cluster in enumerate(raw): |
| 247 | if not isinstance(cluster, dict): |
| 248 | continue |
| 249 | cluster_url_id = cluster.get("clusterUrlId") |
| 250 | if not cluster_url_id: |
| 251 | continue |
| 252 | |
| 253 | title = str(cluster.get("title") or "").strip() |
| 254 | tldr = str(cluster.get("tldr") or "").strip() |
| 255 | rank = cluster.get("rank") |
| 256 | post_count = cluster.get("postCount") or 0 |
| 257 | unique_authors = cluster.get("uniqueAuthors") or 0 |
| 258 | first_post_age = cluster.get("firstPostAge") |
| 259 | date_str = _parse_first_post_age(first_post_age) |
| 260 | if date_str is None and first_post_age: |
| 261 | # firstPostAge present but outside 30d -> drop; last30days contract. |
| 262 | continue |
| 263 | |
| 264 | rank_decay = max(0.3, 1.0 - (i * 0.02)) |
| 265 | if query: |
| 266 | content_score = token_overlap_relevance(query, f"{title} {tldr}".strip()) |
| 267 | else: |
| 268 | content_score = 0.5 |
| 269 | rank_boost = min(0.2, _rank_score(rank) / 250.0) |
| 270 | relevance = min(1.0, 0.55 * rank_decay + 0.35 * content_score + rank_boost) |
| 271 | |
| 272 | items.append( |
| 273 | { |
| 274 | "id": str(cluster_url_id), |
| 275 | "title": title or f"Digg cluster {i + 1}", |
| 276 | "url": _build_url(str(cluster_url_id)), |
| 277 | "tldr": tldr, |
| 278 | "author": "", |
| 279 | "date": date_str, |
| 280 | "engagement": { |
| 281 | "postCount": int(post_count) if isinstance(post_count, (int, float)) else 0, |
| 282 | "uniqueAuthors": int(unique_authors) if isinstance(unique_authors, (int, float)) else 0, |
| 283 | "rank": int(rank) if isinstance(rank, (int, float)) else None, |
| 284 | "rank_score": _rank_score(rank), |
| 285 | }, |
| 286 | "first_post_age": first_post_age, |
| 287 | "posts": [], |
| 288 | "relevance": round(relevance, 2), |
| 289 | "why_relevant": ( |
| 290 | f"Digg cluster (rank {rank}, {post_count} posts, {unique_authors} authors)" |
| 291 | if rank is not None |
| 292 | else f"Digg cluster ({post_count} posts, {unique_authors} authors)" |
| 293 | ), |
| 294 | } |
| 295 | ) |
| 296 | |
| 297 | return items |
| 298 | |
| 299 | |
| 300 | def _is_safe_http_url(url: str) -> bool: |
| 301 | """True iff ``url`` parses with an http or https scheme. |
| 302 | |
| 303 | Used to reject upstream-supplied post URLs whose scheme would be |
| 304 | dangerous in a rendered ``<a href>`` (``javascript:``, ``data:``, |
| 305 | ``file:``, ``vbscript:``, ``about:``). |
| 306 | """ |
| 307 | try: |
| 308 | scheme = urlparse(url).scheme.lower() |
| 309 | except ValueError: |
| 310 | return False |
| 311 | return scheme in ("http", "https") |
| 312 | |
| 313 | |
| 314 | def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]: |
| 315 | """Reduce a digg post payload into the small dict render uses. |
| 316 | |
| 317 | We deliberately keep this minimal: an inline quote needs the author |
| 318 | handle, the body, the post type, and the X URL. |
| 319 | """ |
| 320 | if not isinstance(raw_post, dict): |
| 321 | return None |
| 322 | body = str(raw_post.get("body") or "").strip() |
| 323 | if not body: |
| 324 | return None |
| 325 | author = raw_post.get("author") or {} |
| 326 | if not isinstance(author, dict): |
| 327 | author = {} |
| 328 | username = str(author.get("username") or "").strip() |
| 329 | if not username: |
| 330 | return None |
| 331 | x_url = str(raw_post.get("xUrl") or "").strip() |
| 332 | if not x_url: |
| 333 | return None |
| 334 | if not _is_safe_http_url(x_url): |
| 335 | # Security-class drop: an upstream-supplied URL with a dangerous |
| 336 | # scheme. Force tty_only=False so the rejection is visible in |
| 337 | # non-interactive runs (Claude Code), which is the actual attack |
| 338 | # surface — the default tty_only=True would suppress it there. |
| 339 | log.source_log( |
| 340 | "Digg", |
| 341 | f"dropped post with unsafe xUrl scheme: {x_url!r}", |
| 342 | tty_only=False, |
| 343 | ) |
| 344 | return None |
| 345 | return { |
| 346 | "username": username, |
| 347 | "display_name": str(author.get("display_name") or "").strip() or username, |
| 348 | "category": str(author.get("category") or "").strip(), |
| 349 | "rank": author.get("rank"), |
| 350 | "body": body, |
| 351 | "post_type": str(raw_post.get("post_type") or "tweet").strip(), |
| 352 | "x_url": x_url, |
| 353 | "posted_at": raw_post.get("posted_at"), |
| 354 | } |
| 355 | |
| 356 | |
| 357 | def fetch_top_posts(cluster_url_id: str, posts_per: int = POSTS_PER_CLUSTER) -> List[Dict[str, Any]]: |
| 358 | """Fetch top-ranked X posts attached to a cluster. |
| 359 | |
| 360 | Returns an empty list on any failure (timeout, missing cluster, JSON |
| 361 | error). Never raises. |
| 362 | """ |
| 363 | if posts_per <= 0: |
| 364 | return [] |
| 365 | cmd = _build_posts_args(cluster_url_id, posts_per) |
| 366 | response = _run_cli(cmd, timeout=POSTS_TIMEOUT) |
| 367 | raw = response.get("results") or [] |
| 368 | out: List[Dict[str, Any]] = [] |
| 369 | for entry in raw: |
| 370 | post = _parse_post(entry) |
| 371 | if post is not None: |
| 372 | out.append(post) |
| 373 | return out |
| 374 | |
| 375 | |
| 376 | def enrich_with_top_posts( |
| 377 | items: List[Dict[str, Any]], |
| 378 | top_k: int = 3, |
| 379 | posts_per: int = POSTS_PER_CLUSTER, |
| 380 | ) -> List[Dict[str, Any]]: |
| 381 | """Attach top X posts to the first ``top_k`` clusters by Digg rank order. |
| 382 | |
| 383 | Mutates and returns the same list. Items that already have posts, or |
| 384 | whose ``postCount`` is 0, are skipped. |
| 385 | """ |
| 386 | if top_k <= 0 or posts_per <= 0: |
| 387 | return items |
| 388 | enriched = 0 |
| 389 | for item in items: |
| 390 | if enriched >= top_k: |
| 391 | break |
| 392 | if item.get("posts"): |
| 393 | continue |
| 394 | engagement = item.get("engagement") or {} |
| 395 | if not engagement.get("postCount"): |
| 396 | continue |
| 397 | cluster_url_id = item.get("id") |
| 398 | if not cluster_url_id: |
| 399 | continue |
| 400 | posts = fetch_top_posts(str(cluster_url_id), posts_per=posts_per) |
| 401 | item["posts"] = posts |
| 402 | enriched += 1 |
| 403 | if enriched: |
| 404 | _log(f"enriched {enriched} clusters with X posts") |
| 405 | return items |
| 406 | |
| 407 | |
| 408 | def enrich_source_items(items: list, top_k: int = 3, posts_per: int = POSTS_PER_CLUSTER) -> list: |
| 409 | """Attach top X posts to the first ``top_k`` SourceItems that survived dedupe. |
| 410 | |
| 411 | Reads ``metadata['clusterUrlId']`` and writes ``metadata['posts']`` in |
| 412 | place. Skips items that already carry a non-empty ``metadata['posts']``, |
| 413 | items whose engagement ``postCount`` is 0, and items whose source is not |
| 414 | 'digg'. Designed to run from `_finalize_items_by_source` so enrichment |
| 415 | is spent on the items the brief actually shows. |
| 416 | """ |
| 417 | if top_k <= 0 or posts_per <= 0: |
| 418 | return items |
| 419 | enriched = 0 |
| 420 | for item in items: |
| 421 | if enriched >= top_k: |
| 422 | break |
| 423 | if getattr(item, "source", None) != "digg": |
| 424 | continue |
| 425 | metadata = getattr(item, "metadata", None) or {} |
| 426 | if metadata.get("posts"): |
| 427 | continue |
| 428 | engagement = getattr(item, "engagement", None) or {} |
| 429 | if not engagement.get("postCount"): |
| 430 | continue |
| 431 | cluster_url_id = metadata.get("clusterUrlId") or item.item_id |
| 432 | if not cluster_url_id: |
| 433 | continue |
| 434 | posts = fetch_top_posts(str(cluster_url_id), posts_per=posts_per) |
| 435 | if posts: |
| 436 | metadata["posts"] = posts |
| 437 | enriched += 1 |
| 438 | if enriched: |
| 439 | _log(f"post-dedupe enriched {enriched} clusters with X posts") |
| 440 | return items |
| 441 |