| 1 | """Scan saved last30days research artifacts into a deterministic library.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import hashlib |
| 6 | import json |
| 7 | import re |
| 8 | import uuid |
| 9 | from dataclasses import dataclass |
| 10 | from datetime import date, datetime, timezone |
| 11 | from pathlib import Path |
| 12 | |
| 13 | |
| 14 | DEFAULT_MEMORY_DIR = Path.home() / "Documents" / "Last30Days" |
| 15 | DEFAULT_BRIEFS_DIR = Path.home() / ".local" / "share" / "last30days" / "briefs" |
| 16 | LIBRARY_ID_FILENAME = ".last30days-library-id" |
| 17 | |
| 18 | _REPORT_TITLE = re.compile(r"^#\s+last30days(?:\s+v[^:]+)?:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE) |
| 19 | _FIRST_TITLE = re.compile(r"^#\s+(.+?)\s*$", re.MULTILINE) |
| 20 | _DATE_RANGE = re.compile( |
| 21 | r"^-\s*Date range:\s*\d{4}-\d{2}-\d{2}\s+to\s+(\d{4}-\d{2}-\d{2})\s*$", |
| 22 | re.MULTILINE | re.IGNORECASE, |
| 23 | ) |
| 24 | _DATED_FILENAME = re.compile(r"-(\d{4}-\d{2}-\d{2})(?:-\d+)?$") |
| 25 | _RANKED_HEADLINE = re.compile(r"^###\s+1[.)]\s+(.+?)\s*$", re.MULTILINE) |
| 26 | _SCORE_SUFFIX = re.compile(r"\s+\(score\s+[^)]*\)\s*$", re.IGNORECASE) |
| 27 | _MARKDOWN_LINK = re.compile(r"\[([^]]+)]\([^)]+\)") |
| 28 | _LIBRARY_ID = re.compile(r"[0-9a-f]{32}") |
| 29 | _GENERATED_BRIEF_NAME = re.compile( |
| 30 | r"[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}-\d{4}-\d{2}-\d{2}\.html" |
| 31 | ) |
| 32 | _PRIVATE_CORPUS_BLOCK = re.compile( |
| 33 | r"<!-- LAST30DAYS_PRIVATE_CORPUS_START -->.*?" |
| 34 | r"<!-- LAST30DAYS_PRIVATE_CORPUS_END -->\s*", |
| 35 | re.DOTALL, |
| 36 | ) |
| 37 | |
| 38 | |
| 39 | @dataclass(frozen=True, slots=True) |
| 40 | class LibraryEntry: |
| 41 | """Metadata and source content for one saved research artifact.""" |
| 42 | |
| 43 | slug: str |
| 44 | topic: str |
| 45 | published_date: date |
| 46 | headline: str |
| 47 | summary: str |
| 48 | source_path: Path |
| 49 | content: str |
| 50 | source_updated_at: datetime |
| 51 | source_format: str = "markdown" |
| 52 | |
| 53 | @property |
| 54 | def entry_id(self) -> str: |
| 55 | return f"urn:last30days:{self.slug}:{self.identity_hash}:{self.published_date.isoformat()}" |
| 56 | |
| 57 | @property |
| 58 | def output_name(self) -> str: |
| 59 | return f"{self.slug}-{self.identity_hash}-{self.published_date.isoformat()}.html" |
| 60 | |
| 61 | @property |
| 62 | def identity_hash(self) -> str: |
| 63 | # Include the source filename stem so per-suffix runs of the same |
| 64 | # topic on the same date (--save-suffix per-client workflow) stay |
| 65 | # distinct entries instead of collapsing to one. |
| 66 | seed = f"{self.topic}\n{self.source_path.stem}" |
| 67 | return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:8] |
| 68 | |
| 69 | |
| 70 | def slugify(value: str) -> str: |
| 71 | slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") |
| 72 | return slug or "last30days" |
| 73 | |
| 74 | |
| 75 | def get_or_create_library_id(memory_dir: Path | str) -> str: |
| 76 | """Return the persisted random namespace for one research library.""" |
| 77 | memory_path = Path(memory_dir).expanduser() |
| 78 | memory_path.mkdir(parents=True, exist_ok=True) |
| 79 | id_path = memory_path / LIBRARY_ID_FILENAME |
| 80 | try: |
| 81 | library_id = id_path.read_text(encoding="utf-8").strip() |
| 82 | except FileNotFoundError: |
| 83 | library_id = uuid.uuid4().hex |
| 84 | try: |
| 85 | with id_path.open("x", encoding="utf-8") as handle: |
| 86 | handle.write(f"{library_id}\n") |
| 87 | except FileExistsError: |
| 88 | library_id = id_path.read_text(encoding="utf-8").strip() |
| 89 | if not _LIBRARY_ID.fullmatch(library_id): |
| 90 | raise ValueError(f"invalid library ID in {id_path}") |
| 91 | return library_id |
| 92 | |
| 93 | |
| 94 | def is_generated_brief_name(name: str) -> bool: |
| 95 | """Return whether a filename has the exact library-renderer output shape.""" |
| 96 | return _GENERATED_BRIEF_NAME.fullmatch(name) is not None |
| 97 | |
| 98 | |
| 99 | def scan_library( |
| 100 | memory_dir: Path | str = DEFAULT_MEMORY_DIR, |
| 101 | briefs_dir: Path | str = DEFAULT_BRIEFS_DIR, |
| 102 | ) -> tuple[list[LibraryEntry], list[str]]: |
| 103 | """Return valid saved entries and notes for files that could not be read. |
| 104 | |
| 105 | Hand-edited and foreign files are tolerated: a generic Markdown heading is |
| 106 | enough to include a file, while unreadable or unrecognizable files are |
| 107 | skipped with a note instead of aborting the entire feed generation. |
| 108 | """ |
| 109 | entries: dict[str, LibraryEntry] = {} |
| 110 | notes: list[str] = [] |
| 111 | memory_path = Path(memory_dir).expanduser() |
| 112 | briefs_path = Path(briefs_dir).expanduser() |
| 113 | |
| 114 | if memory_path.is_dir(): |
| 115 | for path in sorted(memory_path.glob("*.md")): |
| 116 | try: |
| 117 | entry = _parse_markdown(path) |
| 118 | _keep_preferred(entries, entry) |
| 119 | except (OSError, UnicodeError, ValueError) as exc: |
| 120 | notes.append(f"Skipped {path}: {exc}") |
| 121 | continue |
| 122 | |
| 123 | if briefs_path.is_dir(): |
| 124 | for path in sorted(briefs_path.glob("*.json")): |
| 125 | try: |
| 126 | entry = _parse_briefing(path) |
| 127 | _keep_preferred(entries, entry) |
| 128 | except (OSError, UnicodeError, ValueError, json.JSONDecodeError) as exc: |
| 129 | notes.append(f"Skipped {path}: {exc}") |
| 130 | continue |
| 131 | |
| 132 | ordered = sorted( |
| 133 | entries.values(), |
| 134 | key=lambda entry: (entry.published_date, entry.topic.casefold(), entry.source_path.name), |
| 135 | reverse=True, |
| 136 | ) |
| 137 | return ordered, notes |
| 138 | |
| 139 | |
| 140 | def _keep_preferred(entries: dict[str, LibraryEntry], entry: LibraryEntry) -> None: |
| 141 | existing = entries.get(entry.entry_id) |
| 142 | if existing is None or entry.source_updated_at > existing.source_updated_at: |
| 143 | entries[entry.entry_id] = entry |
| 144 | |
| 145 | |
| 146 | def _parse_markdown(path: Path) -> LibraryEntry: |
| 147 | content = path.read_text(encoding="utf-8") |
| 148 | public_content = _PRIVATE_CORPUS_BLOCK.sub("", content) |
| 149 | title_match = _REPORT_TITLE.search(public_content) or _FIRST_TITLE.search(public_content) |
| 150 | if not title_match: |
| 151 | raise ValueError("no Markdown title found") |
| 152 | topic = _clean_inline(title_match.group(1)) |
| 153 | if not topic: |
| 154 | raise ValueError("empty Markdown title") |
| 155 | published_date = _markdown_date(public_content, path) |
| 156 | headline = _markdown_headline(public_content) or topic |
| 157 | summary = _markdown_summary(public_content) or headline |
| 158 | return LibraryEntry( |
| 159 | slug=slugify(topic), |
| 160 | topic=topic, |
| 161 | published_date=published_date, |
| 162 | headline=headline, |
| 163 | summary=summary, |
| 164 | source_path=path, |
| 165 | content=content, |
| 166 | source_updated_at=_source_updated_at(path), |
| 167 | ) |
| 168 | |
| 169 | |
| 170 | def _markdown_date(content: str, path: Path) -> date: |
| 171 | if match := _DATE_RANGE.search(content): |
| 172 | return date.fromisoformat(match.group(1)) |
| 173 | if match := _DATED_FILENAME.search(path.stem): |
| 174 | return date.fromisoformat(match.group(1)) |
| 175 | return datetime.fromtimestamp(path.stat().st_mtime).date() |
| 176 | |
| 177 | |
| 178 | def _markdown_headline(content: str) -> str: |
| 179 | if match := _RANKED_HEADLINE.search(content): |
| 180 | return _clean_inline(_SCORE_SUFFIX.sub("", match.group(1))) |
| 181 | return "" |
| 182 | |
| 183 | |
| 184 | def _markdown_summary(content: str) -> str: |
| 185 | learned = re.search( |
| 186 | r"^##\s+What I learned\s*$\n+(.+?)(?=\n#{1,3}\s|\n---|\Z)", |
| 187 | content, |
| 188 | re.MULTILINE | re.DOTALL | re.IGNORECASE, |
| 189 | ) |
| 190 | if learned: |
| 191 | for paragraph in re.split(r"\n\s*\n", learned.group(1)): |
| 192 | cleaned = _clean_inline(paragraph) |
| 193 | if cleaned: |
| 194 | return cleaned[:500] |
| 195 | evidence = re.search(r"^\s*-\s*Evidence:\s*(.+?)\s*$", content, re.MULTILINE | re.IGNORECASE) |
| 196 | if evidence: |
| 197 | return _clean_inline(evidence.group(1))[:500] |
| 198 | return "" |
| 199 | |
| 200 | |
| 201 | def _parse_briefing(path: Path) -> LibraryEntry: |
| 202 | data = json.loads(path.read_text(encoding="utf-8")) |
| 203 | if not isinstance(data, dict): |
| 204 | raise ValueError("briefing JSON is not an object") |
| 205 | is_weekly = data.get("type") == "weekly" or path.stem.endswith("-weekly") |
| 206 | raw_date = path.stem[:10] if is_weekly else data.get("date") or path.stem[:10] |
| 207 | try: |
| 208 | published_date = date.fromisoformat(str(raw_date)) |
| 209 | except ValueError as exc: |
| 210 | raise ValueError("briefing has no valid date") from exc |
| 211 | topic = "Weekly research briefing" if is_weekly else "Daily research briefing" |
| 212 | top = data.get("top_finding") if isinstance(data.get("top_finding"), dict) else {} |
| 213 | headline = str(top.get("title") or topic) |
| 214 | summary = _briefing_summary(data, headline) |
| 215 | markdown = _briefing_markdown(data, topic, published_date, summary) |
| 216 | return LibraryEntry( |
| 217 | slug=slugify(topic), |
| 218 | topic=topic, |
| 219 | published_date=published_date, |
| 220 | headline=headline, |
| 221 | summary=summary, |
| 222 | source_path=path, |
| 223 | content=markdown, |
| 224 | source_updated_at=_source_updated_at(path), |
| 225 | source_format="json", |
| 226 | ) |
| 227 | |
| 228 | |
| 229 | def _source_updated_at(path: Path) -> datetime: |
| 230 | seconds, nanoseconds = divmod(path.stat().st_mtime_ns, 1_000_000_000) |
| 231 | return datetime.fromtimestamp(seconds, tz=timezone.utc).replace( |
| 232 | microsecond=nanoseconds // 1_000 |
| 233 | ) |
| 234 | |
| 235 | |
| 236 | def _briefing_summary(data: dict[str, object], fallback: str) -> str: |
| 237 | total_new = data.get("total_new") |
| 238 | total_topics = data.get("total_topics") |
| 239 | if total_new is not None and total_topics is not None: |
| 240 | return f"{total_new} new findings across {total_topics} monitored topics. {fallback}"[:500] |
| 241 | topics = data.get("topics") |
| 242 | if isinstance(topics, list): |
| 243 | return f"Updates across {len(topics)} monitored topics. {fallback}"[:500] |
| 244 | return fallback[:500] |
| 245 | |
| 246 | |
| 247 | def _briefing_markdown(data: dict[str, object], topic: str, published_date: date, summary: str) -> str: |
| 248 | lines = [f"# {topic}", "", f"- Date: {published_date.isoformat()}", "", summary] |
| 249 | if data.get("type") == "weekly" and data.get("week_of"): |
| 250 | lines[3:3] = [f"- Week of: {data['week_of']}"] |
| 251 | topics = data.get("topics") |
| 252 | if isinstance(topics, list): |
| 253 | lines.extend(["", "## Topics", ""]) |
| 254 | for item in topics: |
| 255 | if not isinstance(item, dict): |
| 256 | continue |
| 257 | name = str(item.get("name") or "Untitled topic") |
| 258 | count = item.get("new_count", item.get("this_week_count", 0)) |
| 259 | lines.append(f"- **{name}** — {count} new findings") |
| 260 | return "\n".join(lines).strip() + "\n" |
| 261 | |
| 262 | |
| 263 | def _clean_inline(value: str) -> str: |
| 264 | value = _MARKDOWN_LINK.sub(r"\1", value) |
| 265 | value = re.sub(r"^\s*>\s?", "", value) |
| 266 | value = re.sub(r"(?<!\w)(\*\*|__)(?=\S)(.+?)(?<=\S)\1(?!\w)", r"\2", value) |
| 267 | value = re.sub(r"(?<!\w)([*_])(?=\S)(.+?)(?<=\S)\1(?!\w)", r"\2", value) |
| 268 | value = re.sub(r"(?<!\w)`(?=\S)(.+?)(?<=\S)`(?!\w)", r"\1", value) |
| 269 | return re.sub(r"\s+", " ", value).strip() |
| 270 |