| 1 | /** |
| 2 | * content-watch.ts — two daily watchers that catch site drift the mechanical |
| 3 | * facts pipeline misses: |
| 4 | * |
| 5 | * runLinkCheck — pings every external URL referenced in the site copy, |
| 6 | * writes a draft per broken link (4xx/5xx). Stores a |
| 7 | * `linkcheck:last` summary so /admin can show last status. |
| 8 | * |
| 9 | * runSemanticDrift — reads recent CHANGELOG / commits, asks deepseek-v4-flash |
| 10 | * whether any specific claims on the site look out of |
| 11 | * date, writes review-required drafts. |
| 12 | * |
| 13 | * Both surface as drafts in CURATED_KV under `draft:linkcheck:<...>` and |
| 14 | * `draft:semantic-drift:<...>`, picked up by the existing /admin listing. |
| 15 | */ |
| 16 | import { agentChat, draftStorageKey, getDraft, saveDraft, type AgentDraft, type DeepSeekEnv, VOICE_CONSTRAINTS } from "./community-agent"; |
| 17 | |
| 18 | interface KVNamespace { |
| 19 | get(k: string): Promise<string | null>; |
| 20 | put(k: string, v: string, o?: { expirationTtl?: number }): Promise<void>; |
| 21 | list(o?: { prefix?: string; limit?: number }): Promise<{ keys: { name: string }[] }>; |
| 22 | delete(k: string): Promise<void>; |
| 23 | } |
| 24 | |
| 25 | // --- Canonical draft identities --- |
| 26 | // |
| 27 | // Watcher draft IDs are deterministic: a readable slug plus a ~64-bit |
| 28 | // SHA-256 suffix over the finding's full identity, capped at 80 characters. |
| 29 | // Re-running a watcher over an unchanged finding reproduces the same key (so |
| 30 | // the open-draft dedup check skips it); any change to the finding's identity |
| 31 | // produces a new key (so changed findings are drafted again instead of being |
| 32 | // silently swallowed by a truncated-prefix collision). |
| 33 | const WATCH_DRAFT_ID_MAX = 80; |
| 34 | const WATCH_DRAFT_HASH_HEX = 16; // 64 bits of SHA-256 |
| 35 | |
| 36 | async function sha256Hex(input: string): Promise<string> { |
| 37 | const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input)); |
| 38 | return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| 39 | } |
| 40 | |
| 41 | function slugify(input: string): string { |
| 42 | return input.replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").toLowerCase(); |
| 43 | } |
| 44 | |
| 45 | export async function watchDraftId(slugSource: string, identity: string): Promise<string> { |
| 46 | const hash = (await sha256Hex(identity)).slice(0, WATCH_DRAFT_HASH_HEX); |
| 47 | const slug = slugify(slugSource).slice(0, WATCH_DRAFT_ID_MAX - WATCH_DRAFT_HASH_HEX - 1); |
| 48 | return `${slug}-${hash}`; |
| 49 | } |
| 50 | |
| 51 | interface WatchEnv { |
| 52 | CURATED_KV?: KVNamespace; |
| 53 | DEEPSEEK_API_KEY?: string; |
| 54 | DEEPSEEK_BASE_URL?: string; |
| 55 | DEEPSEEK_MODEL?: string; |
| 56 | GITHUB_TOKEN?: string; |
| 57 | } |
| 58 | |
| 59 | function dsEnv(env: WatchEnv): DeepSeekEnv { |
| 60 | return { |
| 61 | baseUrl: env.DEEPSEEK_BASE_URL ?? process.env.DEEPSEEK_BASE_URL, |
| 62 | model: env.DEEPSEEK_MODEL ?? process.env.DEEPSEEK_MODEL, |
| 63 | }; |
| 64 | } |
| 65 | |
| 66 | // --- Link checker --- |
| 67 | |
| 68 | // Targets to probe daily. For registries that block bot HEAD/GET (npm, crates.io) |
| 69 | // we hit the public JSON API instead — same upstream, doesn't 403. |
| 70 | const LINK_TARGETS: { url: string; label: string }[] = [ |
| 71 | { url: "https://github.com/Hmbown/CodeWhale", label: "Main repo" }, |
| 72 | { url: "https://github.com/Hmbown/CodeWhale/issues", label: "Issues" }, |
| 73 | { url: "https://github.com/Hmbown/CodeWhale/pulls", label: "Pull Requests" }, |
| 74 | { url: "https://github.com/Hmbown/CodeWhale/discussions", label: "Discussions" }, |
| 75 | { url: "https://github.com/Hmbown/CodeWhale/releases", label: "Releases" }, |
| 76 | { url: "https://github.com/Hmbown/CodeWhale/blob/main/LICENSE", label: "License file" }, |
| 77 | { url: "https://github.com/Hmbown/CodeWhale/blob/main/CODE_OF_CONDUCT.md", label: "Code of Conduct" }, |
| 78 | { url: "https://github.com/Hmbown/CodeWhale/blob/main/SECURITY.md", label: "Security policy" }, |
| 79 | { url: "https://github.com/Hmbown/CodeWhale/blob/main/CONTRIBUTING.md", label: "Contributing guide" }, |
| 80 | { url: "https://github.com/Hmbown/CodeWhale/blob/main/.github/PULL_REQUEST_TEMPLATE.md", label: "PR template" }, |
| 81 | { url: "https://github.com/Hmbown/homebrew-deepseek-tui", label: "Homebrew tap" }, |
| 82 | { url: "https://github.com/sponsors/Hmbown", label: "Support link (GitHub Sponsors)" }, |
| 83 | { url: "https://buymeacoffee.com/hmbown", label: "Support link (BMC)" }, |
| 84 | { url: "https://registry.npmjs.org/codewhale", label: "npm package (registry API)" }, |
| 85 | // crates.io intentionally not in this list — both their HTML and JSON API return 403 to |
| 86 | // Cloudflare Workers, so the check produces false positives. The crate links on the site |
| 87 | // still work for human users. |
| 88 | ]; |
| 89 | |
| 90 | export interface LinkCheckResult { |
| 91 | url: string; |
| 92 | label: string; |
| 93 | status: number | "error"; |
| 94 | ok: boolean; |
| 95 | ms: number; |
| 96 | } |
| 97 | |
| 98 | async function probe(target: { url: string; label: string }): Promise<LinkCheckResult> { |
| 99 | const start = Date.now(); |
| 100 | try { |
| 101 | // Use HEAD where possible; fall back to GET on 405/403 since some hosts |
| 102 | // (e.g. Cloudflare-protected) reject HEAD. |
| 103 | let r = await fetch(target.url, { method: "HEAD", redirect: "follow" }); |
| 104 | if (r.status === 405 || r.status === 403 || r.status === 404) { |
| 105 | // Some sites return 404 to HEAD but 200 to GET (e.g. NPM) |
| 106 | r = await fetch(target.url, { method: "GET", redirect: "follow" }); |
| 107 | } |
| 108 | return { url: target.url, label: target.label, status: r.status, ok: r.ok, ms: Date.now() - start }; |
| 109 | } catch { |
| 110 | return { url: target.url, label: target.label, status: "error", ok: false, ms: Date.now() - start }; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | export async function runLinkCheck(env: WatchEnv): Promise<{ ok: boolean; checked: number; broken: number; results?: LinkCheckResult[] }> { |
| 115 | if (!env.CURATED_KV) return { ok: false, checked: 0, broken: 0 }; |
| 116 | |
| 117 | const results = await Promise.all(LINK_TARGETS.map(probe)); |
| 118 | const broken = results.filter((r) => !r.ok); |
| 119 | |
| 120 | await env.CURATED_KV.put("linkcheck:last", JSON.stringify({ |
| 121 | at: new Date().toISOString(), |
| 122 | checked: results.length, |
| 123 | broken: broken.length, |
| 124 | results, |
| 125 | }), { expirationTtl: 60 * 60 * 24 * 14 }); |
| 126 | |
| 127 | // Write drafts ONLY for new breakages — dedup on the canonical draft key, |
| 128 | // derived from the full URL identity (the hash suffix keeps long URLs with |
| 129 | // identical slug prefixes distinct). |
| 130 | for (const b of broken) { |
| 131 | const id = await watchDraftId(b.url, `linkcheck\n${b.url}`); |
| 132 | const draft: AgentDraft = { |
| 133 | id, |
| 134 | type: "linkcheck", |
| 135 | targetUrl: b.url, |
| 136 | bodyEn: `**Broken link** (auto-detected by daily watch cron)\n\n- Label: **${b.label}**\n- URL: ${b.url}\n- HTTP status: ${b.status}\n- Latency: ${b.ms}ms\n\nThis URL is referenced in codewhale.net copy. Update the source page or fix the destination.\n\n— drafted by community assistant, pending maintainer review`, |
| 137 | bodyZh: `**链接失效**（每日巡检自动发现）\n\n- 名称：**${b.label}**\n- 地址：${b.url}\n- HTTP 状态：${b.status}\n- 延迟：${b.ms}ms\n\n该地址被 codewhale.net 文案引用，请更新源页面或修复目标。\n\n— 由社区助理草拟，待维护者审阅`, |
| 138 | generatedAt: new Date().toISOString(), |
| 139 | posted: false, |
| 140 | }; |
| 141 | const existing = await getDraft(env.CURATED_KV, draftStorageKey(draft)); |
| 142 | if (existing) continue; // already flagged; don't churn |
| 143 | |
| 144 | await saveDraft(env.CURATED_KV, draft); |
| 145 | } |
| 146 | |
| 147 | return { ok: true, checked: results.length, broken: broken.length, results: broken }; |
| 148 | } |
| 149 | |
| 150 | // --- Semantic drift --- |
| 151 | |
| 152 | const SEMANTIC_DRIFT_PROMPT = `You are reviewing copy on a community website (codewhale.net) for the open-source Codewhale project. |
| 153 | |
| 154 | Given: |
| 155 | 1. The CHANGELOG entries below (most recent first) |
| 156 | 2. The current homepage and docs page text below |
| 157 | 3. Recent commit messages |
| 158 | |
| 159 | Identify any factual claims on the site that are CONTRADICTED by recent changes. Be conservative — only flag claims you can directly tie to a CHANGELOG line or commit. Don't speculate. |
| 160 | |
| 161 | Return ONLY this JSON shape (no prose, no markdown fences): |
| 162 | { |
| 163 | "drifts": [ |
| 164 | { |
| 165 | "page": "homepage" | "docs" | "install" | "contribute" | "roadmap", |
| 166 | "claim": "exact text on the site that is now inaccurate", |
| 167 | "evidence": "the CHANGELOG line or commit hash that contradicts it", |
| 168 | "suggested_replacement": "what the site should say instead" |
| 169 | } |
| 170 | ] |
| 171 | } |
| 172 | |
| 173 | If nothing is drifted, return { "drifts": [] }. |
| 174 | |
| 175 | ${VOICE_CONSTRAINTS}`; |
| 176 | |
| 177 | function startsWithAsciiCI(input: string, index: number, needle: string): boolean { |
| 178 | if (index + needle.length > input.length) return false; |
| 179 | return input.slice(index, index + needle.length).toLowerCase() === needle; |
| 180 | } |
| 181 | |
| 182 | function isWhitespace(c: string | undefined): boolean { |
| 183 | return c === " " || c === "\n" || c === "\r" || c === "\t" || c === "\f"; |
| 184 | } |
| 185 | |
| 186 | function tagNameBoundary(input: string, index: number): boolean { |
| 187 | const c = input[index]; |
| 188 | return c === undefined || c === ">" || c === "/" || isWhitespace(c); |
| 189 | } |
| 190 | |
| 191 | function findClosingRawTextTag(input: string, from: number, tagName: "script" | "style"): number { |
| 192 | const closePrefix = `</${tagName}`; |
| 193 | for (let i = from; i < input.length; i += 1) { |
| 194 | if (startsWithAsciiCI(input, i, closePrefix) && tagNameBoundary(input, i + closePrefix.length)) { |
| 195 | const close = input.indexOf(">", i + closePrefix.length); |
| 196 | return close === -1 ? input.length : close + 1; |
| 197 | } |
| 198 | } |
| 199 | return input.length; |
| 200 | } |
| 201 | |
| 202 | function collapseWhitespace(input: string): string { |
| 203 | let out = ""; |
| 204 | let pendingSpace = false; |
| 205 | for (const c of input) { |
| 206 | if (isWhitespace(c)) { |
| 207 | pendingSpace = out.length > 0; |
| 208 | continue; |
| 209 | } |
| 210 | if (pendingSpace) out += " "; |
| 211 | out += c; |
| 212 | pendingSpace = false; |
| 213 | } |
| 214 | return out.trim(); |
| 215 | } |
| 216 | |
| 217 | function stripHtmlForPrompt(input: string): string { |
| 218 | let out = ""; |
| 219 | for (let i = 0; i < input.length;) { |
| 220 | if (input[i] !== "<") { |
| 221 | out += input[i]; |
| 222 | i += 1; |
| 223 | continue; |
| 224 | } |
| 225 | |
| 226 | if (startsWithAsciiCI(input, i, "<script") && tagNameBoundary(input, i + "<script".length)) { |
| 227 | out += " "; |
| 228 | const openEnd = input.indexOf(">", i + 1); |
| 229 | i = openEnd === -1 ? input.length : findClosingRawTextTag(input, openEnd + 1, "script"); |
| 230 | continue; |
| 231 | } |
| 232 | if (startsWithAsciiCI(input, i, "<style") && tagNameBoundary(input, i + "<style".length)) { |
| 233 | out += " "; |
| 234 | const openEnd = input.indexOf(">", i + 1); |
| 235 | i = openEnd === -1 ? input.length : findClosingRawTextTag(input, openEnd + 1, "style"); |
| 236 | continue; |
| 237 | } |
| 238 | |
| 239 | out += " "; |
| 240 | const tagEnd = input.indexOf(">", i + 1); |
| 241 | i = tagEnd === -1 ? input.length : tagEnd + 1; |
| 242 | } |
| 243 | return collapseWhitespace(out).slice(0, 8000); |
| 244 | } |
| 245 | |
| 246 | // The model's drift list is untrusted input: bound it before any KV fanout so |
| 247 | // a runaway or hostile response cannot flood the draft queue, and drop entries |
| 248 | // whose shape does not match the prompt's contract. |
| 249 | const MAX_DRIFT_DRAFTS_PER_RUN = 10; |
| 250 | const DRIFT_FIELD_MAX = 2_000; |
| 251 | const DRIFT_PAGES = new Set(["homepage", "docs", "install", "contribute", "roadmap"]); |
| 252 | |
| 253 | interface DriftFinding { |
| 254 | page: string; |
| 255 | claim: string; |
| 256 | evidence: string; |
| 257 | suggested_replacement: string; |
| 258 | } |
| 259 | |
| 260 | function validateDriftFindings(raw: unknown): DriftFinding[] { |
| 261 | if (!Array.isArray(raw)) return []; |
| 262 | const findings: DriftFinding[] = []; |
| 263 | for (const entry of raw) { |
| 264 | if (findings.length >= MAX_DRIFT_DRAFTS_PER_RUN) break; |
| 265 | if (typeof entry !== "object" || entry === null) continue; |
| 266 | const { page, claim, evidence, suggested_replacement } = entry as Record<string, unknown>; |
| 267 | if (typeof page !== "string" || !DRIFT_PAGES.has(page)) continue; |
| 268 | if (typeof claim !== "string" || !claim.trim()) continue; |
| 269 | if (typeof evidence !== "string" || !evidence.trim()) continue; |
| 270 | if (typeof suggested_replacement !== "string" || !suggested_replacement.trim()) continue; |
| 271 | findings.push({ |
| 272 | page, |
| 273 | claim: claim.slice(0, DRIFT_FIELD_MAX), |
| 274 | evidence: evidence.slice(0, DRIFT_FIELD_MAX), |
| 275 | suggested_replacement: suggested_replacement.slice(0, DRIFT_FIELD_MAX), |
| 276 | }); |
| 277 | } |
| 278 | return findings; |
| 279 | } |
| 280 | |
| 281 | export async function runSemanticDrift(env: WatchEnv): Promise<{ ok: boolean; drafted: number; reason?: string }> { |
| 282 | if (!env.CURATED_KV || !env.DEEPSEEK_API_KEY) { |
| 283 | return { ok: false, drafted: 0, reason: "missing CURATED_KV or DEEPSEEK_API_KEY" }; |
| 284 | } |
| 285 | |
| 286 | const ghHeaders: Record<string, string> = { |
| 287 | Accept: "application/vnd.github+json", |
| 288 | "User-Agent": "codewhale-web-semantic-drift", |
| 289 | }; |
| 290 | if (env.GITHUB_TOKEN) ghHeaders["Authorization"] = `Bearer ${env.GITHUB_TOKEN}`; |
| 291 | |
| 292 | // Fetch CHANGELOG (truncated), recent commits, and live homepage HTML. |
| 293 | const [changelog, commits, homepageHtml, docsHtml] = await Promise.all([ |
| 294 | fetch("https://raw.githubusercontent.com/Hmbown/CodeWhale/main/CHANGELOG.md", { headers: ghHeaders }).then((r) => r.ok ? r.text() : "").catch(() => ""), |
| 295 | fetch("https://api.github.com/repos/Hmbown/CodeWhale/commits?per_page=30", { headers: ghHeaders }).then((r) => r.ok ? r.json() as Promise<{ commit: { message: string }; sha: string }[]> : []).catch(() => []), |
| 296 | fetch("https://codewhale.net/en", { headers: { "User-Agent": "codewhale-watch" } }).then((r) => r.ok ? r.text() : "").catch(() => ""), |
| 297 | fetch("https://codewhale.net/en/docs", { headers: { "User-Agent": "codewhale-watch" } }).then((r) => r.ok ? r.text() : "").catch(() => ""), |
| 298 | ]); |
| 299 | |
| 300 | if (!changelog && (!commits || commits.length === 0)) { |
| 301 | return { ok: false, drafted: 0, reason: "no changelog or commits available" }; |
| 302 | } |
| 303 | |
| 304 | const homepageText = stripHtmlForPrompt(homepageHtml); |
| 305 | const docsText = stripHtmlForPrompt(docsHtml); |
| 306 | const changelogHead = changelog.slice(0, 4000); |
| 307 | const commitMsgs = commits.slice(0, 30).map((c) => `- ${c.sha.slice(0, 7)}: ${c.commit.message.split("\n")[0]}`).join("\n"); |
| 308 | |
| 309 | const userMessage = `## Recent CHANGELOG entries |
| 310 | ${changelogHead || "(no CHANGELOG.md fetched)"} |
| 311 | |
| 312 | ## Last 30 commits |
| 313 | ${commitMsgs || "(no commits fetched)"} |
| 314 | |
| 315 | ## Homepage text (HTML stripped) |
| 316 | ${homepageText} |
| 317 | |
| 318 | ## Docs page text (HTML stripped) |
| 319 | ${docsText}`; |
| 320 | |
| 321 | let response: { content: string; usage: { input: number; output: number } }; |
| 322 | try { |
| 323 | response = await agentChat( |
| 324 | [ |
| 325 | { role: "system", content: SEMANTIC_DRIFT_PROMPT }, |
| 326 | { role: "user", content: userMessage }, |
| 327 | ], |
| 328 | env.DEEPSEEK_API_KEY, |
| 329 | true, |
| 330 | dsEnv(env), |
| 331 | ); |
| 332 | } catch (e) { |
| 333 | return { ok: false, drafted: 0, reason: `LLM call failed: ${e}` }; |
| 334 | } |
| 335 | |
| 336 | // Extract JSON (jsonMode usually returns clean JSON, but defend against fences) |
| 337 | let parsed: { drifts?: unknown }; |
| 338 | try { |
| 339 | const trimmed = response.content.replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/i, "").trim(); |
| 340 | parsed = JSON.parse(trimmed); |
| 341 | } catch { |
| 342 | return { ok: false, drafted: 0, reason: "LLM returned non-JSON" }; |
| 343 | } |
| 344 | |
| 345 | const drifts = validateDriftFindings(parsed.drifts); |
| 346 | let drafted = 0; |
| 347 | for (const d of drifts) { |
| 348 | // Identity covers the full finding — page, claim, evidence, and |
| 349 | // replacement — so a re-run over an unchanged finding dedups, while a |
| 350 | // changed finding lands under a new key as a fresh draft. |
| 351 | const identity = [d.page, d.claim, d.evidence, d.suggested_replacement].join("\n"); |
| 352 | const id = await watchDraftId(`${d.page}-${d.claim.slice(0, 40)}`, `semantic-drift\n${identity}`); |
| 353 | const body = `Page: **${d.page}**\n\nClaim that may be drifted:\n> ${d.claim}\n\nEvidence:\n> ${d.evidence}\n\nSuggested replacement:\n> ${d.suggested_replacement}\n\n— drafted by community assistant, pending maintainer review`; |
| 354 | const draft: AgentDraft = { |
| 355 | id, |
| 356 | type: "semantic-drift", |
| 357 | targetUrl: `https://codewhale.net/en/${d.page === "homepage" ? "" : d.page}`, |
| 358 | bodyEn: body, |
| 359 | bodyZh: body, |
| 360 | generatedAt: new Date().toISOString(), |
| 361 | posted: false, |
| 362 | }; |
| 363 | const existing = await getDraft(env.CURATED_KV, draftStorageKey(draft)); |
| 364 | if (existing) continue; |
| 365 | |
| 366 | await saveDraft(env.CURATED_KV, draft); |
| 367 | drafted++; |
| 368 | } |
| 369 | |
| 370 | return { ok: true, drafted }; |
| 371 | } |
| 372 |