| 1 | /** |
| 2 | * facts-drift.ts — runtime version of scripts/derive-facts.mjs. |
| 3 | * |
| 4 | * Fetches source-of-truth files from raw.githubusercontent.com on a schedule, |
| 5 | * re-derives the same RepoFacts shape, compares to the value cached in KV (or |
| 6 | * to the build-time fallback on first run), and if anything changed writes |
| 7 | * the new facts to CURATED_KV under "facts:current". `getFacts()` accepts the |
| 8 | * KV value only when its exact source provenance is at least as new as the |
| 9 | * deployed build; published-release metadata is resolved separately. |
| 10 | * |
| 11 | * Mechanical drift (provider added, sandbox backend renamed, version bumped) |
| 12 | * fixes itself within one cron tick — no redeploy. Semantic drift (a new |
| 13 | * feature should be advertised on the homepage) is still left to humans. |
| 14 | */ |
| 15 | import type { |
| 16 | PublishedReleaseFact, |
| 17 | RepoFacts, |
| 18 | ProviderFact, |
| 19 | } from "./facts.generated"; |
| 20 | import { FACTS as BUILD_FACTS } from "./facts.generated"; |
| 21 | |
| 22 | const RAW_ROOT = "https://raw.githubusercontent.com/Hmbown/CodeWhale"; |
| 23 | const KV_KEY = "facts:current"; |
| 24 | const LOG_KEY = "facts:drift-log"; |
| 25 | |
| 26 | interface KVNamespace { |
| 27 | get(k: string): Promise<string | null>; |
| 28 | put(k: string, v: string, o?: { expirationTtl?: number }): Promise<void>; |
| 29 | } |
| 30 | |
| 31 | interface SourceMarker { |
| 32 | revision: string; |
| 33 | committedAt: string; |
| 34 | } |
| 35 | |
| 36 | async function fetchText( |
| 37 | path: string, |
| 38 | revision: string, |
| 39 | ghToken?: string, |
| 40 | ): Promise<string | null> { |
| 41 | const headers: Record<string, string> = { |
| 42 | "User-Agent": "codewhale-web-drift", |
| 43 | }; |
| 44 | if (ghToken) headers["Authorization"] = `Bearer ${ghToken}`; |
| 45 | try { |
| 46 | const r = await fetch(`${RAW_ROOT}/${revision}/${path}`, { headers }); |
| 47 | if (!r.ok) return null; |
| 48 | return await r.text(); |
| 49 | } catch { |
| 50 | return null; |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | async function fetchSourceMarker(ghToken?: string): Promise<SourceMarker | null> { |
| 55 | const headers: Record<string, string> = { |
| 56 | Accept: "application/vnd.github+json", |
| 57 | "User-Agent": "codewhale-web-drift", |
| 58 | "X-GitHub-Api-Version": "2022-11-28", |
| 59 | }; |
| 60 | if (ghToken) headers.Authorization = `Bearer ${ghToken}`; |
| 61 | try { |
| 62 | const response = await fetch( |
| 63 | "https://api.github.com/repos/Hmbown/CodeWhale/commits/main", |
| 64 | { headers }, |
| 65 | ); |
| 66 | if (!response.ok) return null; |
| 67 | const json = (await response.json()) as { |
| 68 | sha?: string; |
| 69 | commit?: { committer?: { date?: string } }; |
| 70 | }; |
| 71 | const revision = json.sha; |
| 72 | const committedAt = json.commit?.committer?.date; |
| 73 | if ( |
| 74 | !revision || |
| 75 | !/^[0-9a-f]{40}$/i.test(revision) || |
| 76 | !committedAt || |
| 77 | !Number.isFinite(Date.parse(committedAt)) |
| 78 | ) { |
| 79 | return null; |
| 80 | } |
| 81 | return { revision, committedAt }; |
| 82 | } catch { |
| 83 | return null; |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | function deriveVersion(cargo: string): string | null { |
| 88 | const m = cargo.match(/^version\s*=\s*"([^"]+)"/m); |
| 89 | return m ? m[1] : null; |
| 90 | } |
| 91 | |
| 92 | function deriveCrates(cargo: string): string[] { |
| 93 | const block = cargo.match(/members\s*=\s*\[([\s\S]*?)\]/); |
| 94 | if (!block) return []; |
| 95 | return [...block[1].matchAll(/"crates\/([^"]+)"/g)].map((m) => m[1]).sort(); |
| 96 | } |
| 97 | |
| 98 | function deriveProvidersFromConfig(cfg: string): ProviderFact[] { |
| 99 | const enumBlock = cfg.match(/pub enum ApiProvider \{([\s\S]*?)\}/); |
| 100 | if (!enumBlock) return []; |
| 101 | const variants = [...enumBlock[1].matchAll(/^\s*(\w+)\s*,\s*$/gm)].map((m) => m[1]); |
| 102 | // Match what the published CLI binary's `--provider` flag accepts |
| 103 | // (ProviderArg in crates/cli/src/lib.rs). DeepseekCN exists in the |
| 104 | // legacy tui ApiProvider enum but is not wired through ProviderKind, |
| 105 | // so the binary rejects it — keep it out of the docs. Issue #1104. |
| 106 | const labelMap: Record<string, ProviderFact> = { |
| 107 | Deepseek: { id: "deepseek", label: "DeepSeek", env: "DEEPSEEK_API_KEY" }, |
| 108 | DeepseekAnthropic: { id: "deepseek-anthropic", label: "DeepSeek Anthropic", env: "DEEPSEEK_API_KEY / ANTHROPIC_API_KEY" }, |
| 109 | NvidiaNim: { id: "nvidia-nim", label: "NVIDIA NIM", env: "NVIDIA_API_KEY / NVIDIA_NIM_API_KEY" }, |
| 110 | Openai: { id: "openai", label: "OpenAI-compatible", env: "OPENAI_API_KEY" }, |
| 111 | Atlascloud: { id: "atlascloud", label: "AtlasCloud", env: "ATLASCLOUD_API_KEY" }, |
| 112 | WanjieArk: { id: "wanjie-ark", label: "Wanjie Ark", env: "WANJIE_ARK_API_KEY / WANJIE_API_KEY / WANJIE_MAAS_API_KEY" }, |
| 113 | Volcengine: { id: "volcengine", label: "Volcengine Ark", env: "VOLCENGINE_API_KEY / VOLCENGINE_ARK_API_KEY / ARK_API_KEY" }, |
| 114 | Openrouter: { id: "openrouter", label: "OpenRouter", env: "OPENROUTER_API_KEY" }, |
| 115 | XiaomiMimo: { id: "xiaomi-mimo", label: "Xiaomi MiMo", env: "XIAOMI_MIMO_TOKEN_PLAN_API_KEY / MIMO_TOKEN_PLAN_API_KEY / XIAOMI_MIMO_API_KEY / XIAOMI_API_KEY / MIMO_API_KEY" }, |
| 116 | Novita: { id: "novita", label: "Novita AI", env: "NOVITA_API_KEY" }, |
| 117 | Fireworks: { id: "fireworks", label: "Fireworks AI", env: "FIREWORKS_API_KEY" }, |
| 118 | Siliconflow: { id: "siliconflow", label: "SiliconFlow", env: "SILICONFLOW_API_KEY" }, |
| 119 | SiliconflowCn: { id: "siliconflow-CN", label: "SiliconFlow CN", env: "SILICONFLOW_API_KEY" }, |
| 120 | Arcee: { id: "arcee", label: "Arcee AI", env: "ARCEE_API_KEY" }, |
| 121 | Moonshot: { id: "moonshot", label: "Moonshot/Kimi", env: "MOONSHOT_API_KEY / KIMI_API_KEY" }, |
| 122 | Sglang: { id: "sglang", label: "SGLang", env: "SGLANG_API_KEY" }, |
| 123 | Vllm: { id: "vllm", label: "vLLM", env: "VLLM_API_KEY" }, |
| 124 | Ollama: { id: "ollama", label: "Ollama", env: "OLLAMA_API_KEY" }, |
| 125 | Huggingface: { id: "huggingface", label: "Hugging Face", env: "HUGGINGFACE_API_KEY / HF_TOKEN" }, |
| 126 | Deepinfra: { id: "deepinfra", label: "DeepInfra", env: "DEEPINFRA_API_KEY / DEEPINFRA_TOKEN" }, |
| 127 | Together: { id: "together", label: "Together AI", env: "TOGETHER_API_KEY" }, |
| 128 | Qianfan: { id: "qianfan", label: "Baidu Qianfan", env: "QIANFAN_API_KEY / BAIDU_QIANFAN_API_KEY" }, |
| 129 | OpenaiCodex: { id: "openai-codex", label: "OpenAI Codex", env: "ChatGPT/Codex OAuth via `codex login` (OPENAI_CODEX_ACCESS_TOKEN / CODEX_ACCESS_TOKEN override)" }, |
| 130 | OpencodeGo: { id: "opencode-go", label: "OpenCode Go", env: "OPENCODE_GO_API_KEY" }, |
| 131 | OpencodeZen: { id: "opencode-zen", label: "OpenCode Zen", env: "OPENCODE_ZEN_API_KEY / OPENCODE_API_KEY" }, |
| 132 | Anthropic: { id: "anthropic", label: "Anthropic", env: "ANTHROPIC_API_KEY" }, |
| 133 | Zai: { id: "zai", label: "Z.ai", env: "ZAI_API_KEY / Z_AI_API_KEY" }, |
| 134 | Stepfun: { id: "stepfun", label: "StepFun", env: "STEPFUN_API_KEY / STEP_API_KEY" }, |
| 135 | Minimax: { id: "minimax", label: "MiniMax", env: "MINIMAX_API_KEY" }, |
| 136 | MinimaxAnthropic: { id: "minimax-anthropic", label: "MiniMax (Anthropic-compatible)", env: "MINIMAX_API_KEY" }, |
| 137 | Openmodel: { id: "openmodel", label: "OpenModel", env: "OPENMODEL_API_KEY" }, |
| 138 | Sakana: { id: "sakana", label: "Sakana AI", env: "FUGU_API_KEY / SAKANA_API_KEY" }, |
| 139 | LongCat: { id: "longcat", label: "Meituan LongCat", env: "LONGCAT_API_KEY" }, |
| 140 | Meta: { id: "meta", label: "Meta Model API", env: "META_MODEL_API_KEY / MODEL_API_KEY" }, |
| 141 | Telecomjs: { id: "telecomjs", label: "TelecomJS TokenHub", env: "TELECOMJS_API_KEY" }, |
| 142 | Xai: { id: "xai", label: "xAI", env: "XAI_API_KEY" }, |
| 143 | ModelstudioTokenPlan: { id: "modelstudio-token-plan", label: "Model Studio Token Plan", env: "MODELSTUDIO_API_KEY" }, |
| 144 | ModelstudioTokenPlanAnthropic: { id: "modelstudio-token-plan-anthropic", label: "Model Studio Token Plan (Anthropic-compatible)", env: "MODELSTUDIO_API_KEY" }, |
| 145 | ModelstudioCodingPlan: { id: "modelstudio-coding-plan", label: "Model Studio Coding Plan", env: "MODELSTUDIO_API_KEY" }, |
| 146 | ModelstudioCodingPlanAnthropic: { id: "modelstudio-coding-plan-anthropic", label: "Model Studio Coding Plan (Anthropic-compatible)", env: "MODELSTUDIO_API_KEY" }, |
| 147 | }; |
| 148 | // Log loudly on unmapped variants so a new provider can never be silently |
| 149 | // dropped from the drift-derived facts again. DeepseekCN (#1104) and the |
| 150 | // dynamic Custom meta-provider (#1519, user-defined endpoints) are the |
| 151 | // deliberate exclusions. |
| 152 | const EXCLUDED = new Set(["DeepseekCN", "Custom"]); |
| 153 | const unmapped = variants.filter((v) => !EXCLUDED.has(v) && !labelMap[v]); |
| 154 | if (unmapped.length > 0) { |
| 155 | console.warn( |
| 156 | `[facts-drift] ApiProvider variants missing from labelMap: ${unmapped.join(", ")}. ` + |
| 157 | "Add them to labelMap here AND PROVIDER_LABEL_MAP in web/scripts/facts-lib.mjs (or to EXCLUDED if intentionally hidden).", |
| 158 | ); |
| 159 | } |
| 160 | return variants.map((v) => labelMap[v]).filter(Boolean); |
| 161 | } |
| 162 | |
| 163 | function deriveDefaultModel(cfg: string): string | null { |
| 164 | // Match the const *definition* (`= "..."`); the definition moved to |
| 165 | // config/models.rs in the #3311 split, so callers pass config.rs + models.rs. |
| 166 | const m = cfg.match(/DEFAULT_TEXT_MODEL\s*(?::\s*&str\s*)?=\s*"([^"]+)"/); |
| 167 | return m ? m[1] : null; |
| 168 | } |
| 169 | |
| 170 | function deriveSandboxBackends(source: string): string[] { |
| 171 | const marker = source.match( |
| 172 | /pub const PUBLIC_SANDBOX_BACKENDS\s*:\s*&\[&str\]\s*=\s*&\[([\s\S]*?)\];/, |
| 173 | ); |
| 174 | if (!marker) return []; |
| 175 | return [...marker[1].matchAll(/"([^"]+)"/g)].map((match) => match[1]); |
| 176 | } |
| 177 | |
| 178 | async function fetchLatestPublishedRelease( |
| 179 | ghToken?: string, |
| 180 | ): Promise<PublishedReleaseFact | null> { |
| 181 | const headers: Record<string, string> = { |
| 182 | Accept: "application/vnd.github+json", |
| 183 | "User-Agent": "codewhale-web-drift", |
| 184 | "X-GitHub-Api-Version": "2022-11-28", |
| 185 | }; |
| 186 | if (ghToken) headers["Authorization"] = `Bearer ${ghToken}`; |
| 187 | try { |
| 188 | const r = await fetch("https://api.github.com/repos/Hmbown/CodeWhale/releases/latest", { headers }); |
| 189 | if (!r.ok) return null; |
| 190 | const j = (await r.json()) as { |
| 191 | tag_name?: string; |
| 192 | published_at?: string; |
| 193 | html_url?: string; |
| 194 | }; |
| 195 | if ( |
| 196 | !j.tag_name || |
| 197 | !/^v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(j.tag_name) || |
| 198 | !j.published_at || |
| 199 | !Number.isFinite(Date.parse(j.published_at)) || |
| 200 | !j.html_url |
| 201 | ) { |
| 202 | return null; |
| 203 | } |
| 204 | return { |
| 205 | tag: j.tag_name, |
| 206 | version: j.tag_name.slice(1), |
| 207 | publishedAt: j.published_at, |
| 208 | url: j.html_url, |
| 209 | }; |
| 210 | } catch { |
| 211 | return null; |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | function deriveLicense(licText: string): string | null { |
| 216 | const first = licText.split(/\r?\n/).find((l) => l.trim().length > 0); |
| 217 | if (!first) return null; |
| 218 | if (/^MIT License/i.test(first)) return "MIT"; |
| 219 | if (/Apache.*2\.0/i.test(first)) return "Apache-2.0"; |
| 220 | return first.trim(); |
| 221 | } |
| 222 | |
| 223 | function deriveToolCountFromGeneratedFacts(source: string): number | null { |
| 224 | const match = source.match( |
| 225 | /export\s+const\s+FACTS(?:\s*:\s*RepoFacts)?\s*=\s*(\{[\s\S]*\})\s*;?\s*$/, |
| 226 | ); |
| 227 | if (!match) return null; |
| 228 | |
| 229 | try { |
| 230 | const parsed = JSON.parse(match[1]) as { toolCount?: unknown }; |
| 231 | const toolCount = parsed.toolCount; |
| 232 | return typeof toolCount === "number" && Number.isSafeInteger(toolCount) && toolCount >= 0 |
| 233 | ? toolCount |
| 234 | : null; |
| 235 | } catch { |
| 236 | return null; |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | export async function deriveFactsFromRemote(ghToken?: string): Promise<RepoFacts | null> { |
| 241 | const source = await fetchSourceMarker(ghToken); |
| 242 | if (!source) return null; |
| 243 | |
| 244 | const [cargo, configRs, configModels, sandboxSource, npmPkg, licText, generatedFacts, latestPublishedRelease] = await Promise.all([ |
| 245 | fetchText("Cargo.toml", source.revision, ghToken), |
| 246 | fetchText("crates/tui/src/config.rs", source.revision, ghToken), |
| 247 | fetchText("crates/tui/src/config/models.rs", source.revision, ghToken), |
| 248 | fetchText("crates/tui/src/sandbox/mod.rs", source.revision, ghToken), |
| 249 | fetchText("npm/codewhale/package.json", source.revision, ghToken), |
| 250 | fetchText("LICENSE", source.revision, ghToken), |
| 251 | fetchText("web/lib/facts.generated.ts", source.revision, ghToken), |
| 252 | fetchLatestPublishedRelease(ghToken), |
| 253 | ]); |
| 254 | |
| 255 | if (!cargo || !configRs) return null; |
| 256 | const toolCount = generatedFacts |
| 257 | ? deriveToolCountFromGeneratedFacts(generatedFacts) |
| 258 | : null; |
| 259 | // Never attach current-main provenance to a build-time tool count. The |
| 260 | // checked-in generated snapshot is guarded by the exact revision's CI drift |
| 261 | // check, so an absent or malformed value makes the whole derivation fail. |
| 262 | if (toolCount === null) return null; |
| 263 | |
| 264 | const facts: RepoFacts = { |
| 265 | generatedAt: new Date().toISOString(), |
| 266 | sourceRevision: source.revision, |
| 267 | sourceCommittedAt: source.committedAt, |
| 268 | version: deriveVersion(cargo), |
| 269 | crates: deriveCrates(cargo), |
| 270 | sandboxBackends: sandboxSource |
| 271 | ? deriveSandboxBackends(sandboxSource) |
| 272 | : BUILD_FACTS.sandboxBackends, |
| 273 | providers: deriveProvidersFromConfig(configRs), |
| 274 | defaultModel: deriveDefaultModel(`${configRs}\n${configModels ?? ""}`), |
| 275 | nodeEngines: (() => { |
| 276 | try { return npmPkg ? JSON.parse(npmPkg).engines?.node ?? null : null; } catch { return null; } |
| 277 | })(), |
| 278 | toolCount, |
| 279 | license: licText ? deriveLicense(licText) : BUILD_FACTS.license, |
| 280 | latestPublishedRelease: |
| 281 | latestPublishedRelease ?? BUILD_FACTS.latestPublishedRelease, |
| 282 | }; |
| 283 | |
| 284 | if (!facts.version || facts.crates.length === 0 || facts.providers.length === 0) { |
| 285 | return null; |
| 286 | } |
| 287 | return facts; |
| 288 | } |
| 289 | |
| 290 | interface DriftDiff { |
| 291 | field: keyof RepoFacts; |
| 292 | before: unknown; |
| 293 | after: unknown; |
| 294 | } |
| 295 | |
| 296 | function diff(a: RepoFacts, b: RepoFacts): DriftDiff[] { |
| 297 | const fields: (keyof RepoFacts)[] = [ |
| 298 | "sourceRevision", |
| 299 | "sourceCommittedAt", |
| 300 | "version", |
| 301 | "crates", |
| 302 | "sandboxBackends", |
| 303 | "providers", |
| 304 | "defaultModel", |
| 305 | "nodeEngines", |
| 306 | "toolCount", |
| 307 | "license", |
| 308 | "latestPublishedRelease", |
| 309 | ]; |
| 310 | const out: DriftDiff[] = []; |
| 311 | for (const f of fields) { |
| 312 | const av = JSON.stringify(a[f]); |
| 313 | const bv = JSON.stringify(b[f]); |
| 314 | if (av !== bv) out.push({ field: f, before: a[f], after: b[f] }); |
| 315 | } |
| 316 | return out; |
| 317 | } |
| 318 | |
| 319 | export interface FactsDriftResult { |
| 320 | ok: boolean; |
| 321 | changed?: boolean; |
| 322 | diffs?: DriftDiff[]; |
| 323 | reason?: string; |
| 324 | } |
| 325 | |
| 326 | export async function runFactsDrift(env: { CURATED_KV?: KVNamespace; GITHUB_TOKEN?: string }): Promise<FactsDriftResult> { |
| 327 | if (!env.CURATED_KV) return { ok: false, reason: "CURATED_KV not bound" }; |
| 328 | |
| 329 | const remote = await deriveFactsFromRemote(env.GITHUB_TOKEN); |
| 330 | if (!remote) return { ok: false, reason: "remote derivation failed" }; |
| 331 | |
| 332 | const cachedRaw = await env.CURATED_KV.get(KV_KEY); |
| 333 | let cached: RepoFacts = BUILD_FACTS; |
| 334 | if (cachedRaw) { |
| 335 | try { |
| 336 | const parsed = JSON.parse(cachedRaw) as unknown; |
| 337 | if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { |
| 338 | cached = parsed as RepoFacts; |
| 339 | } |
| 340 | } catch { |
| 341 | // A truncated or legacy cache is replaced by the newly derived snapshot. |
| 342 | } |
| 343 | } |
| 344 | |
| 345 | const diffs = diff(cached, remote); |
| 346 | if (diffs.length === 0) { |
| 347 | return { ok: true, changed: false }; |
| 348 | } |
| 349 | |
| 350 | // Write new facts. No TTL — they live until next drift overwrites them. |
| 351 | await env.CURATED_KV.put(KV_KEY, JSON.stringify(remote)); |
| 352 | |
| 353 | // Append to drift log (last 20 entries). |
| 354 | try { |
| 355 | const logRaw = await env.CURATED_KV.get(LOG_KEY); |
| 356 | const log = logRaw ? (JSON.parse(logRaw) as Array<{ at: string; diffs: DriftDiff[] }>) : []; |
| 357 | log.unshift({ at: remote.generatedAt, diffs }); |
| 358 | await env.CURATED_KV.put(LOG_KEY, JSON.stringify(log.slice(0, 20))); |
| 359 | } catch { |
| 360 | /* non-fatal */ |
| 361 | } |
| 362 | |
| 363 | return { ok: true, changed: true, diffs }; |
| 364 | } |
| 365 |