| 1 | // Ingest + dashboard for desktop crash/feedback/performance reports and the |
| 2 | // anonymous launch ping. Frontend reports are user-initiated; native fatal and |
| 3 | // lifecycle reports are sent on the next launch under the same opt-out desktop |
| 4 | // telemetry gate as pings. |
| 5 | import { z } from "zod"; |
| 6 | import type { Env } from "./env"; |
| 7 | import { html, redirect } from "./shell"; |
| 8 | import { renderStats, type StatsModule } from "./stats"; |
| 9 | import { renderGroup, type Group, type ReportSample } from "./group"; |
| 10 | import { renderAccount } from "./auth_pages"; |
| 11 | import { renderUsers, renderAudit, type UserRow, type AuditRow } from "./admin"; |
| 12 | import { |
| 13 | atLeast, |
| 14 | currentUser, |
| 15 | loginUrl, |
| 16 | logAction, |
| 17 | sameOrigin, |
| 18 | sharedLogout, |
| 19 | type Role, |
| 20 | type User, |
| 21 | } from "./auth"; |
| 22 | import registryApp from "./registry/app"; |
| 23 | import type { Bindings as RegistryBindings } from "./registry/env"; |
| 24 | import { PackageRepo } from "./registry/db/packages"; |
| 25 | import { EventRepo } from "./registry/db/events"; |
| 26 | import { renderCommunity } from "./community"; |
| 27 | import { |
| 28 | cliReleaseChannel, |
| 29 | desktopReleaseChannel, |
| 30 | handleCLIRelease, |
| 31 | handleDesktopReleaseManifest, |
| 32 | handleReleaseGatewayRequest, |
| 33 | } from "./desktop_release"; |
| 34 | import { |
| 35 | crashGroups, |
| 36 | currentWindowSince, |
| 37 | developmentGroupSQL, |
| 38 | diagnosticFacets as loadDiagnosticFacets, |
| 39 | diagnosticWindowWhere, |
| 40 | effectiveGroupSeverity, |
| 41 | groupDiagnosticSummary, |
| 42 | isDevelopmentGroup, |
| 43 | reportAggregateStatements, |
| 44 | type DiagnosticFacets, |
| 45 | } from "./diagnostics_v2"; |
| 46 | import { statsQueryObserver } from "./stats_timing"; |
| 47 | import { loadD1GroupReports, loadGroupDiagnostics } from "./group_queries"; |
| 48 | import { Report, WebRuntimeDiagnostic, type ReportPayload } from "./report_schema"; |
| 49 | import { |
| 50 | crashTitle, groupFingerprintFromPath, isDevelopmentReport, namespaceReportFingerprint, newestReleaseVersion, |
| 51 | regressionDecisionForReport, |
| 52 | reportSubjectIdentity, |
| 53 | severityForReport, |
| 54 | } from "./report_classification"; |
| 55 | import { statsFilters, type StatsFilters } from "./stats_filters"; |
| 56 | import { |
| 57 | acquireFirebaseGroupLease, |
| 58 | claimFirebaseCrash, |
| 59 | crashStorageMode, |
| 60 | enqueueFirebaseCrash, |
| 61 | firebaseEventExists, |
| 62 | firebaseGroupState, |
| 63 | firebaseProjectionExists, |
| 64 | firebaseStorageReady, |
| 65 | projectionCompletionStatements, |
| 66 | purgeFirebaseDeliveryState, |
| 67 | recordFirebaseRetry, |
| 68 | reclaimUnusedFirebaseReservation, |
| 69 | releaseFirebaseGroupLease, |
| 70 | renewFirebaseGroupLease, |
| 71 | reserveFirebaseGroup, |
| 72 | type FirebaseGroupLease, |
| 73 | } from "./crash_delivery"; |
| 74 | import { |
| 75 | readFirebaseCrashGroup, |
| 76 | writeFirebaseGroupMeta, |
| 77 | } from "./firebase_rtdb"; |
| 78 | import { |
| 79 | deliverCrashEventToFirebase, |
| 80 | drainFirebaseCrashOutbox as drainFirebaseOutbox, |
| 81 | type StoredCrashEvent, |
| 82 | } from "./firebase_delivery"; |
| 83 | import { |
| 84 | archiveFirebaseGroupForAdmin, |
| 85 | firebaseStorageSummary, |
| 86 | runFirebaseCrashLifecycle, |
| 87 | FIREBASE_OUTBOX_WARNING, |
| 88 | type FirebaseStorageSummary, |
| 89 | } from "./firebase_lifecycle"; |
| 90 | import { |
| 91 | firebaseMeta, |
| 92 | firebaseSamples, |
| 93 | loadFirebaseGroupMeta, |
| 94 | type FirebaseGroupRow, |
| 95 | } from "./firebase_crash_view"; |
| 96 | export { Report } from "./report_schema"; |
| 97 | export { diagnosticWindowWhere, effectiveGroupSeverity, isDevelopmentGroup } from "./diagnostics_v2"; |
| 98 | const MAX_BODY_BYTES = 96 * 1024; |
| 99 | const LATEST_SAMPLES_PER_GROUP = 5; |
| 100 | |
| 101 | const ClientSurface = z.enum(["desktop", "cli"]); |
| 102 | type ClientSurfaceName = z.infer<typeof ClientSurface>; |
| 103 | |
| 104 | type TelemetryTableNames = { |
| 105 | pings: "pings" | "cli_pings"; |
| 106 | metrics: "metrics" | "cli_metrics"; |
| 107 | }; |
| 108 | |
| 109 | const TELEMETRY_TABLES: Record<ClientSurfaceName, TelemetryTableNames> = { |
| 110 | desktop: { pings: "pings", metrics: "metrics" }, |
| 111 | cli: { pings: "cli_pings", metrics: "cli_metrics" }, |
| 112 | }; |
| 113 | |
| 114 | export function telemetryTableNames(surface: ClientSurfaceName): TelemetryTableNames { |
| 115 | return TELEMETRY_TABLES[surface]; |
| 116 | } |
| 117 | |
| 118 | export const CLI_TELEMETRY_SCHEMA_SQL = [ |
| 119 | `CREATE TABLE IF NOT EXISTS cli_pings ( |
| 120 | date TEXT NOT NULL, |
| 121 | install_id TEXT NOT NULL, |
| 122 | version TEXT NOT NULL, |
| 123 | os TEXT NOT NULL, |
| 124 | arch TEXT NOT NULL, |
| 125 | os_version TEXT NOT NULL DEFAULT '', |
| 126 | os_build INTEGER NOT NULL DEFAULT 0, |
| 127 | os_revision INTEGER NOT NULL DEFAULT 0, |
| 128 | channel TEXT NOT NULL DEFAULT '', |
| 129 | distro_id TEXT NOT NULL DEFAULT '', |
| 130 | distro_version TEXT NOT NULL DEFAULT '', |
| 131 | kernel_version TEXT NOT NULL DEFAULT '', |
| 132 | session_type TEXT NOT NULL DEFAULT '', |
| 133 | runtime_engine TEXT NOT NULL DEFAULT '', |
| 134 | runtime_version TEXT NOT NULL DEFAULT '', |
| 135 | gpu_mode TEXT NOT NULL DEFAULT '', |
| 136 | opens INTEGER NOT NULL DEFAULT 1, |
| 137 | PRIMARY KEY (date, install_id) |
| 138 | )`, |
| 139 | `CREATE TABLE IF NOT EXISTS cli_metrics ( |
| 140 | date TEXT NOT NULL, |
| 141 | version TEXT NOT NULL, |
| 142 | os TEXT NOT NULL, |
| 143 | signal TEXT NOT NULL, |
| 144 | bucket TEXT NOT NULL, |
| 145 | count INTEGER NOT NULL DEFAULT 0, |
| 146 | PRIMARY KEY (date, version, os, signal, bucket) |
| 147 | )`, |
| 148 | // No secondary indexes: each primary key already leads with `date`, which is |
| 149 | // what every dashboard query filters on. See migrate-window-index-fix.sql. |
| 150 | ] as const; |
| 151 | |
| 152 | const cliTelemetrySchemaPromises = new WeakMap<object, Promise<void>>(); |
| 153 | |
| 154 | export function ensureCLITelemetrySchema(env: Pick<Env, "DB">): Promise<void> { |
| 155 | const key = env.DB as unknown as object; |
| 156 | const existing = cliTelemetrySchemaPromises.get(key); |
| 157 | if (existing) return existing; |
| 158 | const creation = env.DB |
| 159 | .batch(CLI_TELEMETRY_SCHEMA_SQL.map((sql) => env.DB.prepare(sql))) |
| 160 | .then(() => undefined) |
| 161 | .catch((err) => { |
| 162 | cliTelemetrySchemaPromises.delete(key); |
| 163 | throw err; |
| 164 | }); |
| 165 | cliTelemetrySchemaPromises.set(key, creation); |
| 166 | return creation; |
| 167 | } |
| 168 | |
| 169 | export const Ping = z.object({ |
| 170 | installId: z.string().regex(/^[0-9a-f]{32}$/), |
| 171 | version: z.string().min(1).max(64), |
| 172 | os: z.string().min(1).max(32), |
| 173 | arch: z.string().min(1).max(32), |
| 174 | osVersion: z.string().max(128).optional(), |
| 175 | osBuild: z.number().int().min(0).max(1_000_000).optional(), |
| 176 | osRevision: z.number().int().min(0).max(1_000_000).optional(), |
| 177 | channel: z.string().max(32).optional(), |
| 178 | distroId: z.string().max(64).optional(), |
| 179 | distroVersion: z.string().max(64).optional(), |
| 180 | kernelVersion: z.string().max(128).optional(), |
| 181 | sessionType: z.enum(["wayland", "x11", "remote", "unknown"]).optional(), |
| 182 | runtimeEngine: z.enum(["webview2", "webkitgtk", "unknown"]).optional(), |
| 183 | runtimeVersion: z.string().max(128).optional(), |
| 184 | gpuMode: z.enum(["enabled", "disabled", "always", "on_demand", "unknown"]).optional(), |
| 185 | surface: ClientSurface.default("desktop"), |
| 186 | }); |
| 187 | |
| 188 | // Opt-in aggregate client metrics: a per-launch snapshot of (signal, bucket) |
| 189 | // counters. The optional surface-specific random install id deduplicates DAU; |
| 190 | // there is no user content. Unknown signals are discarded before storage so |
| 191 | // older workers can accept batches from newer clients safely. |
| 192 | const METRIC_SIGNALS = [ |
| 193 | "finish_reason", |
| 194 | "empty_final", |
| 195 | "provider_error", |
| 196 | "cache_hit", |
| 197 | "tool_error", |
| 198 | "updater_error", |
| 199 | "updater_event", |
| 200 | "compaction", |
| 201 | "turns", |
| 202 | "desktop_hang", |
| 203 | "desktop_hang_age", |
| 204 | "desktop_exit", |
| 205 | "desktop_exit_phase", |
| 206 | "desktop_uptime", |
| 207 | "desktop_install", |
| 208 | "desktop_update_transition", |
| 209 | "desktop_restore", |
| 210 | "desktop_webview2_failure", |
| 211 | "desktop_webview2_outcome", |
| 212 | "desktop_web_runtime_failure", |
| 213 | "desktop_web_runtime_outcome", |
| 214 | "desktop_web_runtime_dropped", |
| 215 | "desktop_legacy_exit", |
| 216 | "desktop_legacy_exit_phase", |
| 217 | "cli_mode", |
| 218 | "cli_profile", |
| 219 | "cli_permission_mode", |
| 220 | "cli_session_mode", |
| 221 | "cli_turn_latency", |
| 222 | "cli_exit", |
| 223 | "recovery_failure", |
| 224 | "recovery_rule_continue", |
| 225 | "recovery_review_continue", |
| 226 | "recovery_human_prompt", |
| 227 | "recovery_human_continue", |
| 228 | "recovery_human_revise", |
| 229 | "recovery_review_error", |
| 230 | "recovery_repeat_prompt", |
| 231 | "recovery_review_latency", |
| 232 | "client_surface", |
| 233 | "client_version", |
| 234 | "settings_language", |
| 235 | "settings_desktop_layout", |
| 236 | "settings_theme", |
| 237 | "settings_theme_style", |
| 238 | "settings_close_behavior", |
| 239 | "settings_display_mode", |
| 240 | "settings_status_bar_style", |
| 241 | "settings_status_bar_items_count", |
| 242 | "settings_check_updates", |
| 243 | "settings_default_model", |
| 244 | "settings_planner_model", |
| 245 | "settings_subagent_model", |
| 246 | "settings_subagent_effort", |
| 247 | "settings_reasoning_language", |
| 248 | "settings_provider_count", |
| 249 | "settings_provider_access_count", |
| 250 | "settings_provider_access", |
| 251 | "settings_bot_enabled", |
| 252 | "settings_bot_model", |
| 253 | "settings_bot_tool_approval", |
| 254 | "settings_bot_allowlist", |
| 255 | "settings_bot_allow_all", |
| 256 | "settings_bot_qq_enabled", |
| 257 | "settings_bot_feishu_enabled", |
| 258 | "settings_bot_weixin_enabled", |
| 259 | "settings_bot_connection_count", |
| 260 | "settings_bot_connection_provider", |
| 261 | "settings_bot_connection_enabled", |
| 262 | "settings_bot_connection_status", |
| 263 | "settings_bot_connection_model", |
| 264 | "settings_bot_connection_approval", |
| 265 | ] as const; |
| 266 | |
| 267 | type MetricSignal = (typeof METRIC_SIGNALS)[number]; |
| 268 | |
| 269 | const METRIC_SIGNAL_SET: ReadonlySet<string> = new Set(METRIC_SIGNALS); |
| 270 | |
| 271 | const KnownMetricCounter = z.object({ |
| 272 | signal: z.enum(METRIC_SIGNALS), |
| 273 | bucket: z |
| 274 | .string() |
| 275 | .min(1) |
| 276 | .max(96) |
| 277 | .regex(/^[a-z0-9_]+$/), |
| 278 | count: z.number().int().min(1).max(1_000_000), |
| 279 | }); |
| 280 | |
| 281 | const UnknownMetricCounter = z |
| 282 | .object({ |
| 283 | signal: z |
| 284 | .string() |
| 285 | .min(1) |
| 286 | .max(96) |
| 287 | .refine((signal) => !METRIC_SIGNAL_SET.has(signal)), |
| 288 | }) |
| 289 | .passthrough() |
| 290 | .transform(() => null); |
| 291 | |
| 292 | export const Metrics = z.object({ |
| 293 | version: z.string().min(1).max(64), |
| 294 | os: z.string().min(1).max(32), |
| 295 | arch: z.string().max(32).optional(), |
| 296 | osBuild: z.number().int().min(0).max(1_000_000).optional(), |
| 297 | osRevision: z.number().int().min(0).max(1_000_000).optional(), |
| 298 | channel: z.string().max(32).optional(), |
| 299 | distroId: z.string().max(64).optional(), |
| 300 | distroVersion: z.string().max(64).optional(), |
| 301 | kernelVersion: z.string().max(128).optional(), |
| 302 | sessionType: z.enum(["wayland", "x11", "remote", "unknown"]).optional(), |
| 303 | runtimeEngine: z.enum(["webview2", "webkitgtk", "unknown"]).optional(), |
| 304 | runtimeVersion: z.string().max(128).optional(), |
| 305 | gpuMode: z.enum(["enabled", "disabled", "always", "on_demand", "unknown"]).optional(), |
| 306 | surface: ClientSurface.default("desktop"), |
| 307 | counters: z |
| 308 | .array(z.union([KnownMetricCounter, UnknownMetricCounter])) |
| 309 | .min(1) |
| 310 | .max(128) |
| 311 | .transform((counters) => |
| 312 | counters.filter( |
| 313 | (counter): counter is z.infer<typeof KnownMetricCounter> & { signal: MetricSignal } => counter !== null, |
| 314 | ), |
| 315 | ), |
| 316 | }); |
| 317 | |
| 318 | type FingerprintInput = { |
| 319 | kind: string; |
| 320 | message: string; |
| 321 | source?: string; |
| 322 | label?: string; |
| 323 | errorType?: string; |
| 324 | errorMessage?: string; |
| 325 | topFrame?: string; |
| 326 | fingerprintHint?: string; |
| 327 | }; |
| 328 | |
| 329 | export function scrubSensitiveText(input: string): string { |
| 330 | return input |
| 331 | .replace(/([A-Z]:\\Users\\)[^/\\:\s"']+/gi, "$1_") |
| 332 | .replace(/(\/(?:home|Users)\/)[^/\\:\s"']+/g, "$1_") |
| 333 | .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, "[redacted-email]") |
| 334 | .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/gi, "Bearer [redacted]") |
| 335 | .replace( |
| 336 | /\b(api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|authorization|secret|password|passwd|pwd|token)\b\s*[:=]\s*(?:Bearer\s+)?['"]?[^'"\s,;]+['"]?/gi, |
| 337 | "$1=[redacted]", |
| 338 | ) |
| 339 | .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[redacted-jwt]") |
| 340 | .replace(/\b(?:sk|rk)-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, "[redacted-key]") |
| 341 | .replace(/\b[0-9a-fA-F]{32,}\b/g, "[redacted-hex]") |
| 342 | .replace(/[A-Za-z0-9+/]{40,}={0,2}/g, "[redacted-token]") |
| 343 | .replace(/\b[A-Za-z0-9_-]{48,}\b/g, "[redacted-token]"); |
| 344 | } |
| 345 | |
| 346 | function normalizeStackFrame(frame: string): string { |
| 347 | return frame |
| 348 | .replace(/[A-Za-z]:\\[^\s)('"]+/g, "<path>") |
| 349 | .replace(/\/(?:home|Users)\/[^\s)('"]+/g, "/<home>") |
| 350 | .replace(/(?:wails|https?|file):\/\/[^\s)('"]+/g, "<url>") |
| 351 | .replace(/0x[0-9a-fA-F]+/g, "<addr>") |
| 352 | .replace(/:\d+(?::\d+)?/g, ":<n>"); |
| 353 | } |
| 354 | |
| 355 | function normalizeFingerprintText(text: string): string { |
| 356 | return text |
| 357 | .replace(/[A-Za-z]:\\[^\s)('"]+/g, "<path>") |
| 358 | .replace(/(?:wails|https?|file):\/\/[^\s)('"]+/g, "<url>") |
| 359 | .replace(/0x[0-9a-fA-F]+/g, "<addr>") |
| 360 | .replace(/^build [0-9a-f]+$/gm, "build <commit>") |
| 361 | .replace(/:\d+(?::\d+)?/g, ":<n>"); |
| 362 | } |
| 363 | |
| 364 | export function normalizeForFingerprint(inputOrKind: FingerprintInput | string, legacyMessage = ""): string { |
| 365 | if (typeof inputOrKind === "string") { |
| 366 | const head = legacyMessage.split("\n").slice(0, 12).join("\n"); |
| 367 | return inputOrKind + "\n" + normalizeFingerprintText(head); |
| 368 | } |
| 369 | const input = inputOrKind; |
| 370 | const messageBasis = input.errorMessage || input.message; |
| 371 | const head = messageBasis.split("\n").slice(0, 6).join("\n"); |
| 372 | return ( |
| 373 | input.kind + |
| 374 | "\n" + |
| 375 | (input.source || "legacy") + |
| 376 | "\n" + |
| 377 | (input.label || "") + |
| 378 | "\n" + |
| 379 | (input.errorType || "") + |
| 380 | "\n" + |
| 381 | normalizeStackFrame(input.topFrame || "") + |
| 382 | "\n" + |
| 383 | (input.fingerprintHint ? `${input.fingerprintHint}\n` : "") + |
| 384 | normalizeFingerprintText(head) |
| 385 | ); |
| 386 | } |
| 387 | |
| 388 | export function nativeWebRuntimeFingerprintBasis(input: { |
| 389 | engine: string; |
| 390 | kind: string; |
| 391 | reason: string; |
| 392 | exitCode?: number; |
| 393 | }): string { |
| 394 | const kind = normalizeRuntimeBucket(input.engine, "kind", input.kind); |
| 395 | const reason = normalizeRuntimeBucket(input.engine, "reason", input.reason); |
| 396 | const normalizedExitCode = input.engine === "webview2" && kind === "render_process_unresponsive" && input.exitCode === 259 ? undefined : input.exitCode; |
| 397 | const exitCode = normalizedExitCode === undefined ? "unknown" : String(normalizedExitCode); |
| 398 | return [input.engine, kind, reason, exitCode].join("\n"); |
| 399 | } |
| 400 | |
| 401 | type NormalizedWebRuntime = z.infer<typeof WebRuntimeDiagnostic>; |
| 402 | |
| 403 | function basenameOnly(value: string | undefined): string { |
| 404 | return (value ?? "").split(/[\\/]/).pop()?.slice(0, 255) ?? ""; |
| 405 | } |
| 406 | |
| 407 | function normalizeRuntimeBucket(engine: string, field: "kind" | "reason", input: string): string { |
| 408 | const buckets = engine === "webview2" |
| 409 | ? field === "kind" |
| 410 | ? ["browser_process_exited", "render_process_exited", "render_process_unresponsive", "frame_render_process_exited", "utility_process_exited", "sandbox_helper_process_exited", "gpu_process_exited", "ppapi_plugin_process_exited", "ppapi_broker_process_exited", "unknown_process_exited", "unknown"] |
| 411 | : ["unexpected", "unresponsive", "terminated", "crashed", "launch_failed", "out_of_memory", "profile_deleted", "normal_exit", "abnormal_exit", "integrity_failure", "unknown"] |
| 412 | : field === "kind" |
| 413 | ? ["web_process", "unknown"] |
| 414 | : ["crashed", "out_of_memory", "terminated_by_api", "unknown"]; |
| 415 | const value = input.trim().toLowerCase(); |
| 416 | return buckets.includes(value) ? value : "unknown"; |
| 417 | } |
| 418 | |
| 419 | function normalizedWebRuntime(r: ReportPayload): NormalizedWebRuntime | undefined { |
| 420 | const input: NormalizedWebRuntime | undefined = r.webRuntime ?? (r.webview2 |
| 421 | ? { |
| 422 | engine: "webview2", |
| 423 | kind: r.webview2.kind, |
| 424 | reason: r.webview2.reason, |
| 425 | exitCode: r.webview2.exitCode, |
| 426 | processDescription: r.webview2.processDescription, |
| 427 | failureSourceModule: r.webview2.failureSourceModule, |
| 428 | runtimeVersion: r.webview2.runtimeVersion, |
| 429 | gpuMode: r.webview2.gpuDisabled ? "disabled" : "enabled", |
| 430 | recovery: r.webview2.recovery, |
| 431 | } |
| 432 | : undefined); |
| 433 | if (!input) return undefined; |
| 434 | return { |
| 435 | ...input, |
| 436 | kind: normalizeRuntimeBucket(input.engine, "kind", input.kind), |
| 437 | reason: normalizeRuntimeBucket(input.engine, "reason", input.reason), |
| 438 | runtimeVersion: input.runtimeVersion.trim() || "unknown", |
| 439 | exitCode: input.engine === "webview2" && normalizeRuntimeBucket(input.engine, "kind", input.kind) === "render_process_unresponsive" && input.exitCode === 259 ? undefined : input.exitCode, |
| 440 | processDescription: scrubSensitiveText(input.processDescription ?? "").slice(0, 255), |
| 441 | failureSourceModule: basenameOnly(input.failureSourceModule), |
| 442 | }; |
| 443 | } |
| 444 | |
| 445 | function hasStructuredCrashFields(r: ReportPayload): boolean { |
| 446 | return Boolean( |
| 447 | r.schemaVersion || |
| 448 | r.source || |
| 449 | r.label || |
| 450 | r.errorType || |
| 451 | r.errorMessage || |
| 452 | r.stack || |
| 453 | r.componentStack || |
| 454 | r.topFrame || |
| 455 | r.fingerprintHint || |
| 456 | r.buildCommit || |
| 457 | r.channel || |
| 458 | r.language || |
| 459 | r.view || |
| 460 | r.breadcrumbs?.length || |
| 461 | r.occurredAt, |
| 462 | ); |
| 463 | } |
| 464 | |
| 465 | async function sha256Hex(s: string): Promise<string> { |
| 466 | const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(s)); |
| 467 | return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| 468 | } |
| 469 | |
| 470 | async function readJSON(request: Request): Promise<unknown | Response> { |
| 471 | const length = Number(request.headers.get("content-length") ?? "0"); |
| 472 | if (!length || length > MAX_BODY_BYTES) return new Response("payload too large", { status: 413 }); |
| 473 | try { |
| 474 | return JSON.parse(await request.text()); |
| 475 | } catch { |
| 476 | return new Response("bad request", { status: 400 }); |
| 477 | } |
| 478 | } |
| 479 | |
| 480 | // Storage operations surface a deliberate 503 with a loud but credential-free |
| 481 | // log instead of an opaque worker exception, so clients retain retryable state. |
| 482 | function storageUnavailable(op: string, err: unknown): Response { |
| 483 | console.error(`${op}: storage unavailable`, err); |
| 484 | return new Response("storage unavailable", { status: 503 }); |
| 485 | } |
| 486 | |
| 487 | async function prepareCrashEvent(r: ReportPayload, keepD1Sample: boolean): Promise<StoredCrashEvent> { |
| 488 | const message = scrubSensitiveText(r.message); |
| 489 | const errorMessage = scrubSensitiveText(r.errorMessage ?? ""); |
| 490 | const stack = scrubSensitiveText(r.stack ?? ""); |
| 491 | const componentStack = scrubSensitiveText(r.componentStack ?? ""); |
| 492 | const topFrame = scrubSensitiveText(r.topFrame ?? ""); |
| 493 | const fingerprintHint = scrubSensitiveText(r.fingerprintHint ?? ""); |
| 494 | const view = scrubSensitiveText(r.view ?? ""); |
| 495 | const breadcrumbs = (r.breadcrumbs ?? []).map((breadcrumb) => ({ |
| 496 | ...breadcrumb, |
| 497 | msg: breadcrumb.msg ? scrubSensitiveText(breadcrumb.msg) : breadcrumb.msg, |
| 498 | })); |
| 499 | const webRuntime = normalizedWebRuntime(r); |
| 500 | const webview2 = r.webview2 |
| 501 | ? { |
| 502 | ...r.webview2, |
| 503 | processDescription: scrubSensitiveText(r.webview2.processDescription ?? "").slice(0, 255), |
| 504 | failureSourceModule: basenameOnly(r.webview2.failureSourceModule), |
| 505 | } |
| 506 | : undefined; |
| 507 | const report: ReportPayload = { |
| 508 | ...r, |
| 509 | eventId: r.eventId ?? crypto.randomUUID().replaceAll("-", ""), |
| 510 | message, |
| 511 | errorMessage, |
| 512 | stack, |
| 513 | componentStack, |
| 514 | topFrame, |
| 515 | fingerprintHint, |
| 516 | view, |
| 517 | breadcrumbs, |
| 518 | webRuntime, |
| 519 | webview2, |
| 520 | }; |
| 521 | const fingerprintBasis = ( |
| 522 | report.source === "web.runtime.native" || report.source === "webview2.process.native" |
| 523 | ) && webRuntime |
| 524 | ? nativeWebRuntimeFingerprintBasis(webRuntime) |
| 525 | : hasStructuredCrashFields(report) |
| 526 | ? normalizeForFingerprint({ |
| 527 | kind: report.kind, |
| 528 | message, |
| 529 | source: report.source, |
| 530 | label: report.label, |
| 531 | errorType: report.errorType, |
| 532 | errorMessage, |
| 533 | topFrame, |
| 534 | fingerprintHint, |
| 535 | }) |
| 536 | : normalizeForFingerprint(report.kind, message); |
| 537 | const severityInput = { |
| 538 | kind: report.kind, |
| 539 | version: reportSubjectIdentity(report).version, |
| 540 | source: report.source ?? "legacy", |
| 541 | label: report.label ?? "", |
| 542 | errorType: report.errorType ?? "", |
| 543 | errorMessage, |
| 544 | topFrame, |
| 545 | channel: reportSubjectIdentity(report).channel, |
| 546 | recovery: webRuntime?.recovery, |
| 547 | }; |
| 548 | const development = isDevelopmentReport(severityInput); |
| 549 | return { |
| 550 | eventId: report.eventId!, |
| 551 | fingerprint: namespaceReportFingerprint(await sha256Hex(fingerprintBasis), development), |
| 552 | receivedAt: new Date().toISOString(), |
| 553 | keepD1Sample, |
| 554 | report, |
| 555 | }; |
| 556 | } |
| 557 | |
| 558 | async function projectCrashEvent(env: Env, event: StoredCrashEvent): Promise<void> { |
| 559 | const existingEvent = await env.DB.prepare("SELECT event_id FROM report_events WHERE event_id = ?1") |
| 560 | .bind(event.eventId) |
| 561 | .first<{ event_id: string }>(); |
| 562 | if (existingEvent) return; |
| 563 | const firebaseDelivery = crashStorageMode(env) !== "d1"; |
| 564 | if (firebaseDelivery && (await firebaseProjectionExists(env, event.eventId))) { |
| 565 | await env.DB.prepare( |
| 566 | "UPDATE firebase_crash_outbox SET state = 'projected', updated_at = ?2 WHERE event_id = ?1", |
| 567 | ).bind(event.eventId, new Date().toISOString()).run(); |
| 568 | return; |
| 569 | } |
| 570 | const r = event.report; |
| 571 | const webRuntime = normalizedWebRuntime(r); |
| 572 | const webview2 = r.webview2; |
| 573 | const message = r.message; |
| 574 | const errorMessage = r.errorMessage ?? ""; |
| 575 | const topFrame = r.topFrame ?? ""; |
| 576 | const source = r.source ?? "legacy"; |
| 577 | const label = r.label ?? ""; |
| 578 | const errorType = r.errorType ?? ""; |
| 579 | const observerBuildCommit = r.buildCommit ?? ""; |
| 580 | const channel = r.channel ?? ""; |
| 581 | const subjectIdentity = reportSubjectIdentity(r); |
| 582 | const subjectVersion = subjectIdentity.version; |
| 583 | const subjectChannel = subjectIdentity.channel; |
| 584 | const buildCommit = r.diagnostics?.subjectBuildCommit || observerBuildCommit; |
| 585 | const severity = severityForReport({ |
| 586 | kind: r.kind, |
| 587 | version: subjectVersion, |
| 588 | source, |
| 589 | label, |
| 590 | errorType, |
| 591 | errorMessage, |
| 592 | topFrame, |
| 593 | channel: subjectChannel, |
| 594 | recovery: webRuntime?.recovery, |
| 595 | }); |
| 596 | const prior = await env.DB.prepare( |
| 597 | "SELECT status, resolved_in, resolution_platform, resolution_runtime FROM groups WHERE fingerprint = ?1", |
| 598 | ) |
| 599 | .bind(event.fingerprint) |
| 600 | .first<{ |
| 601 | status: string; |
| 602 | resolved_in: string; |
| 603 | resolution_platform: string; |
| 604 | resolution_runtime: string; |
| 605 | }>(); |
| 606 | const runtime = webRuntime?.engine ?? ""; |
| 607 | const regressionDecision = regressionDecisionForReport({ |
| 608 | status: prior?.status, |
| 609 | fixedIn: prior?.resolved_in, |
| 610 | resolutionPlatform: prior?.resolution_platform, |
| 611 | resolutionRuntime: prior?.resolution_runtime, |
| 612 | subjectVersion, |
| 613 | os: r.os, |
| 614 | runtime, |
| 615 | }); |
| 616 | const regressedAt = regressionDecision === "confirmed" ? event.receivedAt : ""; |
| 617 | const groupWrite = env.DB.prepare( |
| 618 | `INSERT INTO groups ( |
| 619 | fingerprint, kind, count, first_seen, last_seen, first_version, last_version, |
| 620 | status, title, source, label, error_type, top_frame, severity, |
| 621 | last_os, last_arch, last_build_commit, last_channel, last_sample_at, |
| 622 | regressed_at, regression_review, last_category |
| 623 | ) |
| 624 | VALUES (?1, ?2, 1, ?3, ?3, ?4, ?4, 'open', ?5, ?6, ?7, ?8, ?9, ?10, |
| 625 | ?11, ?12, ?13, ?14, ?3, ?15, '', ?17) |
| 626 | ON CONFLICT (fingerprint) DO UPDATE SET |
| 627 | kind = CASE |
| 628 | WHEN severity = 'critical' THEN kind |
| 629 | WHEN (CASE ?10 WHEN 'critical' THEN 4 WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END) > |
| 630 | (CASE severity WHEN 'critical' THEN 4 WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END) |
| 631 | THEN ?2 ELSE kind END, |
| 632 | count = count + 1, last_seen = ?3, last_version = ?4, title = ?5, |
| 633 | source = ?6, label = ?7, error_type = ?8, top_frame = ?9, |
| 634 | severity = CASE |
| 635 | WHEN severity = 'critical' THEN severity |
| 636 | WHEN (CASE ?10 WHEN 'critical' THEN 4 WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END) > |
| 637 | (CASE severity WHEN 'critical' THEN 4 WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END) |
| 638 | THEN ?10 ELSE severity END, |
| 639 | last_os = ?11, last_arch = ?12, last_build_commit = ?13, last_channel = ?14, |
| 640 | last_sample_at = ?3, |
| 641 | status = CASE WHEN status = 'resolved' AND ?16 = 'confirmed' THEN 'open' ELSE status END, |
| 642 | regressed_at = CASE WHEN status = 'resolved' AND ?16 = 'confirmed' THEN ?3 ELSE regressed_at END, |
| 643 | regression_review = CASE |
| 644 | WHEN status = 'resolved' AND ?16 = 'suspected' THEN 'suspected' |
| 645 | WHEN ?16 = 'confirmed' THEN '' |
| 646 | ELSE regression_review END, |
| 647 | last_category = ?17`, |
| 648 | ).bind( |
| 649 | event.fingerprint, r.kind, event.receivedAt, subjectVersion, crashTitle(message), source, |
| 650 | label, errorType, topFrame, severity, r.os, r.arch, buildCommit, subjectChannel, |
| 651 | regressedAt, regressionDecision, r.diagnostics?.category ?? "", |
| 652 | ); |
| 653 | const statements: D1PreparedStatement[] = [ |
| 654 | env.DB.prepare( |
| 655 | "INSERT INTO report_events (event_id, incident_id, fingerprint, received_at, projected_at) VALUES (?1, ?2, ?3, ?4, ?4)", |
| 656 | ).bind(event.eventId, r.diagnostics?.incidentId ?? "", event.fingerprint, event.receivedAt), |
| 657 | env.DB.prepare( |
| 658 | `INSERT INTO audit_log (at, actor_id, actor_email, action, target, detail) |
| 659 | SELECT ?1, NULL, 'system@reasonix.local', 'automatic_regression', ?2, ?3 |
| 660 | FROM groups |
| 661 | WHERE fingerprint = ?4 AND status = 'resolved' AND ?5 = 'confirmed'`, |
| 662 | ).bind( |
| 663 | event.receivedAt, |
| 664 | event.fingerprint.slice(0, 8), |
| 665 | JSON.stringify({ |
| 666 | subjectVersion, |
| 667 | fixedIn: prior?.resolved_in ?? "", |
| 668 | platform: r.os, |
| 669 | runtime, |
| 670 | }), |
| 671 | event.fingerprint, |
| 672 | regressionDecision, |
| 673 | ), |
| 674 | groupWrite, |
| 675 | ]; |
| 676 | if (event.keepD1Sample) { |
| 677 | statements.push(env.DB.prepare( |
| 678 | `INSERT INTO reports ( |
| 679 | fingerprint, kind, version, os, arch, message, device, created_at, |
| 680 | source, label, error_type, error_message, error_family, top_frame, build_commit, channel, |
| 681 | language, view, breadcrumbs, component_stack, stack, occurred_at, webview2, web_runtime, |
| 682 | event_id, incident_id, diagnostics |
| 683 | ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, |
| 684 | ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27)`, |
| 685 | ).bind( |
| 686 | event.fingerprint, r.kind, subjectVersion, r.os, r.arch, message, |
| 687 | JSON.stringify(r.device ?? {}), event.receivedAt, source, label, errorType, errorMessage, r.errorFamily ?? "", |
| 688 | topFrame, buildCommit, subjectChannel, r.language ?? "", r.view ?? "", |
| 689 | JSON.stringify(r.breadcrumbs ?? []), r.componentStack ?? "", r.stack ?? "", |
| 690 | r.occurredAt ?? "", webview2 ? JSON.stringify(webview2) : "", |
| 691 | webRuntime ? JSON.stringify(webRuntime) : "", event.eventId, |
| 692 | r.diagnostics?.incidentId ?? "", r.diagnostics ? JSON.stringify(r.diagnostics) : "", |
| 693 | )); |
| 694 | } |
| 695 | statements.push(...reportAggregateStatements(env.DB, r, event.fingerprint, channel, webRuntime)); |
| 696 | statements.push( |
| 697 | env.DB.prepare( |
| 698 | `INSERT INTO report_attribution_daily ( |
| 699 | date, fingerprint, subject_version, observer_version, subject_channel, observer_channel, category, evidence, events |
| 700 | ) VALUES (date('now'), ?1, ?2, ?3, ?4, ?5, ?6, ?7, 1) |
| 701 | ON CONFLICT (date, fingerprint, subject_version, observer_version, subject_channel, observer_channel, category, evidence) |
| 702 | DO UPDATE SET events = events + 1`, |
| 703 | ).bind( |
| 704 | event.fingerprint, |
| 705 | subjectVersion, |
| 706 | r.diagnostics?.observerVersion || r.version, |
| 707 | subjectChannel, |
| 708 | channel, |
| 709 | r.diagnostics?.category ?? "", |
| 710 | r.diagnostics?.evidence ?? "", |
| 711 | ), |
| 712 | ); |
| 713 | if (r.diagnostics?.incidentId) { |
| 714 | statements.push( |
| 715 | env.DB.prepare( |
| 716 | "INSERT OR IGNORE INTO report_incidents (date, fingerprint, incident_id, subject_version) VALUES (date('now'), ?1, ?2, ?3)", |
| 717 | ).bind(event.fingerprint, r.diagnostics.incidentId, subjectVersion), |
| 718 | ); |
| 719 | } |
| 720 | if (event.keepD1Sample) { |
| 721 | statements.push(env.DB.prepare( |
| 722 | `DELETE FROM reports WHERE fingerprint = ?1 AND id NOT IN ( |
| 723 | SELECT id FROM (SELECT id FROM reports WHERE fingerprint = ?1 ORDER BY id ASC LIMIT 1) |
| 724 | UNION |
| 725 | SELECT id FROM (SELECT id FROM reports WHERE fingerprint = ?1 ORDER BY id DESC LIMIT ?2) |
| 726 | )`, |
| 727 | ).bind(event.fingerprint, LATEST_SAMPLES_PER_GROUP)); |
| 728 | } |
| 729 | if (firebaseDelivery) { |
| 730 | statements.push(...projectionCompletionStatements( |
| 731 | env.DB, event.eventId, event.fingerprint, event.receivedAt, |
| 732 | )); |
| 733 | } |
| 734 | await env.DB.batch(statements); |
| 735 | } |
| 736 | |
| 737 | export async function drainFirebaseCrashOutbox(env: Env): Promise<void> { |
| 738 | return drainFirebaseOutbox(env, projectCrashEvent); |
| 739 | } |
| 740 | |
| 741 | async function handleReport(request: Request, env: Env): Promise<Response> { |
| 742 | const ip = request.headers.get("cf-connecting-ip") ?? "unknown"; |
| 743 | const { success } = await env.RATE_LIMITER.limit({ key: ip }); |
| 744 | if (!success) return new Response("rate limited", { status: 429 }); |
| 745 | |
| 746 | const raw = await readJSON(request); |
| 747 | if (raw instanceof Response) return raw; |
| 748 | const parsed = Report.safeParse(raw); |
| 749 | if (!parsed.success) return new Response("bad request", { status: 400 }); |
| 750 | let mode; |
| 751 | try { |
| 752 | mode = crashStorageMode(env); |
| 753 | if (!firebaseStorageReady(env)) throw new Error("firebase crash storage is not configured"); |
| 754 | } catch (err) { |
| 755 | console.error("report: crash storage configuration failed", err); |
| 756 | return new Response("storage unavailable", { status: 503 }); |
| 757 | } |
| 758 | const event = await prepareCrashEvent(parsed.data, mode !== "firebase"); |
| 759 | if (mode !== "d1") { |
| 760 | let lease: FirebaseGroupLease | null = null; |
| 761 | try { |
| 762 | if (await firebaseEventExists(env, event.eventId)) return new Response("ok", { status: 202 }); |
| 763 | if (await reserveFirebaseGroup(env, event.fingerprint, event.receivedAt) === "full") { |
| 764 | return new Response("storage unavailable", { status: 503 }); |
| 765 | } |
| 766 | const enqueued = await enqueueFirebaseCrash( |
| 767 | env, event.eventId, event.fingerprint, JSON.stringify(event), event.receivedAt, |
| 768 | ); |
| 769 | if (enqueued === "duplicate") return new Response("ok", { status: 202 }); |
| 770 | if (enqueued === "full") { |
| 771 | await reclaimUnusedFirebaseReservation(env, event.fingerprint); |
| 772 | return new Response("storage unavailable", { status: 503 }); |
| 773 | } |
| 774 | lease = await acquireFirebaseGroupLease(env, event.fingerprint); |
| 775 | } catch (err) { |
| 776 | return storageUnavailable("report outbox", err); |
| 777 | } |
| 778 | if (!lease) return new Response("ok", { status: 202 }); |
| 779 | try { |
| 780 | if (!await claimFirebaseCrash(env, event.eventId, new Date().toISOString())) { |
| 781 | return new Response("ok", { status: 202 }); |
| 782 | } |
| 783 | try { |
| 784 | await projectCrashEvent(env, event); |
| 785 | } catch (err) { |
| 786 | console.error("report: buffered D1 projection failed", err); |
| 787 | await recordFirebaseRetry(env, event.eventId, "queued", 0); |
| 788 | return new Response("ok", { status: 202 }); |
| 789 | } |
| 790 | await deliverCrashEventToFirebase(env, event, 0, lease); |
| 791 | return new Response("ok", { status: 202 }); |
| 792 | } finally { |
| 793 | await releaseFirebaseGroupLease(env, event.fingerprint, lease).catch((error) => { |
| 794 | console.error("firebase crash group lease release failed", error); |
| 795 | }); |
| 796 | } |
| 797 | } |
| 798 | try { |
| 799 | await projectCrashEvent(env, event); |
| 800 | } catch (err) { |
| 801 | return storageUnavailable("report", err); |
| 802 | } |
| 803 | return new Response("ok", { status: 202 }); |
| 804 | } |
| 805 | |
| 806 | async function handlePing(request: Request, env: Env): Promise<Response> { |
| 807 | const ip = request.headers.get("cf-connecting-ip") ?? "unknown"; |
| 808 | const { success } = await env.PING_LIMITER.limit({ key: ip }); |
| 809 | if (!success) return new Response("rate limited", { status: 429 }); |
| 810 | |
| 811 | const raw = await readJSON(request); |
| 812 | if (raw instanceof Response) return raw; |
| 813 | const parsed = Ping.safeParse(raw); |
| 814 | if (!parsed.success) return new Response("bad request", { status: 400 }); |
| 815 | const p = parsed.data; |
| 816 | const tables = telemetryTableNames(p.surface); |
| 817 | |
| 818 | try { |
| 819 | if (p.surface === "cli") await ensureCLITelemetrySchema(env); |
| 820 | await env.DB.prepare( |
| 821 | `INSERT INTO ${tables.pings} ( |
| 822 | date, install_id, version, os, arch, os_version, os_build, os_revision, channel, |
| 823 | distro_id, distro_version, kernel_version, session_type, runtime_engine, runtime_version, gpu_mode, opens |
| 824 | ) |
| 825 | VALUES (date('now'), ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, 1) |
| 826 | ON CONFLICT (date, install_id) DO UPDATE SET |
| 827 | opens = opens + 1, version = ?2, os_version = ?5, os_build = ?6, os_revision = ?7, |
| 828 | channel = ?8, distro_id = ?9, distro_version = ?10, kernel_version = ?11, |
| 829 | session_type = ?12, runtime_engine = ?13, runtime_version = ?14, gpu_mode = ?15`, |
| 830 | ) |
| 831 | .bind( |
| 832 | p.installId, p.version, p.os, p.arch, p.osVersion ?? "", p.osBuild ?? 0, p.osRevision ?? 0, |
| 833 | p.channel ?? "", p.distroId ?? "", p.distroVersion ?? "", p.kernelVersion ?? "", |
| 834 | p.sessionType ?? "", p.runtimeEngine ?? "", p.runtimeVersion ?? "", p.gpuMode ?? "", |
| 835 | ) |
| 836 | .run(); |
| 837 | } catch (err) { |
| 838 | return storageUnavailable("ping", err); |
| 839 | } |
| 840 | |
| 841 | return new Response("ok", { status: 202 }); |
| 842 | } |
| 843 | |
| 844 | async function handleMetrics(request: Request, env: Env): Promise<Response> { |
| 845 | const ip = request.headers.get("cf-connecting-ip") ?? "unknown"; |
| 846 | const { success } = await env.METRICS_LIMITER.limit({ key: ip }); |
| 847 | if (!success) return new Response("rate limited", { status: 429 }); |
| 848 | |
| 849 | const raw = await readJSON(request); |
| 850 | if (raw instanceof Response) return raw; |
| 851 | const parsed = Metrics.safeParse(raw); |
| 852 | if (!parsed.success) return new Response("bad request", { status: 400 }); |
| 853 | const m = parsed.data; |
| 854 | if (m.counters.length === 0) return new Response("ok", { status: 202 }); |
| 855 | const tables = telemetryTableNames(m.surface); |
| 856 | |
| 857 | try { |
| 858 | if (m.surface === "cli") await ensureCLITelemetrySchema(env); |
| 859 | const upsert = env.DB.prepare( |
| 860 | `INSERT INTO ${tables.metrics} (date, version, os, signal, bucket, count) |
| 861 | VALUES (date('now'), ?1, ?2, ?3, ?4, ?5) |
| 862 | ON CONFLICT (date, version, os, signal, bucket) DO UPDATE SET |
| 863 | count = count + ?5`, |
| 864 | ); |
| 865 | await env.DB.batch(m.counters.map((c) => upsert.bind(m.version, m.os, c.signal, c.bucket, c.count))); |
| 866 | } catch (err) { |
| 867 | return storageUnavailable("metrics", err); |
| 868 | } |
| 869 | return new Response("ok", { status: 202 }); |
| 870 | } |
| 871 | |
| 872 | const UserAction = z.object({ |
| 873 | action: z.enum(["role", "delete"]), |
| 874 | userId: z.coerce.number().int().positive(), |
| 875 | role: z.enum(["pending", "viewer", "admin"]).optional(), |
| 876 | }); |
| 877 | |
| 878 | const GroupAction = z.object({ |
| 879 | action: z.enum(["status", "delete", "note", "resolution", "severity"]), |
| 880 | status: z.enum(["open", "resolved", "ignored"]).optional(), |
| 881 | note: z.string().max(500).optional(), |
| 882 | resolvedIn: z.string().max(64).optional(), |
| 883 | resolutionPlatform: z.string().max(32).optional(), |
| 884 | resolutionRuntime: z.string().max(32).optional(), |
| 885 | resolutionBasis: z.string().max(500).optional(), |
| 886 | severity: z.enum(["low", "medium", "high", "critical"]).optional(), |
| 887 | }); |
| 888 | |
| 889 | async function formObject(request: Request): Promise<Record<string, string>> { |
| 890 | const form = await request.formData(); |
| 891 | const out: Record<string, string> = {}; |
| 892 | for (const [k, v] of form) out[k] = typeof v === "string" ? v : ""; |
| 893 | return out; |
| 894 | } |
| 895 | |
| 896 | async function latestObservedVersion(env: Env, surface: ClientSurfaceName): Promise<string> { |
| 897 | const table = telemetryTableNames(surface).pings; |
| 898 | // Require independent installations and use pings as the sole source of |
| 899 | // release truth. A single synthetic diagnostic must never promote v9.9.9 (or |
| 900 | // a prerelease) to "latest" for every report group. |
| 901 | const sql = `SELECT version FROM ${table} |
| 902 | WHERE date >= date('now', '-29 day') AND version <> '' |
| 903 | GROUP BY version HAVING COUNT(DISTINCT install_id) >= 2`; |
| 904 | const rows = await env.DB.prepare(sql).all<{ version: string }>(); |
| 905 | return newestReleaseVersion(rows.results.map((r) => r.version)); |
| 906 | } |
| 907 | |
| 908 | type OverviewCounts = { |
| 909 | latestAdoptionPct: number | null; |
| 910 | openReports: number; |
| 911 | newLatestReports: number; |
| 912 | regressedReports: number; |
| 913 | criticalOpenReports: number; |
| 914 | }; |
| 915 | |
| 916 | async function latestAdoptionPct(env: Env, latestVersion: string, days: 7 | 30, surface: ClientSurfaceName): Promise<number | null> { |
| 917 | if (!latestVersion) return null; |
| 918 | const table = telemetryTableNames(surface).pings; |
| 919 | const row = await env.DB.prepare( |
| 920 | `SELECT |
| 921 | COUNT(DISTINCT install_id) AS total_installs, |
| 922 | COUNT(DISTINCT CASE WHEN version = ?1 THEN install_id END) AS latest_installs |
| 923 | FROM ${table} WHERE date >= date('now', '${currentWindowSince(days)}')`, |
| 924 | ) |
| 925 | .bind(latestVersion) |
| 926 | .first<{ total_installs: number; latest_installs: number }>(); |
| 927 | const total = Number(row?.total_installs ?? 0); |
| 928 | if (!total) return null; |
| 929 | return (Number(row?.latest_installs ?? 0) / total) * 100; |
| 930 | } |
| 931 | |
| 932 | async function diagnosticOverview(env: Env, latestVersion: string, days: 7 | 30, surface: ClientSurfaceName): Promise<OverviewCounts> { |
| 933 | if (surface === "cli") { |
| 934 | return { |
| 935 | latestAdoptionPct: await latestAdoptionPct(env, latestVersion, days, surface), |
| 936 | openReports: 0, |
| 937 | newLatestReports: 0, |
| 938 | regressedReports: 0, |
| 939 | criticalOpenReports: 0, |
| 940 | }; |
| 941 | } |
| 942 | // Keep the overview's red state aligned with the effective severity used by |
| 943 | // the diagnostics list. Historical rows retain their stored severity, so |
| 944 | // known browser notices and development builds must be discounted here too. |
| 945 | const criticalActionable = `(severity = 'critical' OR ( |
| 946 | severity = 'high' |
| 947 | AND kind <> 'performance' |
| 948 | AND NOT ${developmentGroupSQL} |
| 949 | AND title <> '[window.error] Script error.' |
| 950 | AND title NOT LIKE '%ResizeObserver loop %' |
| 951 | AND title NOT LIKE '%Minified React error #520%' |
| 952 | AND title NOT LIKE '%additional File object is not a file on the disk%' |
| 953 | ))`; |
| 954 | const diagnosticCounts = latestVersion |
| 955 | ? env.DB.prepare( |
| 956 | `SELECT |
| 957 | SUM(CASE WHEN status = 'open' THEN 1 ELSE 0 END) AS open_reports, |
| 958 | SUM(CASE WHEN first_version = ?1 THEN 1 ELSE 0 END) AS new_latest_reports, |
| 959 | SUM(CASE WHEN regressed_at <> '' THEN 1 ELSE 0 END) AS regressed_reports, |
| 960 | SUM(CASE WHEN status = 'open' AND ${criticalActionable} THEN 1 ELSE 0 END) AS critical_open_reports |
| 961 | FROM groups WHERE ${diagnosticWindowWhere(days)}`, |
| 962 | ) |
| 963 | .bind(latestVersion) |
| 964 | .first<{ open_reports: number; new_latest_reports: number; regressed_reports: number; critical_open_reports: number }>() |
| 965 | : env.DB.prepare( |
| 966 | `SELECT |
| 967 | SUM(CASE WHEN status = 'open' THEN 1 ELSE 0 END) AS open_reports, |
| 968 | 0 AS new_latest_reports, |
| 969 | SUM(CASE WHEN regressed_at <> '' THEN 1 ELSE 0 END) AS regressed_reports, |
| 970 | SUM(CASE WHEN status = 'open' AND ${criticalActionable} THEN 1 ELSE 0 END) AS critical_open_reports |
| 971 | FROM groups WHERE ${diagnosticWindowWhere(days)}`, |
| 972 | ).first<{ open_reports: number; new_latest_reports: number; regressed_reports: number; critical_open_reports: number }>(); |
| 973 | const [row, adoptionPct] = await Promise.all([ |
| 974 | diagnosticCounts, |
| 975 | latestAdoptionPct(env, latestVersion, days, surface), |
| 976 | ]); |
| 977 | return { |
| 978 | latestAdoptionPct: adoptionPct, |
| 979 | openReports: Number(row?.open_reports ?? 0), |
| 980 | newLatestReports: Number(row?.new_latest_reports ?? 0), |
| 981 | regressedReports: Number(row?.regressed_reports ?? 0), |
| 982 | criticalOpenReports: Number(row?.critical_open_reports ?? 0), |
| 983 | }; |
| 984 | } |
| 985 | |
| 986 | function previousWindowSince(days: 7 | 30): string { |
| 987 | return `-${days * 2 - 1} day`; |
| 988 | } |
| 989 | |
| 990 | function previousWindowUntil(days: 7 | 30): string { |
| 991 | return currentWindowSince(days); |
| 992 | } |
| 993 | |
| 994 | async function metricRows(env: Env, days: 7 | 30, surface: ClientSurfaceName, previous = false): Promise<{ signal: string; bucket: string; total: number }[]> { |
| 995 | const where = previous |
| 996 | ? `date >= date('now', '${previousWindowSince(days)}') AND date < date('now', '${previousWindowUntil(days)}')` |
| 997 | : `date >= date('now', '${currentWindowSince(days)}')`; |
| 998 | const table = telemetryTableNames(surface).metrics; |
| 999 | const rows = await env.DB.prepare( |
| 1000 | `SELECT signal, bucket, SUM(count) AS total FROM ${table} WHERE ${where} GROUP BY signal, bucket ORDER BY signal, total DESC`, |
| 1001 | ).all<{ signal: string; bucket: string; total: number }>(); |
| 1002 | return rows.results; |
| 1003 | } |
| 1004 | |
| 1005 | type Bar = { label: string; users: number }; |
| 1006 | type MetricTotals = { signal: string; bucket: string; total: number }[]; |
| 1007 | // Each stats module renders only its own section, so a page load queries only |
| 1008 | // what that section shows. |
| 1009 | async function handleStats(request: Request, env: Env, user: User, activeModule: StatsModule): Promise<Response> { |
| 1010 | const url = new URL(request.url); |
| 1011 | const filters = statsFilters(url); |
| 1012 | const days = filters.windowDays; |
| 1013 | const since = currentWindowSince(days); |
| 1014 | const surface = activeModule === "diagnostics" ? "desktop" : filters.surface; |
| 1015 | if (activeModule === "diagnostics") filters.surface = "desktop"; |
| 1016 | if (surface === "cli") await ensureCLITelemetrySchema(env); |
| 1017 | const pingsTable = telemetryTableNames(surface).pings; |
| 1018 | const bars = (sql: string) => env.DB.prepare(sql).all<Bar>().then((r) => r.results); |
| 1019 | const pingVersions = () => |
| 1020 | bars(`SELECT version AS label, COUNT(DISTINCT install_id) AS users FROM ${pingsTable} WHERE date >= date('now', '${since}') GROUP BY label ORDER BY users DESC LIMIT 15`); |
| 1021 | const pingPlatforms = () => |
| 1022 | bars(`SELECT os || ' ' || arch AS label, COUNT(DISTINCT install_id) AS users FROM ${pingsTable} WHERE date >= date('now', '${since}') GROUP BY label ORDER BY users DESC`); |
| 1023 | |
| 1024 | let daily: { date: string; users: number; opens: number }[] = []; |
| 1025 | let versions: Bar[] = []; |
| 1026 | let platforms: Bar[] = []; |
| 1027 | let crashes: Awaited<ReturnType<typeof crashGroups>>["results"] = []; |
| 1028 | let metrics: MetricTotals = []; |
| 1029 | let previousMetrics: MetricTotals = []; |
| 1030 | let sources: Bar[] = []; |
| 1031 | let diagnosticFacets: DiagnosticFacets = { |
| 1032 | versions: [], platforms: [], |
| 1033 | osBuilds: [], osRevisions: [], distros: [], distroVersions: [], kernels: [], sessions: [], |
| 1034 | architectures: [], channels: [], runtimes: [], runtimeEngines: [], |
| 1035 | failureKinds: [], failureReasons: [], exitCodes: [], recoveries: [], gpuStates: [], |
| 1036 | }; |
| 1037 | let installationLinkedSince = ""; |
| 1038 | let structuredAttributionSince = ""; |
| 1039 | let overview: OverviewCounts = { |
| 1040 | latestAdoptionPct: null, |
| 1041 | openReports: 0, |
| 1042 | newLatestReports: 0, |
| 1043 | regressedReports: 0, |
| 1044 | criticalOpenReports: 0, |
| 1045 | }; |
| 1046 | let latestVersion = ""; |
| 1047 | let firebaseStorage: FirebaseStorageSummary | undefined; |
| 1048 | |
| 1049 | if (activeModule === "usage") { |
| 1050 | latestVersion = await latestObservedVersion(env, surface); |
| 1051 | const [dailyR, versionsR, platformsR, metricsR, overviewR] = await Promise.all([ |
| 1052 | env.DB.prepare( |
| 1053 | `SELECT date, COUNT(*) AS users, SUM(opens) AS opens FROM ${pingsTable} WHERE date >= date('now', '${since}') GROUP BY date`, |
| 1054 | ).all<{ date: string; users: number; opens: number }>(), |
| 1055 | pingVersions(), |
| 1056 | pingPlatforms(), |
| 1057 | metricRows(env, days, surface), |
| 1058 | diagnosticOverview(env, latestVersion, days, surface), |
| 1059 | ]); |
| 1060 | daily = dailyR.results; |
| 1061 | versions = versionsR; |
| 1062 | platforms = platformsR; |
| 1063 | metrics = metricsR; |
| 1064 | overview = overviewR; |
| 1065 | } else if (activeModule === "diagnostics") { |
| 1066 | latestVersion = await latestObservedVersion(env, "desktop"); |
| 1067 | const [crashesR, sourcesR, facets, linkedSince, attributionSince] = await Promise.all([ |
| 1068 | crashGroups(env, filters, latestVersion, statsQueryObserver("/stats/diagnostics")), |
| 1069 | bars(`SELECT source AS label, COUNT(*) AS users FROM groups WHERE ${diagnosticWindowWhere(days)} GROUP BY source ORDER BY users DESC`), |
| 1070 | loadDiagnosticFacets(env, days, statsQueryObserver("/stats/diagnostics")), |
| 1071 | env.DB.prepare("SELECT value FROM diagnostics_meta WHERE key = 'installation_linked_since'").first<{ value: string }>(), |
| 1072 | env.DB.prepare("SELECT value FROM diagnostics_meta WHERE key = 'structured_attribution_since'").first<{ value: string }>(), |
| 1073 | ]); |
| 1074 | crashes = crashesR.results; |
| 1075 | sources = sourcesR; |
| 1076 | versions = facets.versions; |
| 1077 | platforms = facets.platforms; |
| 1078 | diagnosticFacets = facets; |
| 1079 | installationLinkedSince = linkedSince?.value ?? ""; |
| 1080 | structuredAttributionSince = attributionSince?.value ?? ""; |
| 1081 | if (crashStorageMode(env) !== "d1") firebaseStorage = await firebaseStorageSummary(env); |
| 1082 | } else if (activeModule === "preferences") { |
| 1083 | metrics = await metricRows(env, days, surface); |
| 1084 | } else { |
| 1085 | const [metricsR, previousMetricsR] = await Promise.all([ |
| 1086 | metricRows(env, days, surface), |
| 1087 | metricRows(env, days, surface, true), |
| 1088 | ]); |
| 1089 | metrics = metricsR; |
| 1090 | previousMetrics = previousMetricsR; |
| 1091 | } |
| 1092 | |
| 1093 | return html( |
| 1094 | renderStats( |
| 1095 | { daily, versions, platforms, crashes, metrics, previousMetrics, sources, diagnosticFacets, |
| 1096 | installationLinkedSince, |
| 1097 | structuredAttributionSince, overview, latestVersion, filters, firebaseStorage }, |
| 1098 | user, |
| 1099 | activeModule, |
| 1100 | ), |
| 1101 | ); |
| 1102 | } |
| 1103 | |
| 1104 | async function handleGroup(env: Env, fingerprint: string, user: User): Promise<Response> { |
| 1105 | const group = await env.DB.prepare("SELECT * FROM groups WHERE fingerprint = ?1").bind(fingerprint).first<Group>(); |
| 1106 | if (!group) return new Response("not found", { status: 404 }); |
| 1107 | group.severity = effectiveGroupSeverity(group); |
| 1108 | const state = crashStorageMode(env) === "d1" ? null : await firebaseGroupState(env, fingerprint); |
| 1109 | let reports: ReportSample[] = [], samplesUnavailable = false; |
| 1110 | if (crashStorageMode(env) === "firebase") { |
| 1111 | if (state?.sample_state === "archived") { |
| 1112 | reports = []; |
| 1113 | } else { |
| 1114 | try { |
| 1115 | const stored = await readFirebaseCrashGroup(env, fingerprint); |
| 1116 | reports = firebaseSamples(stored?.samples); |
| 1117 | } catch (error) { |
| 1118 | return storageUnavailable("firebase group detail", error); |
| 1119 | } |
| 1120 | } |
| 1121 | } else { |
| 1122 | const loaded = await loadD1GroupReports(env, fingerprint, LATEST_SAMPLES_PER_GROUP, statsQueryObserver("/stats/group")); |
| 1123 | reports = loaded.reports; |
| 1124 | samplesUnavailable = loaded.unavailable; |
| 1125 | } |
| 1126 | const loadedDiagnostics = await loadGroupDiagnostics(env, fingerprint, statsQueryObserver("/stats/group")); |
| 1127 | return html(renderGroup( |
| 1128 | group, reports, user, loadedDiagnostics.summary, |
| 1129 | state ? { state: state.sample_state, epoch: Number(state.sample_epoch) } : undefined, |
| 1130 | { samplesUnavailable, diagnosticsUnavailable: loadedDiagnostics.unavailable }, |
| 1131 | )); |
| 1132 | } |
| 1133 | |
| 1134 | async function syncFirebaseGroupMetaLocked( |
| 1135 | env: Env, |
| 1136 | fingerprint: string, |
| 1137 | lease?: FirebaseGroupLease, |
| 1138 | ): Promise<void> { |
| 1139 | if (crashStorageMode(env) === "d1") return; |
| 1140 | if (!lease) throw new Error("firebase crash group lease is missing"); |
| 1141 | const [group, state] = await Promise.all([ |
| 1142 | loadFirebaseGroupMeta(env, fingerprint), |
| 1143 | firebaseGroupState(env, fingerprint), |
| 1144 | ]); |
| 1145 | if (!group || !state) return; |
| 1146 | if (state.sample_state === "archived") return; |
| 1147 | await writeFirebaseGroupMeta( |
| 1148 | env, fingerprint, firebaseMeta(group), lease.generation, Number(state.sample_epoch), |
| 1149 | state.sample_state, () => renewFirebaseGroupLease(env, fingerprint, lease), |
| 1150 | ); |
| 1151 | } |
| 1152 | |
| 1153 | async function withFirebaseGroupLease<T>( |
| 1154 | env: Env, |
| 1155 | fingerprint: string, |
| 1156 | operation: (lease?: FirebaseGroupLease) => Promise<T>, |
| 1157 | ): Promise<T> { |
| 1158 | if (crashStorageMode(env) === "d1") return operation(); |
| 1159 | const lease = await acquireFirebaseGroupLease(env, fingerprint); |
| 1160 | if (!lease) throw new Error("firebase crash group is busy"); |
| 1161 | try { |
| 1162 | return await operation(lease); |
| 1163 | } finally { |
| 1164 | await releaseFirebaseGroupLease(env, fingerprint, lease).catch((error) => { |
| 1165 | console.error("firebase crash group lease release failed", error); |
| 1166 | }); |
| 1167 | } |
| 1168 | } |
| 1169 | |
| 1170 | async function handleGroupAction(request: Request, env: Env, admin: User, fingerprint: string): Promise<Response> { |
| 1171 | if (!sameOrigin(request)) return new Response("forbidden", { status: 403 }); |
| 1172 | const parsed = GroupAction.safeParse(await formObject(request)); |
| 1173 | if (!parsed.success) return redirect(`/stats/group/${fingerprint}`); |
| 1174 | const a = parsed.data; |
| 1175 | |
| 1176 | if (a.action === "delete") { |
| 1177 | try { |
| 1178 | if (crashStorageMode(env) !== "d1") { |
| 1179 | await archiveFirebaseGroupForAdmin(env, fingerprint); |
| 1180 | } else { |
| 1181 | await env.DB.batch([ |
| 1182 | env.DB.prepare("DELETE FROM reports WHERE fingerprint = ?1").bind(fingerprint), |
| 1183 | env.DB.prepare("DELETE FROM report_daily WHERE fingerprint = ?1").bind(fingerprint), |
| 1184 | env.DB.prepare("DELETE FROM report_installations WHERE fingerprint = ?1").bind(fingerprint), |
| 1185 | env.DB.prepare("DELETE FROM report_event_dimensions WHERE fingerprint = ?1").bind(fingerprint), |
| 1186 | env.DB.prepare("DELETE FROM report_events WHERE fingerprint = ?1").bind(fingerprint), |
| 1187 | env.DB.prepare("DELETE FROM report_attribution_daily WHERE fingerprint = ?1").bind(fingerprint), |
| 1188 | env.DB.prepare("DELETE FROM report_incidents WHERE fingerprint = ?1").bind(fingerprint), |
| 1189 | env.DB.prepare("DELETE FROM firebase_crash_outbox WHERE fingerprint = ?1").bind(fingerprint), |
| 1190 | env.DB.prepare("DELETE FROM groups WHERE fingerprint = ?1").bind(fingerprint), |
| 1191 | ]); |
| 1192 | } |
| 1193 | } catch (error) { |
| 1194 | return storageUnavailable("firebase group deletion", error); |
| 1195 | } |
| 1196 | await logAction(env, admin, "delete_group", fingerprint.slice(0, 8)); |
| 1197 | return redirect("/stats"); |
| 1198 | } |
| 1199 | if (a.action === "status") { |
| 1200 | const status = a.status ?? "open"; |
| 1201 | try { |
| 1202 | await withFirebaseGroupLease(env, fingerprint, async (lease) => { |
| 1203 | await env.DB.prepare( |
| 1204 | "UPDATE groups SET status = ?1, resolved_at = CASE WHEN ?1 = 'resolved' THEN ?3 ELSE resolved_at END WHERE fingerprint = ?2", |
| 1205 | ).bind(status, fingerprint, new Date().toISOString()).run(); |
| 1206 | await syncFirebaseGroupMetaLocked(env, fingerprint, lease); |
| 1207 | }); |
| 1208 | } catch (error) { |
| 1209 | return storageUnavailable("firebase group metadata", error); |
| 1210 | } |
| 1211 | await logAction(env, admin, "set_status", fingerprint.slice(0, 8), status); |
| 1212 | return redirect(`/stats/group/${fingerprint}`); |
| 1213 | } |
| 1214 | if (a.action === "resolution") { |
| 1215 | await env.DB.prepare( |
| 1216 | "UPDATE groups SET resolved_in = ?1, resolution_platform = ?3, resolution_runtime = ?4, resolution_basis = ?5 WHERE fingerprint = ?2", |
| 1217 | ) |
| 1218 | .bind(a.resolvedIn ?? "", fingerprint, a.resolutionPlatform ?? "", a.resolutionRuntime ?? "", a.resolutionBasis ?? "") |
| 1219 | .run(); |
| 1220 | await logAction( |
| 1221 | env, |
| 1222 | admin, |
| 1223 | "set_resolution", fingerprint.slice(0, 8), |
| 1224 | JSON.stringify({ |
| 1225 | version: a.resolvedIn ?? "", |
| 1226 | platform: a.resolutionPlatform ?? "", |
| 1227 | runtime: a.resolutionRuntime ?? "", |
| 1228 | basis: a.resolutionBasis ?? "", |
| 1229 | }), |
| 1230 | ); |
| 1231 | return redirect(`/stats/group/${fingerprint}`); |
| 1232 | } |
| 1233 | if (a.action === "severity") { |
| 1234 | const severity = a.severity ?? "medium"; |
| 1235 | try { |
| 1236 | await withFirebaseGroupLease(env, fingerprint, async (lease) => { |
| 1237 | await env.DB.prepare("UPDATE groups SET severity = ?1 WHERE fingerprint = ?2") |
| 1238 | .bind(severity, fingerprint) |
| 1239 | .run(); |
| 1240 | await syncFirebaseGroupMetaLocked(env, fingerprint, lease); |
| 1241 | }); |
| 1242 | } catch (error) { |
| 1243 | return storageUnavailable("firebase group metadata", error); |
| 1244 | } |
| 1245 | await logAction(env, admin, "set_severity", fingerprint.slice(0, 8), severity); |
| 1246 | return redirect(`/stats/group/${fingerprint}`); |
| 1247 | } |
| 1248 | await env.DB.prepare("UPDATE groups SET note = ?1 WHERE fingerprint = ?2").bind(a.note ?? "", fingerprint).run(); |
| 1249 | await logAction(env, admin, "set_note", fingerprint.slice(0, 8)); |
| 1250 | return redirect(`/stats/group/${fingerprint}`); |
| 1251 | } |
| 1252 | |
| 1253 | async function handleAdminUsers(request: Request, env: Env, admin: User): Promise<Response> { |
| 1254 | if (!sameOrigin(request)) return new Response("forbidden", { status: 403 }); |
| 1255 | const parsed = UserAction.safeParse(await formObject(request)); |
| 1256 | if (!parsed.success) return redirect("/admin"); |
| 1257 | const a = parsed.data; |
| 1258 | if (a.userId === admin.id) return redirect("/admin"); |
| 1259 | |
| 1260 | const target = await env.DB.prepare("SELECT email, role FROM access WHERE id = ?1") |
| 1261 | .bind(a.userId) |
| 1262 | .first<{ email: string; role: Role }>(); |
| 1263 | if (!target) return redirect("/admin"); |
| 1264 | |
| 1265 | if (a.action === "delete") { |
| 1266 | await env.DB.prepare("DELETE FROM access WHERE id = ?1").bind(a.userId).run(); |
| 1267 | await logAction(env, admin, "delete_user", target.email); |
| 1268 | return redirect("/admin"); |
| 1269 | } |
| 1270 | |
| 1271 | const role: Role = a.role ?? "pending"; |
| 1272 | const now = new Date().toISOString(); |
| 1273 | await env.DB.prepare("UPDATE access SET role = ?1, approved_at = ?2, approved_by = ?3 WHERE id = ?4") |
| 1274 | .bind(role, role === "pending" ? null : now, admin.email, a.userId) |
| 1275 | .run(); |
| 1276 | await logAction(env, admin, "set_role", target.email, `${target.role} → ${role}`); |
| 1277 | return redirect("/admin"); |
| 1278 | } |
| 1279 | |
| 1280 | async function handleAdminList(env: Env, admin: User): Promise<Response> { |
| 1281 | const users = await env.DB.prepare( |
| 1282 | "SELECT id, email, role, created_at, approved_at FROM access ORDER BY (role = 'pending') DESC, created_at DESC", |
| 1283 | ).all<UserRow>(); |
| 1284 | return html(renderUsers(admin, users.results)); |
| 1285 | } |
| 1286 | |
| 1287 | async function handleAdminAudit(env: Env, admin: User): Promise<Response> { |
| 1288 | const rows = await env.DB.prepare( |
| 1289 | "SELECT at, actor_email, action, target, detail FROM audit_log ORDER BY id DESC LIMIT 200", |
| 1290 | ).all<AuditRow>(); |
| 1291 | return html(renderAudit(admin, rows.results)); |
| 1292 | } |
| 1293 | |
| 1294 | function requireViewer(user: User | null, login: string): Response | null { |
| 1295 | if (!user) return redirect(login); |
| 1296 | if (!atLeast(user.role, "viewer")) return redirect("/account"); |
| 1297 | return null; |
| 1298 | } |
| 1299 | |
| 1300 | // The folded registry API runs against its own database and resolves identity |
| 1301 | // itself; hand it the second binding plus the account/site origins it expects. |
| 1302 | function registryBindings(env: Env): RegistryBindings { |
| 1303 | return { |
| 1304 | DB: env.REGISTRY_DB, |
| 1305 | WRITE_LIMITER: env.WRITE_LIMITER, |
| 1306 | ACCOUNTS_ORIGIN: env.ID_ORIGIN ?? "https://id.reasonix.io", |
| 1307 | APP_ORIGIN: env.APP_ORIGIN ?? "https://reasonix.io", |
| 1308 | ALLOWED_ORIGINS: env.ALLOWED_ORIGINS ?? "https://reasonix.io,https://www.reasonix.io", |
| 1309 | }; |
| 1310 | } |
| 1311 | |
| 1312 | function communityStatus(url: URL): string { |
| 1313 | const s = url.searchParams.get("status") ?? "pending"; |
| 1314 | return ["pending", "active", "hidden", "rejected"].includes(s) ? s : "pending"; |
| 1315 | } |
| 1316 | |
| 1317 | async function handleCommunityList(env: Env, admin: User, status: string): Promise<Response> { |
| 1318 | const rows = await new PackageRepo(env.REGISTRY_DB).listByStatus(status, 200); |
| 1319 | return html(renderCommunity(admin, rows, status)); |
| 1320 | } |
| 1321 | |
| 1322 | async function handleCommunityAction( |
| 1323 | request: Request, |
| 1324 | env: Env, |
| 1325 | admin: User, |
| 1326 | handle: string, |
| 1327 | name: string, |
| 1328 | action: string, |
| 1329 | ): Promise<Response> { |
| 1330 | if (!sameOrigin(request)) return new Response("forbidden", { status: 403 }); |
| 1331 | const form = await formObject(request); |
| 1332 | const backStatus = ["pending", "active", "hidden", "rejected"].includes(form.status) ? form.status : "pending"; |
| 1333 | const back = redirect(`/community?status=${backStatus}`); |
| 1334 | const slug = `${handle}/${name}`; |
| 1335 | const repo = new PackageRepo(env.REGISTRY_DB); |
| 1336 | const now = new Date().toISOString(); |
| 1337 | |
| 1338 | if (action === "verify" || action === "unverify") { |
| 1339 | await repo.setVerified(slug, action === "verify", now); |
| 1340 | await logAction(env, admin, `pkg_${action}`, slug); |
| 1341 | return back; |
| 1342 | } |
| 1343 | if (action === "approve") { |
| 1344 | const expectedStatus = ["pending", "hidden", "rejected"].includes(form.expectedStatus) |
| 1345 | ? form.expectedStatus |
| 1346 | : ""; |
| 1347 | if (!form.expectedVersion || !form.expectedUpdatedAt || !expectedStatus) { |
| 1348 | return new Response("Package review revision is missing. Refresh the review page and try again.", { |
| 1349 | status: 409, |
| 1350 | }); |
| 1351 | } |
| 1352 | const row = await repo.setStatusIfCurrent( |
| 1353 | slug, |
| 1354 | "active", |
| 1355 | form.expectedVersion, |
| 1356 | form.expectedUpdatedAt, |
| 1357 | expectedStatus, |
| 1358 | now, |
| 1359 | ); |
| 1360 | if (!row) { |
| 1361 | return new Response("Package changed since it was reviewed. Refresh and review the latest version.", { |
| 1362 | status: 409, |
| 1363 | }); |
| 1364 | } |
| 1365 | // Emit the publish event only after the reviewed revision becomes public. |
| 1366 | await new EventRepo(env.REGISTRY_DB).log({ |
| 1367 | type: "publish", |
| 1368 | packageId: row.id, |
| 1369 | actorHandle: row.scope_handle, |
| 1370 | summary: `published ${row.slug}@${row.latest_version}`, |
| 1371 | now, |
| 1372 | }); |
| 1373 | await logAction(env, admin, "pkg_approve", slug); |
| 1374 | return back; |
| 1375 | } |
| 1376 | await repo.setStatus(slug, action === "reject" ? "rejected" : "hidden", now); |
| 1377 | await logAction(env, admin, `pkg_${action}`, slug); |
| 1378 | return back; |
| 1379 | } |
| 1380 | |
| 1381 | // Time-series retention, run by the daily cron trigger. Every dashboard query |
| 1382 | // against the per-install tables reads at most the current window (-29 day), |
| 1383 | // while the aggregate `metrics` table also serves the 30d view's |
| 1384 | // previous-window delta (back to -59 day), so it keeps a doubled horizon. |
| 1385 | // `reports`/`groups` are excluded on purpose: they are the triage queue and |
| 1386 | // the regression baseline, are not date-partitioned, and their growth is |
| 1387 | // already bounded by per-group sampling. Without this purge the database |
| 1388 | // grows until D1's size cap, at which point every ingest write starts |
| 1389 | // throwing (all of /v1/ping, /v1/metrics and /v1/report 500 while reads keep |
| 1390 | // working — exactly the 2026-07-03 stats blackout). |
| 1391 | const RETENTION = [ |
| 1392 | { table: "report_daily", keepDays: 30 }, |
| 1393 | { table: "report_installations", keepDays: 30 }, |
| 1394 | { table: "report_event_dimensions", keepDays: 30 }, |
| 1395 | { table: "report_events", keepDays: 90, dateColumn: "received_at" }, |
| 1396 | { table: "report_attribution_daily", keepDays: 30 }, |
| 1397 | { table: "report_incidents", keepDays: 30 }, |
| 1398 | { table: "pings", keepDays: 30 }, |
| 1399 | { table: "metrics", keepDays: 60 }, |
| 1400 | { table: "cli_pings", keepDays: 30 }, |
| 1401 | { table: "cli_metrics", keepDays: 60 }, |
| 1402 | ] as const; |
| 1403 | // Deletes run in rowid chunks so a run never holds one giant transaction. |
| 1404 | // Steady state is one expired day per table; the chunk cap is a backstop that |
| 1405 | // still drains ~2M rows per table per run after an ingest outage or backlog. |
| 1406 | const RETENTION_CHUNK_ROWS = 10_000; |
| 1407 | const RETENTION_MAX_CHUNKS = 200; |
| 1408 | |
| 1409 | // Must match the sentinel entry in wrangler.toml [triggers] exactly — the |
| 1410 | // scheduled handler dispatches on controller.cron; every other trigger |
| 1411 | // (the retention cron, manual runs) falls through to the purge. |
| 1412 | const SENTINEL_CRON = "17 1,7,13,19 * * *"; |
| 1413 | // Ingest sentinel. The 2026-07-03 blackout went unnoticed for ten days because |
| 1414 | // clients swallow ping failures by design and nothing watched the write path. |
| 1415 | // Four times a day (hours chosen so the UTC day always has >1h of traffic; |
| 1416 | // ~14k DAU means a healthy hour is never empty) this probes the two failure |
| 1417 | // shapes independently: |
| 1418 | // 1. canary write into `pings` (immediately deleted) — catches writes |
| 1419 | // throwing, e.g. the D1 size cap, regardless of traffic; |
| 1420 | // 2. today's real ping and open totals compared with the previous run — |
| 1421 | // catches ingest dying upstream of the worker (edge blocking, client |
| 1422 | // regression) even after the UTC day already has traffic. |
| 1423 | // Alerts go to the optional ALERT_WEBHOOK secret; without it they still land |
| 1424 | // in the worker logs. While broken this fires at most 4 alerts/day. |
| 1425 | const CANARY_INSTALL_ID = "ffffffffffffffffffffffffffffffff"; |
| 1426 | |
| 1427 | function errText(err: unknown): string { |
| 1428 | return err instanceof Error ? err.message : String(err); |
| 1429 | } |
| 1430 | |
| 1431 | async function sendAlert(env: Env, text: string): Promise<void> { |
| 1432 | if (!env.ALERT_WEBHOOK) return; |
| 1433 | try { |
| 1434 | const webhook = new URL(env.ALERT_WEBHOOK); |
| 1435 | const feishu = webhook.hostname === "open.feishu.cn" || webhook.hostname === "open.larksuite.com"; |
| 1436 | const body = feishu ? { msg_type: "text", content: { text } } : { text }; |
| 1437 | const res = await fetch(webhook.toString(), { |
| 1438 | method: "POST", |
| 1439 | headers: { "content-type": "application/json" }, |
| 1440 | body: JSON.stringify(body), |
| 1441 | }); |
| 1442 | if (!res.ok) console.error(`alert webhook responded ${res.status}`); |
| 1443 | } catch (err) { |
| 1444 | console.error("alert webhook unreachable", err); |
| 1445 | } |
| 1446 | } |
| 1447 | |
| 1448 | async function runIngestSentinel(env: Env): Promise<void> { |
| 1449 | const problems: string[] = []; |
| 1450 | if (crashStorageMode(env) !== "d1") { |
| 1451 | try { |
| 1452 | const storage = await firebaseStorageSummary(env); |
| 1453 | if (storage.reservedBytes >= storage.budgetBytes * 0.8) { |
| 1454 | problems.push(`Firebase reserved storage is ${Math.round(storage.reservedBytes / 1048576)} MiB`); |
| 1455 | } |
| 1456 | if (storage.stuckArchiving > 0) problems.push(`${storage.stuckArchiving} Firebase archives are stuck`); |
| 1457 | if (storage.outboxCount >= FIREBASE_OUTBOX_WARNING) { |
| 1458 | problems.push(`Firebase outbox contains ${storage.outboxCount} rows`); |
| 1459 | } |
| 1460 | } catch (err) { |
| 1461 | problems.push(`Firebase storage sentinel failed: ${errText(err)}`); |
| 1462 | } |
| 1463 | } |
| 1464 | try { |
| 1465 | await env.DB.prepare( |
| 1466 | `INSERT INTO pings (date, install_id, version, os, arch, opens) |
| 1467 | VALUES (date('now'), ?1, 'canary', 'canary', 'canary', 0) |
| 1468 | ON CONFLICT (date, install_id) DO NOTHING`, |
| 1469 | ) |
| 1470 | .bind(CANARY_INSTALL_ID) |
| 1471 | .run(); |
| 1472 | // Also removes any leftover canary from a run that died mid-way. |
| 1473 | await env.DB.prepare("DELETE FROM pings WHERE install_id = ?1").bind(CANARY_INSTALL_ID).run(); |
| 1474 | } catch (err) { |
| 1475 | problems.push(`canary write failed: ${errText(err)}`); |
| 1476 | } |
| 1477 | try { |
| 1478 | // Auto-create the one-row checkpoint so existing databases do not need a |
| 1479 | // manual migration before this worker version is deployed. |
| 1480 | await env.DB.prepare( |
| 1481 | `CREATE TABLE IF NOT EXISTS ingest_sentinel_state ( |
| 1482 | id INTEGER PRIMARY KEY CHECK (id = 1), |
| 1483 | day TEXT NOT NULL, |
| 1484 | ping_count INTEGER NOT NULL, |
| 1485 | open_count INTEGER NOT NULL, |
| 1486 | checked_at TEXT NOT NULL |
| 1487 | )`, |
| 1488 | ).run(); |
| 1489 | const row = await env.DB.prepare( |
| 1490 | `SELECT date('now') AS day, |
| 1491 | COUNT(*) AS ping_count, |
| 1492 | COALESCE(SUM(opens), 0) AS open_count |
| 1493 | FROM pings |
| 1494 | WHERE date = date('now') AND install_id <> ?1`, |
| 1495 | ) |
| 1496 | .bind(CANARY_INSTALL_ID) |
| 1497 | .first<{ day: string; ping_count: number; open_count: number }>(); |
| 1498 | const day = row?.day ?? ""; |
| 1499 | const pingCount = Number(row?.ping_count ?? 0); |
| 1500 | const openCount = Number(row?.open_count ?? 0); |
| 1501 | const previous = await env.DB.prepare( |
| 1502 | "SELECT day, ping_count, open_count, checked_at FROM ingest_sentinel_state WHERE id = 1", |
| 1503 | ).first<{ |
| 1504 | day: string; |
| 1505 | ping_count: number; |
| 1506 | open_count: number; |
| 1507 | checked_at: string; |
| 1508 | }>(); |
| 1509 | if (!pingCount) { |
| 1510 | problems.push("no launch pings recorded today (UTC)"); |
| 1511 | } else if ( |
| 1512 | previous?.day === day && |
| 1513 | pingCount <= Number(previous.ping_count) && |
| 1514 | openCount <= Number(previous.open_count) |
| 1515 | ) { |
| 1516 | problems.push( |
| 1517 | `launch ping totals unchanged since ${previous.checked_at} UTC (${pingCount} install rows, ${openCount} opens)`, |
| 1518 | ); |
| 1519 | } |
| 1520 | await env.DB.prepare( |
| 1521 | `INSERT INTO ingest_sentinel_state (id, day, ping_count, open_count, checked_at) |
| 1522 | VALUES (1, ?1, ?2, ?3, datetime('now')) |
| 1523 | ON CONFLICT (id) DO UPDATE SET |
| 1524 | day = ?1, ping_count = ?2, open_count = ?3, checked_at = datetime('now')`, |
| 1525 | ) |
| 1526 | .bind(day, pingCount, openCount) |
| 1527 | .run(); |
| 1528 | } catch (err) { |
| 1529 | problems.push(`ping progress check failed: ${errText(err)}`); |
| 1530 | } |
| 1531 | if (!problems.length) return; |
| 1532 | const message = `crash.reasonix.io ingest sentinel: ${problems.join("; ")} — https://crash.reasonix.io/stats`; |
| 1533 | console.error(message); |
| 1534 | await sendAlert(env, message); |
| 1535 | } |
| 1536 | |
| 1537 | async function purgeExpiredStatsRows(env: Env): Promise<void> { |
| 1538 | try { |
| 1539 | await ensureCLITelemetrySchema(env); |
| 1540 | } catch (err) { |
| 1541 | console.error("retention: CLI telemetry schema unavailable", err); |
| 1542 | } |
| 1543 | for (const { table, keepDays, ...options } of RETENTION) { |
| 1544 | // Keep exactly the newest `keepDays` dates: today plus keepDays-1 back, |
| 1545 | // matching the `date >= date('now', '-{keepDays-1} day')` reads. |
| 1546 | const cutoff = `-${keepDays - 1} day`; |
| 1547 | const dateColumn = "dateColumn" in options ? options.dateColumn : "date"; |
| 1548 | let purged = 0; |
| 1549 | try { |
| 1550 | for (let i = 0; i < RETENTION_MAX_CHUNKS; i++) { |
| 1551 | const res = await env.DB.prepare( |
| 1552 | `DELETE FROM ${table} WHERE rowid IN ( |
| 1553 | SELECT rowid FROM ${table} WHERE date(${dateColumn}) < date('now', ?1) LIMIT ${RETENTION_CHUNK_ROWS} |
| 1554 | )`, |
| 1555 | ) |
| 1556 | .bind(cutoff) |
| 1557 | .run(); |
| 1558 | const changes = res.meta.changes ?? 0; |
| 1559 | purged += changes; |
| 1560 | if (changes < RETENTION_CHUNK_ROWS) break; |
| 1561 | } |
| 1562 | console.log(`retention: purged ${purged} rows from ${table} (keep ${keepDays}d)`); |
| 1563 | } catch (err) { |
| 1564 | // One broken table must not stop the others; the cron retries tomorrow. |
| 1565 | console.error(`retention: purge failed for ${table} after ${purged} rows`, err); |
| 1566 | } |
| 1567 | } |
| 1568 | } |
| 1569 | |
| 1570 | export default { |
| 1571 | async fetch(request: Request, env: Env): Promise<Response> { |
| 1572 | const url = new URL(request.url); |
| 1573 | const path = url.pathname; |
| 1574 | const method = request.method; |
| 1575 | |
| 1576 | const desktopRelease = desktopReleaseChannel(path); |
| 1577 | if (desktopRelease) { |
| 1578 | return handleReleaseGatewayRequest(method, () => handleDesktopReleaseManifest(desktopRelease)); |
| 1579 | } |
| 1580 | const cliRelease = cliReleaseChannel(path); |
| 1581 | if (cliRelease) { |
| 1582 | return handleReleaseGatewayRequest(method, () => handleCLIRelease(cliRelease)); |
| 1583 | } |
| 1584 | |
| 1585 | if (path === "/v1/report" && method === "POST") return handleReport(request, env); |
| 1586 | if (path === "/v1/ping" && method === "POST") return handlePing(request, env); |
| 1587 | if (path === "/v1/metrics" && method === "POST") return handleMetrics(request, env); |
| 1588 | |
| 1589 | // Skill/MCP registry API — the folded Hono app handles its own auth, CORS |
| 1590 | // and rate limiting against the registry database (public reads + publish, |
| 1591 | // plus the JSON /v1/admin the site's moderation panel calls). |
| 1592 | if (path.startsWith("/v1/packages") || path === "/v1/activity" || path.startsWith("/v1/admin")) { |
| 1593 | return registryApp.fetch(request, registryBindings(env)); |
| 1594 | } |
| 1595 | |
| 1596 | const login = loginUrl(env, request); |
| 1597 | |
| 1598 | // Authentication moved to id.reasonix.io; these paths just bounce there. |
| 1599 | if ((path === "/login" || path === "/register") && method === "GET") return redirect(login); |
| 1600 | if (path === "/logout" && method === "POST") return redirect(login, await sharedLogout(request, env)); |
| 1601 | |
| 1602 | const user = await currentUser(request, env); |
| 1603 | |
| 1604 | if (path === "/") return redirect(user ? (atLeast(user.role, "viewer") ? "/stats" : "/account") : login); |
| 1605 | |
| 1606 | if (path === "/account" && method === "GET") return user ? html(renderAccount(user)) : redirect(login); |
| 1607 | |
| 1608 | const groupFingerprint = groupFingerprintFromPath(path); |
| 1609 | const statsModuleMatch = path.match(/^\/stats\/(diagnostics|usage|preferences|health)$/); |
| 1610 | if ((path === "/stats" || statsModuleMatch) && method === "GET") |
| 1611 | return requireViewer(user, login) ?? handleStats(request, env, user as User, (statsModuleMatch?.[1] as StatsModule | undefined) ?? "usage"); |
| 1612 | if (groupFingerprint && method === "GET") return requireViewer(user, login) ?? handleGroup(env, groupFingerprint, user as User); |
| 1613 | if (groupFingerprint && method === "POST") { |
| 1614 | if (user?.role !== "admin") return new Response("forbidden", { status: 403 }); |
| 1615 | return handleGroupAction(request, env, user, groupFingerprint); |
| 1616 | } |
| 1617 | |
| 1618 | if (path === "/admin" && method === "GET") { |
| 1619 | if (!user) return redirect(login); |
| 1620 | return user.role === "admin" ? handleAdminList(env, user) : redirect("/account"); |
| 1621 | } |
| 1622 | if (path === "/admin/audit" && method === "GET") { |
| 1623 | if (!user) return redirect(login); |
| 1624 | return user.role === "admin" ? handleAdminAudit(env, user) : redirect("/account"); |
| 1625 | } |
| 1626 | if (path === "/admin/users" && method === "POST") { |
| 1627 | if (user?.role !== "admin") return new Response("forbidden", { status: 403 }); |
| 1628 | return handleAdminUsers(request, env, user); |
| 1629 | } |
| 1630 | |
| 1631 | if (path === "/community" && method === "GET") { |
| 1632 | if (!user) return redirect(login); |
| 1633 | return user.role === "admin" ? handleCommunityList(env, user, communityStatus(url)) : redirect("/account"); |
| 1634 | } |
| 1635 | const pkgActionMatch = path.match(/^\/community\/([^/]+)\/([^/]+)\/(approve|reject|hide|verify|unverify)$/); |
| 1636 | if (pkgActionMatch && method === "POST") { |
| 1637 | if (user?.role !== "admin") return new Response("forbidden", { status: 403 }); |
| 1638 | return handleCommunityAction(request, env, user, pkgActionMatch[1], pkgActionMatch[2], pkgActionMatch[3]); |
| 1639 | } |
| 1640 | |
| 1641 | if ( |
| 1642 | path === "/v1/report" || |
| 1643 | path === "/v1/ping" || |
| 1644 | path === "/v1/metrics" || |
| 1645 | path === "/login" || |
| 1646 | path === "/register" || |
| 1647 | path === "/logout" || |
| 1648 | path === "/account" || |
| 1649 | path.startsWith("/stats") || |
| 1650 | path.startsWith("/admin") || |
| 1651 | path.startsWith("/community") |
| 1652 | ) { |
| 1653 | return new Response("method not allowed", { status: 405 }); |
| 1654 | } |
| 1655 | return new Response("not found", { status: 404 }); |
| 1656 | }, |
| 1657 | |
| 1658 | async scheduled(controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void> { |
| 1659 | if (controller.cron === SENTINEL_CRON) { |
| 1660 | ctx.waitUntil(Promise.all([ |
| 1661 | runIngestSentinel(env), |
| 1662 | drainFirebaseCrashOutbox(env), |
| 1663 | ]).then(() => undefined)); |
| 1664 | return; |
| 1665 | } |
| 1666 | ctx.waitUntil(Promise.all([ |
| 1667 | purgeExpiredStatsRows(env), |
| 1668 | crashStorageMode(env) === "d1" ? Promise.resolve() : purgeFirebaseDeliveryState(env), |
| 1669 | drainFirebaseCrashOutbox(env), |
| 1670 | crashStorageMode(env) === "d1" ? Promise.resolve() : runFirebaseCrashLifecycle(env), |
| 1671 | ]).then(() => undefined)); |
| 1672 | }, |
| 1673 | }; |
| 1674 |