| 1 | package telemetry |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "net" |
| 7 | "regexp" |
| 8 | "runtime" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "time" |
| 12 | |
| 13 | "reasonix/internal/event" |
| 14 | "reasonix/internal/netclient" |
| 15 | "reasonix/internal/provider" |
| 16 | "reasonix/internal/recovery" |
| 17 | ) |
| 18 | |
| 19 | type Options struct { |
| 20 | Mode string |
| 21 | Version string |
| 22 | HomeDir string |
| 23 | Interactive bool |
| 24 | Proxy netclient.ProxySpec |
| 25 | CLIMode string |
| 26 | PermissionMode string |
| 27 | SessionMode string |
| 28 | Language string |
| 29 | } |
| 30 | |
| 31 | type Reporter struct { |
| 32 | client *Client |
| 33 | version string |
| 34 | home string |
| 35 | static []Counter |
| 36 | } |
| 37 | |
| 38 | func Start(opts Options) *Reporter { |
| 39 | if !Enabled(opts.Mode, opts.Version, opts.Interactive) { |
| 40 | if strings.EqualFold(strings.TrimSpace(opts.Mode), "off") || envOptOut() { |
| 41 | _ = Cleanup(opts.HomeDir) |
| 42 | } |
| 43 | return nil |
| 44 | } |
| 45 | client, err := newClient(opts.HomeDir, opts.Version, opts.Proxy) |
| 46 | if err != nil { |
| 47 | return nil |
| 48 | } |
| 49 | r := &Reporter{ |
| 50 | client: client, |
| 51 | version: opts.Version, |
| 52 | home: opts.HomeDir, |
| 53 | static: []Counter{ |
| 54 | {Signal: "client_surface", Bucket: "cli", Count: 1}, |
| 55 | {Signal: "client_version", Bucket: safeBucket(opts.Version, "other"), Count: 1}, |
| 56 | {Signal: "cli_mode", Bucket: enumBucket(opts.CLIMode, "run", "tui"), Count: 1}, |
| 57 | {Signal: "cli_permission_mode", Bucket: permissionBucket(opts.PermissionMode), Count: 1}, |
| 58 | {Signal: "cli_session_mode", Bucket: enumBucket(opts.SessionMode, "fresh", "resume", "continue", "copy"), Count: 1}, |
| 59 | {Signal: "settings_language", Bucket: languageBucket(opts.Language), Count: 1}, |
| 60 | }, |
| 61 | } |
| 62 | go client.backgroundFlush() |
| 63 | return r |
| 64 | } |
| 65 | |
| 66 | func (r *Reporter) Wrap(inner event.Sink) event.Sink { |
| 67 | if r == nil { |
| 68 | return inner |
| 69 | } |
| 70 | return &sink{AuditForwarder: event.AuditForwarder{Inner: inner}, inner: inner, reporter: r, counts: countersFrom(r.static)} |
| 71 | } |
| 72 | |
| 73 | func (r *Reporter) RecordRecovery(m recovery.Metrics) { |
| 74 | if r == nil { |
| 75 | return |
| 76 | } |
| 77 | counts := map[string]int{} |
| 78 | addMetric(counts, "recovery_failure", "count", m.FailureEvents) |
| 79 | addMetric(counts, "recovery_rule_continue", "count", m.RuleContinues) |
| 80 | addMetric(counts, "recovery_review_continue", "count", m.ReviewContinues) |
| 81 | addMetric(counts, "recovery_human_prompt", "count", m.HumanPrompts) |
| 82 | addMetric(counts, "recovery_human_continue", "count", m.HumanContinues) |
| 83 | addMetric(counts, "recovery_human_revise", "count", m.HumanRevises) |
| 84 | addMetric(counts, "recovery_review_error", "count", m.ReviewErrors) |
| 85 | addMetric(counts, "recovery_repeat_prompt", "count", m.RepeatPrompts) |
| 86 | if m.ReviewLatencyCount > 0 { |
| 87 | add(counts, "recovery_review_latency", latencyBucket(time.Duration(m.ReviewLatencyMsSum/m.ReviewLatencyCount)*time.Millisecond), int(m.ReviewLatencyCount)) |
| 88 | } |
| 89 | r.append(counts) |
| 90 | } |
| 91 | |
| 92 | func addMetric(counts map[string]int, signal, bucket string, count int64) { |
| 93 | if count > 0 { |
| 94 | add(counts, signal, bucket, int(count)) |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | func (r *Reporter) append(counts map[string]int) { |
| 99 | if r == nil || len(counts) == 0 { |
| 100 | return |
| 101 | } |
| 102 | counters := make([]Counter, 0, len(counts)) |
| 103 | for key, count := range counts { |
| 104 | signal, bucket, _ := strings.Cut(key, "\x00") |
| 105 | if count > 1_000_000 { |
| 106 | count = 1_000_000 |
| 107 | } |
| 108 | counters = append(counters, Counter{Signal: signal, Bucket: bucket, Count: count}) |
| 109 | } |
| 110 | _ = appendPending(r.home, pendingPayload{Version: r.version, OS: runtime.GOOS, Counters: counters}) |
| 111 | } |
| 112 | |
| 113 | type sink struct { |
| 114 | event.AuditForwarder |
| 115 | inner event.Sink |
| 116 | reporter *Reporter |
| 117 | counts map[string]int |
| 118 | started time.Time |
| 119 | hasText bool |
| 120 | emptyFinalSeen bool |
| 121 | } |
| 122 | |
| 123 | func (s *sink) Emit(e event.Event) { |
| 124 | s.observe(e) |
| 125 | s.inner.Emit(e) |
| 126 | } |
| 127 | |
| 128 | func (s *sink) RecordProtocolRecovery(a event.ProtocolRecoveryAudit) { |
| 129 | add(s.counts, "tool_call_reasoning_recovery", string(a.Kind), 1) |
| 130 | event.RecordProtocolRecovery(s.inner, a) |
| 131 | } |
| 132 | |
| 133 | func (s *sink) observe(e event.Event) { |
| 134 | switch e.Kind { |
| 135 | case event.TurnStarted: |
| 136 | s.started = time.Now() |
| 137 | s.hasText = false |
| 138 | s.emptyFinalSeen = false |
| 139 | add(s.counts, "turns", "count", 1) |
| 140 | case event.Text: |
| 141 | if e.Text != "" { |
| 142 | s.hasText = true |
| 143 | } |
| 144 | case event.Message: |
| 145 | if e.Text != "" { |
| 146 | s.hasText = true |
| 147 | } |
| 148 | case event.Usage: |
| 149 | if e.Usage != nil { |
| 150 | add(s.counts, "finish_reason", finishReasonBucket(e.Usage.FinishReason), 1) |
| 151 | add(s.counts, "cache_hit", cacheBucket(e.Usage.CacheHitTokens, e.Usage.CacheMissTokens), 1) |
| 152 | } |
| 153 | case event.ToolResult: |
| 154 | if e.Tool.Err != "" { |
| 155 | add(s.counts, "tool_error", toolErrorBucket(e.Tool.Err), 1) |
| 156 | } |
| 157 | case event.Notice: |
| 158 | if e.Code == event.NoticeCodeEmptyFinal { |
| 159 | add(s.counts, "empty_final", "yes", 1) |
| 160 | s.emptyFinalSeen = true |
| 161 | } |
| 162 | case event.CompactionStarted: |
| 163 | add(s.counts, "compaction", enumBucket(e.Compaction.Trigger, "auto", "manual"), 1) |
| 164 | case event.TurnDone: |
| 165 | if !s.hasText && e.Err == nil && !s.emptyFinalSeen { |
| 166 | add(s.counts, "empty_final", "yes", 1) |
| 167 | } |
| 168 | if bucket := providerErrorBucket(e.Err); bucket != "" { |
| 169 | add(s.counts, "provider_error", bucket, 1) |
| 170 | } |
| 171 | add(s.counts, "cli_exit", exitBucket(e), 1) |
| 172 | if !s.started.IsZero() { |
| 173 | add(s.counts, "cli_turn_latency", latencyBucket(time.Since(s.started)), 1) |
| 174 | } |
| 175 | s.reporter.append(s.counts) |
| 176 | s.counts = map[string]int{} |
| 177 | s.started = time.Time{} |
| 178 | s.hasText = false |
| 179 | s.emptyFinalSeen = false |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | func countersFrom(in []Counter) map[string]int { |
| 184 | out := map[string]int{} |
| 185 | for _, c := range in { |
| 186 | add(out, c.Signal, c.Bucket, c.Count) |
| 187 | } |
| 188 | return out |
| 189 | } |
| 190 | |
| 191 | func add(counts map[string]int, signal, bucket string, count int) { |
| 192 | if count <= 0 || signal == "" || bucket == "" { |
| 193 | return |
| 194 | } |
| 195 | counts[signal+"\x00"+bucket] += count |
| 196 | } |
| 197 | |
| 198 | var unsafeBucketChars = regexp.MustCompile(`[^a-z0-9_]+`) |
| 199 | |
| 200 | func safeBucket(value, fallback string) string { |
| 201 | value = strings.ToLower(strings.TrimSpace(value)) |
| 202 | value = unsafeBucketChars.ReplaceAllString(value, "_") |
| 203 | value = strings.Trim(value, "_") |
| 204 | if value == "" { |
| 205 | return fallback |
| 206 | } |
| 207 | if len(value) > 96 { |
| 208 | value = value[:96] |
| 209 | } |
| 210 | return value |
| 211 | } |
| 212 | |
| 213 | func enumBucket(value string, allowed ...string) string { |
| 214 | value = strings.ToLower(strings.TrimSpace(value)) |
| 215 | if slices.Contains(allowed, value) { |
| 216 | return value |
| 217 | } |
| 218 | return "other" |
| 219 | } |
| 220 | |
| 221 | func permissionBucket(value string) string { |
| 222 | switch strings.ToLower(strings.TrimSpace(value)) { |
| 223 | case "manual", "ask": |
| 224 | return "ask" |
| 225 | case "auto", "acceptedits": |
| 226 | return "auto" |
| 227 | case "dontask": |
| 228 | return "dont_ask" |
| 229 | case "plan": |
| 230 | return "plan" |
| 231 | case "bypasspermissions", "yolo": |
| 232 | return "yolo" |
| 233 | default: |
| 234 | return "other" |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | func languageBucket(value string) string { |
| 239 | value = strings.ToLower(strings.TrimSpace(value)) |
| 240 | if strings.HasPrefix(value, "zh") { |
| 241 | return "zh" |
| 242 | } |
| 243 | if strings.HasPrefix(value, "en") { |
| 244 | return "en" |
| 245 | } |
| 246 | if value == "" || value == "auto" { |
| 247 | return "auto" |
| 248 | } |
| 249 | return "other" |
| 250 | } |
| 251 | |
| 252 | func finishReasonBucket(value string) string { |
| 253 | switch strings.ToLower(strings.TrimSpace(value)) { |
| 254 | case "stop", "tool_calls", "length", "content_filter", "repetition_truncation": |
| 255 | return safeBucket(value, "unknown") |
| 256 | case "": |
| 257 | return "unknown" |
| 258 | default: |
| 259 | return "other" |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | func cacheBucket(hit, miss int) string { |
| 264 | total := hit + miss |
| 265 | if total <= 0 { |
| 266 | return "unknown" |
| 267 | } |
| 268 | pct := hit * 100 / total |
| 269 | switch { |
| 270 | case pct == 0: |
| 271 | return "0" |
| 272 | case pct < 25: |
| 273 | return "1_24" |
| 274 | case pct < 50: |
| 275 | return "25_49" |
| 276 | case pct < 75: |
| 277 | return "50_74" |
| 278 | case pct < 90: |
| 279 | return "75_89" |
| 280 | default: |
| 281 | return "90_100" |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | func toolErrorBucket(value string) string { |
| 286 | v := strings.ToLower(value) |
| 287 | switch { |
| 288 | case strings.Contains(v, "permission"), strings.Contains(v, "blocked"), strings.Contains(v, "denied"): |
| 289 | return "permission" |
| 290 | case strings.Contains(v, "timeout"), strings.Contains(v, "deadline"): |
| 291 | return "timeout" |
| 292 | case strings.Contains(v, "cancel"): |
| 293 | return "cancelled" |
| 294 | case strings.Contains(v, "not found"), strings.Contains(v, "no such"): |
| 295 | return "not_found" |
| 296 | default: |
| 297 | return "other" |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | func providerErrorBucket(err error) string { |
| 302 | if err == nil { |
| 303 | return "" |
| 304 | } |
| 305 | var auth *provider.AuthError |
| 306 | if errors.As(err, &auth) { |
| 307 | return "auth" |
| 308 | } |
| 309 | var api *provider.APIError |
| 310 | if errors.As(err, &api) { |
| 311 | switch { |
| 312 | case api.Status == 429: |
| 313 | return "rate_limit" |
| 314 | case api.Status >= 500: |
| 315 | return "server" |
| 316 | case api.Status >= 400: |
| 317 | return "request" |
| 318 | default: |
| 319 | return "http" |
| 320 | } |
| 321 | } |
| 322 | if errors.Is(err, context.DeadlineExceeded) { |
| 323 | return "timeout" |
| 324 | } |
| 325 | if errors.Is(err, context.Canceled) { |
| 326 | return "cancelled" |
| 327 | } |
| 328 | var netErr net.Error |
| 329 | if errors.As(err, &netErr) { |
| 330 | return "network" |
| 331 | } |
| 332 | if provider.IsStreamInterrupted(err) { |
| 333 | return "interrupted" |
| 334 | } |
| 335 | return "" |
| 336 | } |
| 337 | |
| 338 | func latencyBucket(d time.Duration) string { |
| 339 | switch { |
| 340 | case d < time.Second: |
| 341 | return "lt_1s" |
| 342 | case d < 5*time.Second: |
| 343 | return "s_1_5" |
| 344 | case d < 15*time.Second: |
| 345 | return "s_5_15" |
| 346 | case d < time.Minute: |
| 347 | return "s_15_60" |
| 348 | case d < 5*time.Minute: |
| 349 | return "m_1_5" |
| 350 | case d < 15*time.Minute: |
| 351 | return "m_5_15" |
| 352 | default: |
| 353 | return "m_15_plus" |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | func exitBucket(e event.Event) string { |
| 358 | if e.Cancelled || errors.Is(e.Err, context.Canceled) { |
| 359 | return "cancelled" |
| 360 | } |
| 361 | if e.Outcome == event.TurnOutcomeRecoveryPaused { |
| 362 | return "recovery_paused" |
| 363 | } |
| 364 | if e.Outcome == event.TurnOutcomeCompletionUncertain { |
| 365 | return "completion_uncertain" |
| 366 | } |
| 367 | if e.Outcome == event.TurnOutcomeIncompleteRead { |
| 368 | return "incomplete_read" |
| 369 | } |
| 370 | if e.Err != nil { |
| 371 | return "error" |
| 372 | } |
| 373 | return "success" |
| 374 | } |
| 375 |