| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "reasonix/internal/config" |
| 11 | "reasonix/internal/event" |
| 12 | "reasonix/internal/provider" |
| 13 | "reasonix/internal/recovery" |
| 14 | ) |
| 15 | |
| 16 | type recoveryMetricsDeltaStub struct { |
| 17 | deltas []recovery.Metrics |
| 18 | } |
| 19 | |
| 20 | func (s *recoveryMetricsDeltaStub) DrainRecoveryMetrics() recovery.Metrics { |
| 21 | if len(s.deltas) == 0 { |
| 22 | return recovery.Metrics{} |
| 23 | } |
| 24 | next := s.deltas[0] |
| 25 | s.deltas = s.deltas[1:] |
| 26 | return next |
| 27 | } |
| 28 | |
| 29 | func TestObserveClassifiesEvents(t *testing.T) { |
| 30 | m := newMetricsAggregator(t.TempDir()) |
| 31 | feed := []event.Event{ |
| 32 | {Kind: event.Usage, Usage: &provider.Usage{FinishReason: "stop", CacheHitTokens: 99, CacheMissTokens: 1}}, |
| 33 | {Kind: event.Usage, Usage: &provider.Usage{FinishReason: "tool_calls", CacheHitTokens: 60, CacheMissTokens: 40}}, |
| 34 | {Kind: event.ToolResult, Tool: event.Tool{Name: "bash", Err: "blocked by permission policy"}}, |
| 35 | {Kind: event.CompactionDone}, |
| 36 | {Kind: event.Notice, Code: event.NoticeCodeEmptyFinal, Text: "No visible answer was produced; asking the assistant to respond again.", Detail: "empty final answer blocked: model returned no visible answer text; retrying"}, |
| 37 | {Kind: event.TurnDone, Err: errors.New("deepseek-flash: status 429: rate limited")}, |
| 38 | {Kind: event.TurnDone, Err: errors.New("automatic recovery paused"), Outcome: event.TurnOutcomeRecoveryPaused}, |
| 39 | {Kind: event.TurnDone, Err: errors.New("incomplete read"), Outcome: event.TurnOutcomeIncompleteRead}, |
| 40 | {Kind: event.TurnDone}, |
| 41 | } |
| 42 | for _, e := range feed { |
| 43 | m.observe(e) |
| 44 | } |
| 45 | |
| 46 | want := map[string]map[string]int{ |
| 47 | "finish_reason": {"stop": 1, "tool_calls": 1}, |
| 48 | "cache_hit": {"99_100": 1, "50_80": 1}, |
| 49 | "tool_error": {"permission": 1}, |
| 50 | "compaction": {"total": 1}, |
| 51 | "empty_final": {"total": 1}, |
| 52 | "provider_error": {"http_429": 1}, |
| 53 | "turns": {"total": 4}, |
| 54 | } |
| 55 | for sig, buckets := range want { |
| 56 | for b, n := range buckets { |
| 57 | if got := m.c[sig][b]; got != n { |
| 58 | t.Errorf("%s/%s = %d, want %d", sig, b, got, n) |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestObserveControllerRecoveryMetricsConsumesOnlyNewDelta(t *testing.T) { |
| 65 | m := newMetricsAggregator(t.TempDir()) |
| 66 | ctrl := &recoveryMetricsDeltaStub{deltas: []recovery.Metrics{ |
| 67 | {FailureEvents: 1, HumanPrompts: 1, ReviewLatencyMsSum: 750, ReviewLatencyCount: 1}, |
| 68 | {}, |
| 69 | }} |
| 70 | |
| 71 | observeControllerRecoveryMetrics(m, ctrl) |
| 72 | observeControllerRecoveryMetrics(m, ctrl) |
| 73 | |
| 74 | if got := m.c["recovery_failure"]["total"]; got != 1 { |
| 75 | t.Fatalf("recovery_failure/total = %d, want 1", got) |
| 76 | } |
| 77 | if got := m.c["recovery_human_prompt"]["total"]; got != 1 { |
| 78 | t.Fatalf("recovery_human_prompt/total = %d, want 1", got) |
| 79 | } |
| 80 | if got := m.c["recovery_review_latency"]["lt_2s"]; got != 1 { |
| 81 | t.Fatalf("recovery_review_latency/lt_2s = %d, want 1", got) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | func TestObserveReadsNoMessageText(t *testing.T) { |
| 86 | m := newMetricsAggregator(t.TempDir()) |
| 87 | // A notice that merely mentions the phrase mid-string must not count. |
| 88 | m.observe(event.Event{Kind: event.Notice, Text: "see docs: empty final answer blocked is a guard"}) |
| 89 | if m.c["empty_final"] != nil { |
| 90 | t.Errorf("empty_final should only match the notice prefix, got %v", m.c["empty_final"]) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | func TestObserveClassifiesLocalizedEmptyFinalByCode(t *testing.T) { |
| 95 | m := newMetricsAggregator(t.TempDir()) |
| 96 | m.observe(event.Event{ |
| 97 | Kind: event.Notice, Code: event.NoticeCodeEmptyFinal, |
| 98 | Text: "没有生成可见回复，已要求助手重新作答。", |
| 99 | }) |
| 100 | if got := m.c["empty_final"]["total"]; got != 1 { |
| 101 | t.Fatalf("localized empty_final/total = %d, want 1", got) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestObserveSettingsSnapshotUsesSafeBuckets(t *testing.T) { |
| 106 | cfg := config.Default() |
| 107 | if err := cfg.SetDesktopLanguage(""); err != nil { |
| 108 | t.Fatalf("SetDesktopLanguage: %v", err) |
| 109 | } |
| 110 | if err := cfg.SetDesktopLayoutStyle("workbench"); err != nil { |
| 111 | t.Fatalf("SetDesktopLayoutStyle: %v", err) |
| 112 | } |
| 113 | if err := cfg.SetDesktopAppearance("dark", "graphite"); err != nil { |
| 114 | t.Fatalf("SetDesktopAppearance: %v", err) |
| 115 | } |
| 116 | if err := cfg.SetDesktopCloseBehavior("quit"); err != nil { |
| 117 | t.Fatalf("SetDesktopCloseBehavior: %v", err) |
| 118 | } |
| 119 | if err := cfg.SetDesktopDisplayMode("compact"); err != nil { |
| 120 | t.Fatalf("SetDesktopDisplayMode: %v", err) |
| 121 | } |
| 122 | if err := cfg.SetDesktopStatusBarStyle("icon"); err != nil { |
| 123 | t.Fatalf("SetDesktopStatusBarStyle: %v", err) |
| 124 | } |
| 125 | if err := cfg.SetDesktopStatusBarItems([]string{"model", "cache", "balance"}); err != nil { |
| 126 | t.Fatalf("SetDesktopStatusBarItems: %v", err) |
| 127 | } |
| 128 | if err := cfg.SetDesktopCheckUpdates(false); err != nil { |
| 129 | t.Fatalf("SetDesktopCheckUpdates: %v", err) |
| 130 | } |
| 131 | customProvider := "Local OpenAI" |
| 132 | customModel := "Qwen-72B-Instruct.private" |
| 133 | cfg.Providers = append(cfg.Providers, config.ProviderEntry{ |
| 134 | Name: customProvider, |
| 135 | Kind: "openai", |
| 136 | BaseURL: "http://127.0.0.1:9999/v1", |
| 137 | Models: []string{customModel}, |
| 138 | Default: customModel, |
| 139 | }) |
| 140 | cfg.Agent.PlannerModel = customProvider + "/" + customModel |
| 141 | cfg.Desktop.ProviderAccess = []string{customProvider} |
| 142 | cfg.Bot.Connections = []config.BotConnectionConfig{{ |
| 143 | Provider: "feishu", |
| 144 | Enabled: true, |
| 145 | Status: "connected", |
| 146 | Model: customProvider + "/" + customModel, |
| 147 | }} |
| 148 | |
| 149 | m := newMetricsAggregator(t.TempDir()) |
| 150 | m.observeSettingsSnapshot(cfg) |
| 151 | |
| 152 | want := map[string]string{ |
| 153 | "settings_language": "auto", |
| 154 | "client_surface": "desktop", |
| 155 | "client_version": metricBucket(version), |
| 156 | "settings_desktop_layout": "workbench", |
| 157 | "settings_theme": "dark", |
| 158 | "settings_theme_style": "graphite", |
| 159 | "settings_close_behavior": "quit", |
| 160 | "settings_display_mode": "compact", |
| 161 | "settings_status_bar_style": "icon", |
| 162 | "settings_status_bar_items_count": "n_3", |
| 163 | "settings_check_updates": "off", |
| 164 | "settings_default_model": "deepseek_deepseek_v4_flash", |
| 165 | "settings_planner_model": metricBucket("custom_" + customProvider + "_" + customModel), |
| 166 | "settings_provider_access": metricBucket("custom_" + customProvider), |
| 167 | "settings_bot_enabled": "off", |
| 168 | "settings_bot_connection_count": "n_1", |
| 169 | "settings_bot_connection_provider": "feishu", |
| 170 | "settings_bot_connection_enabled": "on", |
| 171 | "settings_bot_connection_status": "connected", |
| 172 | "settings_bot_connection_model": metricBucket("custom_" + customProvider + "_" + customModel), |
| 173 | } |
| 174 | for signal, bucket := range want { |
| 175 | if got := m.c[signal][bucket]; got != 1 { |
| 176 | t.Errorf("%s/%s = %d, want 1", signal, bucket, got) |
| 177 | } |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | func TestObserveSettingsSnapshotCountsDisabledPlannerAsOff(t *testing.T) { |
| 182 | cfg := config.Default() |
| 183 | cfg.Agent.PlannerModel = "" |
| 184 | |
| 185 | m := newMetricsAggregator(t.TempDir()) |
| 186 | m.observeSettingsSnapshot(cfg) |
| 187 | |
| 188 | if got := m.c["settings_planner_model"]["off"]; got != 1 { |
| 189 | t.Fatalf("settings_planner_model/off = %d, want 1", got) |
| 190 | } |
| 191 | if got := m.c["settings_planner_model"][safeModelBucket(cfg, cfg.DefaultModel)]; got != 0 { |
| 192 | t.Fatalf("disabled planner should not count the default model, got %d", got) |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | func TestErrorClass(t *testing.T) { |
| 197 | cases := map[string]string{ |
| 198 | "deepseek: status 400: bad": "http_400", |
| 199 | "status 401 unauthorized": "http_401", |
| 200 | "status 403 forbidden": "http_401", |
| 201 | "status 429 too many": "http_429", |
| 202 | "status 503 unavailable": "http_5xx", |
| 203 | "read: connection reset by peer": "stream_interrupted", |
| 204 | "stream interrupted mid-flight": "stream_interrupted", |
| 205 | "context deadline exceeded (timeout)": "timeout", |
| 206 | "update: authorization cancelled": "authorization_cancelled", |
| 207 | "update: authorization failed": "authorization_failed", |
| 208 | "update: package manager busy": "package_manager_busy", |
| 209 | "update: package install failed": "package_install_failed", |
| 210 | "update: package verify failed": "package_verify_failed", |
| 211 | "some unrecognized failure": "other", |
| 212 | } |
| 213 | for msg, want := range cases { |
| 214 | if got := errorClass(msg); got != want { |
| 215 | t.Errorf("errorClass(%q) = %q, want %q", msg, got, want) |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | func TestCacheBucket(t *testing.T) { |
| 221 | cases := []struct { |
| 222 | hit, miss int |
| 223 | want string |
| 224 | }{ |
| 225 | {0, 100, "0_50"}, |
| 226 | {49, 51, "0_50"}, |
| 227 | {60, 40, "50_80"}, |
| 228 | {90, 10, "80_95"}, |
| 229 | {97, 3, "95_99"}, |
| 230 | {999, 1, "99_100"}, |
| 231 | } |
| 232 | for _, c := range cases { |
| 233 | if got := cacheBucket(c.hit, c.miss); got != c.want { |
| 234 | t.Errorf("cacheBucket(%d,%d) = %q, want %q", c.hit, c.miss, got, c.want) |
| 235 | } |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | func TestPersistMergesAcrossSessions(t *testing.T) { |
| 240 | dir := t.TempDir() |
| 241 | path := filepath.Join(dir, metricsPendingFile) |
| 242 | |
| 243 | s1 := newMetricsAggregator(dir) |
| 244 | s1.observe(event.Event{Kind: event.TurnDone}) |
| 245 | s1.persist() |
| 246 | if _, err := os.Stat(path); err != nil { |
| 247 | t.Fatalf("pending file should exist after persist: %v", err) |
| 248 | } |
| 249 | |
| 250 | // A second session merges into the same file rather than overwriting. |
| 251 | s2 := newMetricsAggregator(dir) |
| 252 | s2.observe(event.Event{Kind: event.TurnDone}) |
| 253 | s2.persist() |
| 254 | |
| 255 | if got := readCounters(path)["turns"]["total"]; got != 2 { |
| 256 | t.Errorf("merged turns/total = %d, want 2", got) |
| 257 | } |
| 258 | if n := len(flatten(readCounters(path))); n != 1 { |
| 259 | t.Errorf("flatten produced %d counters, want 1", n) |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | func TestErrorClassSeparatesBadRequestCauses(t *testing.T) { |
| 264 | // Verbatim provider bodies from reported issues, each needing a different fix. |
| 265 | cases := map[string]string{ |
| 266 | `status 400: messages[203]: unknown variant ` + "`image_url`" + `, expected ` + "`text`": "http_400_content", |
| 267 | `status 400: Invalid schema for function 'ls': null is not of type "array"`: "http_400_schema", |
| 268 | "status 400: The `content[].thinking` in the thinking mode must be passed back": "http_400_reasoning_replay", |
| 269 | "status 400: thinking: invalid type: map, expected a boolean": "http_400_thinking_shape", |
| 270 | "status 400: This model's maximum context length is 65536 tokens": "http_400_context_length", |
| 271 | "status 400: missing field name at line 1": "http_400_tool_calls", |
| 272 | "status 400: something nobody has classified yet": "http_400", |
| 273 | } |
| 274 | for msg, want := range cases { |
| 275 | if got := errorClass(msg); got != want { |
| 276 | t.Errorf("errorClass(%q) = %q, want %q", msg, got, want) |
| 277 | } |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | func TestErrorClassNeverEchoesProviderText(t *testing.T) { |
| 282 | // The bucket is uploaded; the message is not. A label must be a fixed token, |
| 283 | // so no substring of a body carrying user code or prompt text can reach it. |
| 284 | secrets := []string{ |
| 285 | "status 400: Invalid schema for function 'ls': /home/alice/secret-project is not of type \"array\"", |
| 286 | "status 400: unknown variant `image_url` in ~/work/client-contract.pdf", |
| 287 | "status 400: thinking: invalid type: map, expected a boolean, key=sk-abc123", |
| 288 | } |
| 289 | for _, msg := range secrets { |
| 290 | got := errorClass(msg) |
| 291 | for _, leak := range []string{"alice", "secret", "client-contract", "sk-abc123", "/home", "~/work"} { |
| 292 | if strings.Contains(got, leak) { |
| 293 | t.Fatalf("errorClass(%q) = %q, leaked %q", msg, got, leak) |
| 294 | } |
| 295 | } |
| 296 | if !strings.HasPrefix(got, "http_400") { |
| 297 | t.Errorf("errorClass(%q) = %q, want an http_400 label", msg, got) |
| 298 | } |
| 299 | } |
| 300 | } |
| 301 |