| 1 | package doctor |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "reasonix/internal/agent" |
| 11 | "reasonix/internal/provider" |
| 12 | ) |
| 13 | |
| 14 | func TestCollectQualityProducesPublicSafeSummary(t *testing.T) { |
| 15 | dir := t.TempDir() |
| 16 | path := filepath.Join(dir, "private-project-session.jsonl") |
| 17 | const secret = "private-source-and-prompt-secret" |
| 18 | messages := []string{ |
| 19 | `{"role":"system","content":"` + secret + `"}`, |
| 20 | `{"role":"user","content":"fix /private/example/repo and keep ` + secret + `"}`, |
| 21 | `{"role":"assistant","tool_calls":[{"id":"edit-1","name":"edit_file","arguments":"{\"path\":\"/private/example/repo/app.go\"}"}]}`, |
| 22 | `{"role":"tool","tool_call_id":"edit-1","name":"edit_file","content":"updated ` + secret + `"}`, |
| 23 | `{"role":"assistant","reasoning_content":"verify the change","tool_calls":[{"id":"test-1","name":"bash","arguments":"{\"command\":\"go test ./...\"}"}]}`, |
| 24 | `{"role":"tool","tool_call_id":"test-1","name":"bash","content":"ok"}`, |
| 25 | `{"role":"user","content":"<compaction-summary>\nprivate summary ` + secret + `"}`, |
| 26 | `{"role":"assistant","content":"done"}`, |
| 27 | } |
| 28 | if err := os.WriteFile(path, []byte(strings.Join(messages, "\n")+"\n"), 0o644); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | if err := agent.SaveBranchMeta(path, agent.BranchMeta{ |
| 32 | ID: "private-session-id", |
| 33 | Model: "corp.private.example/supersecret-model", |
| 34 | TokenMode: "economy", |
| 35 | Mode: "plan-yolo", |
| 36 | ToolApprovalMode: "yolo", |
| 37 | Goal: "finish " + secret, |
| 38 | Recovered: true, |
| 39 | }); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | telemetry := `{ |
| 43 | "version":2, |
| 44 | "usage":{ |
| 45 | "promptTokens":1000, |
| 46 | "completionTokens":200, |
| 47 | "reasoningTokens":80, |
| 48 | "cacheHitTokens":750, |
| 49 | "cacheMissTokens":250, |
| 50 | "requestCount":5, |
| 51 | "sources":{ |
| 52 | "executor":{"requestCount":2}, |
| 53 | "planner":{"requestCount":1}, |
| 54 | "subagent":{"requestCount":1}, |
| 55 | "` + secret + `":{"requestCount":1} |
| 56 | } |
| 57 | } |
| 58 | }` |
| 59 | if err := os.WriteFile(path+".telemetry.json", []byte(telemetry), 0o644); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | |
| 63 | report, err := CollectQuality(QualityOptions{Version: "v-test", SessionRef: path}) |
| 64 | if err != nil { |
| 65 | t.Fatalf("CollectQuality: %v", err) |
| 66 | } |
| 67 | if report.Profile.ModelFamily != "custom/unknown" || report.Profile.RuntimeProfile != "economy" { |
| 68 | t.Fatalf("profile = %+v", report.Profile) |
| 69 | } |
| 70 | if report.Profile.CollaborationMode != "plan" || report.Profile.ToolApprovalMode != "yolo" || !report.Profile.GoalActive || !report.Profile.Recovered { |
| 71 | t.Fatalf("profile modes = %+v", report.Profile) |
| 72 | } |
| 73 | if report.Transcript.ToolCalls != 2 || report.Transcript.WriterCalls != 1 || report.Transcript.VerificationCalls != 1 || report.Transcript.CompactionSummaries != 1 { |
| 74 | t.Fatalf("transcript = %+v", report.Transcript) |
| 75 | } |
| 76 | if report.Transcript.ToolCallTurnsWithoutReasoning != 1 { |
| 77 | t.Fatalf("tool-call turns without reasoning = %d", report.Transcript.ToolCallTurnsWithoutReasoning) |
| 78 | } |
| 79 | if report.Usage.CacheHitPercent == nil || *report.Usage.CacheHitPercent != 75 { |
| 80 | t.Fatalf("usage = %+v", report.Usage) |
| 81 | } |
| 82 | if report.Signals.ExecutorRequests != 2 || report.Signals.PlannerRequests != 1 || report.Signals.SubagentRequests != 1 || report.Signals.OtherRequests != 1 { |
| 83 | t.Fatalf("signals = %+v", report.Signals) |
| 84 | } |
| 85 | |
| 86 | jsonData, err := json.Marshal(report) |
| 87 | if err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | text := RenderQualityText(report) |
| 91 | for _, output := range []string{string(jsonData), text} { |
| 92 | for _, forbidden := range []string{secret, dir, "/private/example", "private-session-id", "supersecret-model", "corp.private.example"} { |
| 93 | if strings.Contains(output, forbidden) { |
| 94 | t.Fatalf("quality summary leaked %q:\n%s", forbidden, output) |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | if !strings.Contains(text, "privacy: transcript text, paths") { |
| 99 | t.Fatalf("text missing privacy statement:\n%s", text) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | func TestCollectQualityWithoutTelemetryStaysUseful(t *testing.T) { |
| 104 | path := filepath.Join(t.TempDir(), "session.jsonl") |
| 105 | if err := os.WriteFile(path, []byte(`{"role":"user","content":"hello"}`+"\n"), 0o644); err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | report, err := CollectQuality(QualityOptions{Version: "v-test", SessionRef: path}) |
| 109 | if err != nil { |
| 110 | t.Fatalf("CollectQuality: %v", err) |
| 111 | } |
| 112 | if report.Usage.Available { |
| 113 | t.Fatal("usage should be unavailable without desktop telemetry") |
| 114 | } |
| 115 | if len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "telemetry") { |
| 116 | t.Fatalf("warnings = %v", report.Warnings) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func TestQualityTranscriptExcludesPinnedContextRevisions(t *testing.T) { |
| 121 | report := summarizeQualityTranscript([]provider.Message{ |
| 122 | {Role: provider.RoleUser, Origin: provider.MessageOriginHost, Content: "<pinned_context_revision>private pinned body</pinned_context_revision>"}, |
| 123 | {Role: provider.RoleUser, Content: "visible question"}, |
| 124 | }) |
| 125 | if report.Messages != 1 || report.UserMessages != 1 { |
| 126 | t.Fatalf("quality transcript = %+v, want one visible user message", report) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestPublicTokenModeUsesRuntimeProfileNames(t *testing.T) { |
| 131 | for input, want := range map[string]string{ |
| 132 | "": "balanced", |
| 133 | "full": "balanced", |
| 134 | "balanced": "balanced", |
| 135 | "economy": "economy", |
| 136 | "delivery": "delivery", |
| 137 | } { |
| 138 | if got := publicTokenMode(input); got != want { |
| 139 | t.Errorf("publicTokenMode(%q) = %q, want %q", input, got, want) |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 |