| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "reasonix/internal/agent" |
| 13 | "reasonix/internal/control" |
| 14 | "reasonix/internal/event" |
| 15 | "reasonix/internal/provider" |
| 16 | ) |
| 17 | |
| 18 | func TestRunOutputTextPrintsOnlyFinalMessage(t *testing.T) { |
| 19 | var out bytes.Buffer |
| 20 | sink := newRunOutputSink(&out, runOutputText) |
| 21 | sink.Emit(event.Event{Kind: event.Text, Text: "streamed "}) |
| 22 | sink.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "bash", Output: "noise"}}) |
| 23 | sink.Emit(event.Event{Kind: event.Message, Text: "final answer"}) |
| 24 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 25 | t.Fatal(err) |
| 26 | } |
| 27 | if got := out.String(); got != "final answer\n" { |
| 28 | t.Fatalf("text output = %q", got) |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | func TestRunOutputJSONResult(t *testing.T) { |
| 33 | var out bytes.Buffer |
| 34 | sink := newRunOutputSink(&out, runOutputJSON) |
| 35 | sink.Emit(event.Event{Kind: event.Message, Text: "done"}) |
| 36 | sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{ |
| 37 | PromptTokens: 12, CompletionTokens: 3, CacheHitTokens: 8, CacheMissTokens: 4, Estimated: true, |
| 38 | }}) |
| 39 | sink.Emit(event.Event{Kind: event.TurnDone}) |
| 40 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | var result runResult |
| 44 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 45 | t.Fatalf("decode result: %v\n%s", err, out.String()) |
| 46 | } |
| 47 | if result.Type != "result" || result.Subtype != "success" || result.IsError || result.Result != "done" || result.SessionID != "abc" { |
| 48 | t.Fatalf("result = %+v", result) |
| 49 | } |
| 50 | if result.Usage.InputTokens != 12 || result.Usage.OutputTokens != 3 || result.Usage.CacheReadInputTokens != 8 || result.Usage.CacheCreationInputTokens != 4 { |
| 51 | t.Fatalf("usage = %+v", result.Usage) |
| 52 | } |
| 53 | if !result.Usage.Estimated { |
| 54 | t.Fatalf("usage lost estimated marker: %+v", result.Usage) |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | func TestRunOutputJSONIncludesAuthenticationRecovery(t *testing.T) { |
| 59 | var out bytes.Buffer |
| 60 | sink := newRunOutputSink(&out, runOutputJSON) |
| 61 | err := &control.AuthenticationError{State: control.AuthenticationState{ |
| 62 | Status: control.AuthenticationMissingCredential, |
| 63 | Code: "missing_credential", |
| 64 | }} |
| 65 | if finalizeErr := sink.Finalize("", time.Now(), err); finalizeErr != nil { |
| 66 | t.Fatal(finalizeErr) |
| 67 | } |
| 68 | var result runResult |
| 69 | if decodeErr := json.Unmarshal(out.Bytes(), &result); decodeErr != nil { |
| 70 | t.Fatal(decodeErr) |
| 71 | } |
| 72 | if result.ErrorCode != "missing_credential" || result.Authentication != string(control.AuthenticationMissingCredential) { |
| 73 | t.Fatalf("authentication metadata = %+v", result) |
| 74 | } |
| 75 | if got := strings.Join(result.Recovery, ","); got != "configure_credentials,select_model,diagnose_credentials" { |
| 76 | t.Fatalf("recovery actions = %q", got) |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | func TestRunOutputJSONIncludesCurrencyAwareCostFields(t *testing.T) { |
| 81 | for _, tt := range []struct { |
| 82 | name string |
| 83 | currency string |
| 84 | wantCode string |
| 85 | }{ |
| 86 | {name: "USD", currency: "$", wantCode: "USD"}, |
| 87 | {name: "CNY", currency: "¥", wantCode: "CNY"}, |
| 88 | } { |
| 89 | t.Run(tt.name, func(t *testing.T) { |
| 90 | var out bytes.Buffer |
| 91 | sink := newRunOutputSink(&out, runOutputJSON) |
| 92 | sink.Emit(event.Event{ |
| 93 | Kind: event.Usage, |
| 94 | Usage: &provider.Usage{PromptTokens: 1_000_000, CompletionTokens: 500_000}, |
| 95 | Pricing: &provider.Pricing{Input: 1, Output: 2, Currency: tt.currency}, |
| 96 | }) |
| 97 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | var result runResult |
| 101 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if result.TotalCost != 2 || result.TotalCostUSD != result.TotalCost || result.Currency != tt.wantCode { |
| 105 | t.Fatalf("currency-aware result = %+v", result) |
| 106 | } |
| 107 | }) |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | func TestRunOutputJSONTotalsMoreThanAuditLimit(t *testing.T) { |
| 112 | var out bytes.Buffer |
| 113 | sink := newRunOutputSink(&out, runOutputJSON) |
| 114 | for range 65 { |
| 115 | sink.Emit(event.Event{ |
| 116 | Kind: event.Usage, |
| 117 | Usage: &provider.Usage{PromptTokens: 1_000_000}, |
| 118 | Pricing: &provider.Pricing{Input: 1, Currency: "USD"}, |
| 119 | }) |
| 120 | } |
| 121 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | var result runResult |
| 125 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | if !result.CostComplete || result.TotalCost != 65 || result.Currency != "USD" { |
| 129 | t.Fatalf("65-event total was truncated: %+v", result) |
| 130 | } |
| 131 | if result.CostQuote == nil || result.CostQuote.Selected == nil || result.CostQuote.Selected.Amount != "65" { |
| 132 | t.Fatalf("65-event aggregate quote = %+v", result.CostQuote) |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | func TestRunOutputJSONRejectsMixedPricingCurrencies(t *testing.T) { |
| 137 | // Mixed originals no longer error: they emit original_costs + cost_complete=false |
| 138 | // when a shared display valuation is unavailable (no FX table in unit test). |
| 139 | var out bytes.Buffer |
| 140 | sink := newRunOutputSink(&out, runOutputJSON) |
| 141 | for _, currency := range []string{"$", "¥"} { |
| 142 | sink.Emit(event.Event{ |
| 143 | Kind: event.Usage, |
| 144 | Usage: &provider.Usage{PromptTokens: 1_000_000}, |
| 145 | Pricing: &provider.Pricing{Input: 1, Currency: currency}, |
| 146 | }) |
| 147 | } |
| 148 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 149 | t.Fatalf("Finalize mixed currencies: %v", err) |
| 150 | } |
| 151 | var result runResult |
| 152 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | if !result.CostComplete || result.DisplayComplete || result.DisplayStatus != "bucketed" { |
| 156 | t.Fatalf("expected complete cost facts but bucketed display, got %+v", result) |
| 157 | } |
| 158 | if len(result.OriginalCosts) < 2 { |
| 159 | t.Fatalf("expected per-currency original_costs, got %+v", result.OriginalCosts) |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | func TestRunOutputSessionIDPreservesExistingFormats(t *testing.T) { |
| 164 | const raw = "20260723-120000.000000000-model" |
| 165 | identityKey := bytes.Repeat([]byte{0x41}, machineIdentityKeyBytes) |
| 166 | for _, format := range []runOutputFormat{runOutputText, runOutputJSON, runOutputStreamJSON} { |
| 167 | if got := runOutputSessionID(format, raw, nil); got != raw { |
| 168 | t.Fatalf("format %q session id = %q, want raw id %q", format, got, raw) |
| 169 | } |
| 170 | } |
| 171 | if got := runOutputSessionID(runOutputEventsJSONL, raw, identityKey); got != machineSessionIDWithKey(raw, identityKey) { |
| 172 | t.Fatalf("events-jsonl session id = %q, want machine id %q", got, machineSessionIDWithKey(raw, identityKey)) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | func TestRunOutputStreamJSONEndsWithErrorResult(t *testing.T) { |
| 177 | var out bytes.Buffer |
| 178 | sink := newRunOutputSink(&out, runOutputStreamJSON) |
| 179 | sink.Emit(event.Event{Kind: event.Text, Text: "partial"}) |
| 180 | runErr := errors.New("provider failed") |
| 181 | if err := sink.Finalize("abc", time.Now(), runErr); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | lines := strings.Split(strings.TrimSpace(out.String()), "\n") |
| 185 | if len(lines) != 2 { |
| 186 | t.Fatalf("stream lines = %d, want 2\n%s", len(lines), out.String()) |
| 187 | } |
| 188 | var wire map[string]any |
| 189 | if err := json.Unmarshal([]byte(lines[0]), &wire); err != nil || wire["kind"] != "text" { |
| 190 | t.Fatalf("wire event = %#v, err=%v", wire, err) |
| 191 | } |
| 192 | var result runResult |
| 193 | if err := json.Unmarshal([]byte(lines[1]), &result); err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | if !result.IsError || result.Subtype != "error_during_execution" || result.Result != runErr.Error() { |
| 197 | t.Fatalf("error result = %+v", result) |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | func TestRunOutputEventsJSONLIsStructuredAndRedacted(t *testing.T) { |
| 202 | var out bytes.Buffer |
| 203 | sink := newRunOutputSink(&out, runOutputEventsJSONL) |
| 204 | sink.Emit(event.Event{Kind: event.Text, Text: "PRIVATE ANSWER"}) |
| 205 | sink.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{ |
| 206 | ID: "PRIVATE TOOL ID", Name: "PRIVATE TOOL NAME", Args: `{"command":"PRIVATE COMMAND"}`, Output: "PRIVATE OUTPUT", Err: "PRIVATE ERROR", |
| 207 | }}) |
| 208 | sink.Emit(event.Event{Kind: event.ToolProgress, Tool: event.Tool{ID: "PRIVATE TOOL ID", Name: "PRIVATE TOOL NAME"}}) |
| 209 | sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{PromptTokens: 4, CompletionTokens: 2, Estimated: true}}) |
| 210 | if err := sink.Finalize("session-1", time.Now(), nil); err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | lines := strings.Split(strings.TrimSpace(out.String()), "\n") |
| 214 | if len(lines) != 5 { |
| 215 | t.Fatalf("event lines = %d, output = %s", len(lines), out.String()) |
| 216 | } |
| 217 | var toolAliases []struct { |
| 218 | ToolID string `json:"tool_id"` |
| 219 | ToolName string `json:"tool_name"` |
| 220 | } |
| 221 | var sawEstimatedUsage bool |
| 222 | for i, line := range lines { |
| 223 | var payload map[string]any |
| 224 | if err := json.Unmarshal([]byte(line), &payload); err != nil { |
| 225 | t.Fatalf("line %d: %v", i, err) |
| 226 | } |
| 227 | if payload["schema_version"] != float64(machineSchemaVersion) || payload["sequence"] != float64(i+1) { |
| 228 | t.Fatalf("line %d envelope = %#v", i, payload) |
| 229 | } |
| 230 | if payload["kind"] == "tool_result" || payload["kind"] == "tool_progress" { |
| 231 | var aliases struct { |
| 232 | ToolID string `json:"tool_id"` |
| 233 | ToolName string `json:"tool_name"` |
| 234 | } |
| 235 | if err := json.Unmarshal([]byte(line), &aliases); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | toolAliases = append(toolAliases, aliases) |
| 239 | } |
| 240 | if payload["kind"] == "usage" { |
| 241 | usage, ok := payload["usage"].(map[string]any) |
| 242 | sawEstimatedUsage = ok && usage["estimated"] == true |
| 243 | } |
| 244 | } |
| 245 | if strings.Contains(out.String(), "PRIVATE") || !strings.Contains(out.String(), `"kind":"run_done"`) { |
| 246 | t.Fatalf("event stream was not redacted or terminated: %s", out.String()) |
| 247 | } |
| 248 | if len(toolAliases) != 2 || toolAliases[0].ToolID != "tool_1" || toolAliases[0].ToolName != "tool_name_1" || toolAliases[1] != toolAliases[0] { |
| 249 | t.Fatalf("tool aliases = %+v, want stable per-run opaque identities", toolAliases) |
| 250 | } |
| 251 | if !sawEstimatedUsage { |
| 252 | t.Fatalf("event stream lost estimated usage marker: %s", out.String()) |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | func TestEventsJSONLHasOneCanonicalFlag(t *testing.T) { |
| 257 | if _, err := parseRunOutputFormat("events-jsonl"); err == nil { |
| 258 | t.Fatal("events-jsonl must use the dedicated --events-jsonl flag") |
| 259 | } |
| 260 | var code int |
| 261 | stderr := captureStderr(t, func() { |
| 262 | code = runAgent([]string{"--events-jsonl", "--output-format", "json", "task"}, "dev") |
| 263 | }) |
| 264 | if code != 2 || !strings.Contains(stderr, "cannot be combined") { |
| 265 | t.Fatalf("exit=%d stderr=%q", code, stderr) |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | func TestRunOutputJSONClassifiesRecoveryPauseAsControlledOutcome(t *testing.T) { |
| 270 | var out bytes.Buffer |
| 271 | sink := newRunOutputSink(&out, runOutputJSON) |
| 272 | runErr := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "automatic recovery paused"}) |
| 273 | if err := sink.Finalize("abc", time.Now(), runErr); err != nil { |
| 274 | t.Fatal(err) |
| 275 | } |
| 276 | var result runResult |
| 277 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 278 | t.Fatal(err) |
| 279 | } |
| 280 | if result.IsError || result.Subtype != event.TurnOutcomeRecoveryPaused || result.Result != runErr.Error() || result.NumTurns != 1 { |
| 281 | t.Fatalf("recovery pause result = %+v", result) |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | func TestRunOutputEventsJSONLClassifiesRecoveryPauseAsControlledOutcome(t *testing.T) { |
| 286 | var out bytes.Buffer |
| 287 | sink := newRunOutputSink(&out, runOutputEventsJSONL) |
| 288 | runErr := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "automatic recovery paused"}) |
| 289 | if err := sink.Finalize("machine-session", time.Now(), runErr); err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | var result machineRunDone |
| 293 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | if !result.OK || result.NumTurns != 1 || result.SessionID != "machine-session" { |
| 297 | t.Fatalf("recovery pause result = %+v", result) |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | func TestRunOutputJSONPreservesCompletionUncertainAsControlledOutcome(t *testing.T) { |
| 302 | var out bytes.Buffer |
| 303 | sink := newRunOutputSink(&out, runOutputJSON) |
| 304 | runErr := &agent.CompletionUncertainError{Cause: agent.CompletionUncertainContextTool} |
| 305 | if err := sink.Finalize("abc", time.Now(), runErr); err != nil { |
| 306 | t.Fatal(err) |
| 307 | } |
| 308 | var result runResult |
| 309 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | if result.IsError || result.Subtype != event.TurnOutcomeCompletionUncertain || result.Result != runErr.Error() || result.NumTurns != 1 { |
| 313 | t.Fatalf("completion uncertain result = %+v", result) |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | func TestClassifyRunCompletion(t *testing.T) { |
| 318 | pause := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "paused"}) |
| 319 | if got := classifyRunCompletion(pause); got.outcome != event.TurnOutcomeRecoveryPaused || got.isError || got.exitCode != 0 { |
| 320 | t.Fatalf("pause completion = %+v", got) |
| 321 | } |
| 322 | uncertain := fmt.Errorf("wrapped: %w", &agent.CompletionUncertainError{Cause: agent.CompletionUncertainContextTool}) |
| 323 | if got := classifyRunCompletion(uncertain); got.outcome != event.TurnOutcomeCompletionUncertain || got.subtype != event.TurnOutcomeCompletionUncertain || got.isError || got.exitCode != 1 { |
| 324 | t.Fatalf("completion uncertain = %+v", got) |
| 325 | } |
| 326 | incomplete := fmt.Errorf("wrapped: %w", &agent.IncompleteReadError{Reason: "page budget"}) |
| 327 | if got := classifyRunCompletion(incomplete); got.outcome != event.TurnOutcomeIncompleteRead || got.subtype != event.TurnOutcomeIncompleteRead || got.isError || got.exitCode != 1 { |
| 328 | t.Fatalf("incomplete read = %+v", got) |
| 329 | } |
| 330 | if got := classifyRunCompletion(errors.New("provider failed")); got.outcome != "" || !got.isError || got.exitCode != 1 { |
| 331 | t.Fatalf("error completion = %+v", got) |
| 332 | } |
| 333 | if got := classifyRunCompletion(nil); got.outcome != "" || got.isError || got.exitCode != 0 { |
| 334 | t.Fatalf("success completion = %+v", got) |
| 335 | } |
| 336 | } |
| 337 |