| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "os" |
| 8 | "strings" |
| 9 | "sync/atomic" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "reasonix/internal/event" |
| 14 | "reasonix/internal/provider" |
| 15 | "reasonix/internal/provider/anthropic" |
| 16 | "reasonix/internal/provider/openai" |
| 17 | "reasonix/internal/provider/responses" |
| 18 | "reasonix/internal/tool" |
| 19 | ) |
| 20 | |
| 21 | // TestLiveOpenCodeGoDeepSeekAgentToolLoops repeatedly exercises the production |
| 22 | // Agent recovery boundary against both strict OpenCode Go Flash protocols. |
| 23 | func TestLiveOpenCodeGoDeepSeekAgentToolLoops(t *testing.T) { |
| 24 | key := os.Getenv("OPENCODE_GO_API_KEY") |
| 25 | if key == "" { |
| 26 | t.Skip("OPENCODE_GO_API_KEY not set") |
| 27 | } |
| 28 | providers := []struct { |
| 29 | name string |
| 30 | new func() (provider.Provider, error) |
| 31 | }{ |
| 32 | {name: "anthropic", new: func() (provider.Provider, error) { |
| 33 | return anthropic.New(provider.Config{ |
| 34 | Name: "opencode-go-deepseek-anthropic", BaseURL: "https://opencode.ai/zen/go", Model: "deepseek-v4-flash", APIKey: key, |
| 35 | Extra: map[string]any{ |
| 36 | "api_key_env": "OPENCODE_GO_API_KEY", "reasoning_protocol": "deepseek", |
| 37 | "thinking": "adaptive", "effort": "high", "web_search": true, |
| 38 | }, |
| 39 | }) |
| 40 | }}, |
| 41 | {name: "responses", new: func() (provider.Provider, error) { |
| 42 | return responses.New(responses.Config{ |
| 43 | Name: "opencode-go-deepseek-responses", BaseURL: "https://opencode.ai/zen/go/v1", Model: "deepseek-v4-flash", |
| 44 | APIKey: key, KeyEnv: "OPENCODE_GO_API_KEY", Effort: "high", Mode: "stateless", WebSearch: true, MaxOutputTokens: 512, |
| 45 | }), nil |
| 46 | }}, |
| 47 | } |
| 48 | |
| 49 | for _, tc := range providers { |
| 50 | t.Run(tc.name, func(t *testing.T) { |
| 51 | prov, err := tc.new() |
| 52 | if err != nil { |
| 53 | t.Fatalf("new provider: %v", err) |
| 54 | } |
| 55 | if closer, ok := prov.(interface{ CloseIdleConnections() }); ok { |
| 56 | t.Cleanup(closer.CloseIdleConnections) |
| 57 | } |
| 58 | retries, recovered := runLiveAgentToolLoops(t, prov, 10) |
| 59 | t.Logf("protocol=%s runs=10 tool_executions=10 retry_attempts=%d recovered=%d", tc.name, retries, recovered) |
| 60 | }) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestLiveCompatibleProviderDefaultReasoningAgentToolLoops(t *testing.T) { |
| 65 | providers := []struct { |
| 66 | name, keyEnv, baseURL, model string |
| 67 | extra map[string]any |
| 68 | }{ |
| 69 | {name: "longcat", keyEnv: "LONGCAT_API_KEY", baseURL: "https://api.longcat.chat/openai/v1", model: "LongCat-2.0", extra: map[string]any{"thinking": "enabled", "effort": "enabled"}}, |
| 70 | {name: "zhipu-coding-plan", keyEnv: "GLM_PLAN_API_KEY", baseURL: "https://open.bigmodel.cn/api/coding/paas/v4", model: "glm-5.2", extra: map[string]any{"reasoning_protocol": "glm"}}, |
| 71 | } |
| 72 | for _, tc := range providers { |
| 73 | t.Run(tc.name, func(t *testing.T) { |
| 74 | key := os.Getenv(tc.keyEnv) |
| 75 | if key == "" { |
| 76 | t.Skip(tc.keyEnv + " not set") |
| 77 | } |
| 78 | tc.extra["api_key_env"] = tc.keyEnv |
| 79 | prov, err := openai.New(provider.Config{Name: tc.name, BaseURL: tc.baseURL, Model: tc.model, APIKey: key, Extra: tc.extra}) |
| 80 | if err != nil { |
| 81 | t.Fatalf("new provider: %v", err) |
| 82 | } |
| 83 | if closer, ok := prov.(interface{ CloseIdleConnections() }); ok { |
| 84 | t.Cleanup(closer.CloseIdleConnections) |
| 85 | } |
| 86 | retries, recovered := runLiveAgentToolLoops(t, prov, 5) |
| 87 | t.Logf("provider=%s runs=5 tool_executions=5 retry_attempts=%d recovered=%d", tc.name, retries, recovered) |
| 88 | }) |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | func runLiveAgentToolLoops(t *testing.T, prov provider.Provider, runs int) (retries, recovered int) { |
| 93 | t.Helper() |
| 94 | stateDir := t.TempDir() |
| 95 | for attempt := 1; attempt <= runs; attempt++ { |
| 96 | var executions atomic.Int32 |
| 97 | registry := tool.NewRegistry() |
| 98 | registry.Add(liveRecoveryEchoTool{executions: &executions}) |
| 99 | sink := &recordSink{} |
| 100 | session := NewSession("You are a concise tool-using assistant. Call the requested tool exactly once before answering.") |
| 101 | a := New(prov, registry, session, Options{ |
| 102 | MaxSteps: 4, MaxOutputTokens: 512, MissingReasoningWarnStateDir: stateDir, |
| 103 | }, sink) |
| 104 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) |
| 105 | err := a.Run(ctx, "Call echo exactly once, then reply with the marker result.") |
| 106 | cancel() |
| 107 | if err != nil { |
| 108 | t.Fatalf("attempt %d: %v", attempt, err) |
| 109 | } |
| 110 | if got := executions.Load(); got != 1 { |
| 111 | t.Fatalf("attempt %d tool executions = %d, want 1", attempt, got) |
| 112 | } |
| 113 | messages := session.Snapshot() |
| 114 | if len(messages) == 0 || strings.TrimSpace(messages[len(messages)-1].Content) == "" { |
| 115 | t.Fatalf("attempt %d produced no final assistant text", attempt) |
| 116 | } |
| 117 | retries += sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted) |
| 118 | recovered += sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryRecovered) |
| 119 | } |
| 120 | return retries, recovered |
| 121 | } |
| 122 |