| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "encoding/base64" |
| 9 | "encoding/json" |
| 10 | "image" |
| 11 | "image/color" |
| 12 | "image/png" |
| 13 | "net/http/httptest" |
| 14 | "os" |
| 15 | "reasonix/internal/event" |
| 16 | "reasonix/internal/provider" |
| 17 | "reasonix/internal/tool" |
| 18 | "reflect" |
| 19 | "strings" |
| 20 | "sync/atomic" |
| 21 | "testing" |
| 22 | "time" |
| 23 | ) |
| 24 | |
| 25 | func TestLiveMultiProviderNativeSearch(t *testing.T) { |
| 26 | if os.Getenv("REASONIX_LIVE_WEB_SEARCH") != "1" { |
| 27 | t.Skip("live search not enabled") |
| 28 | } |
| 29 | for _, tc := range multiProviderCases() { |
| 30 | if os.Getenv(tc.keyEnv) == "" || tc.protocol == "chat" || !strings.HasPrefix(tc.model, "deepseek-v4-") { |
| 31 | continue |
| 32 | } |
| 33 | t.Run(tc.vendor+"/"+tc.model+"/"+tc.protocol, func(t *testing.T) { |
| 34 | proxy := &officialRecoveryProxy{protocol: tc.protocol, scenario: "search", upstreamURL: tc.upstream()} |
| 35 | srv := httptest.NewServer(proxy) |
| 36 | defer srv.Close() |
| 37 | p := tc.new(t, srv.URL, "search") |
| 38 | reg := tool.NewRegistry() |
| 39 | var calls atomic.Int32 |
| 40 | reg.Add(liveRecoveryEchoTool{executions: &calls}) |
| 41 | sess := NewSession("Use real web search when asked. After searching, call echo once, then cite a source and report the marker.") |
| 42 | sink := &recordSink{} |
| 43 | a := New(p, reg, sess, Options{MaxSteps: 4, MaxOutputTokens: 4096, MissingReasoningWarnStateDir: t.TempDir()}, sink) |
| 44 | ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) |
| 45 | defer cancel() |
| 46 | err := a.Run(ctx, "Search the web for OpenCode Go documentation. Then call echo exactly once. Report one source URL and the marker.") |
| 47 | searches, hits := 0, 0 |
| 48 | for _, m := range sess.Snapshot() { |
| 49 | searches += len(m.ServerSearch) |
| 50 | for _, s := range m.ServerSearch { |
| 51 | hits += len(s.Results) |
| 52 | if provider.HasUsableSearchSources(s.Results) && s.SourcesStatus != provider.SourcesAvailable { |
| 53 | t.Fatal("sources availability missing") |
| 54 | } |
| 55 | if !provider.HasUsableSearchSources(s.Results) && s.SourcesStatus != provider.SourcesNotProvided { |
| 56 | t.Fatal("completed source-free search has no availability state") |
| 57 | } |
| 58 | if len(s.Raw) > 0 { |
| 59 | var raw any |
| 60 | if json.Unmarshal(s.Raw, &raw) == nil { |
| 61 | shape, _ := json.Marshal(liveJSONShape(raw)) |
| 62 | t.Logf("search_item_shape=%s", shape) |
| 63 | } |
| 64 | } |
| 65 | } |
| 66 | } |
| 67 | proxy.mu.Lock() |
| 68 | replies := append([][]byte(nil), proxy.searchResponses...) |
| 69 | requestsOnWire := append([][]byte(nil), proxy.bodies...) |
| 70 | proxy.mu.Unlock() |
| 71 | if tc.protocol == "responses" { |
| 72 | for _, m := range sess.Snapshot() { |
| 73 | for _, s := range m.ServerSearch { |
| 74 | var item map[string]any |
| 75 | if json.Unmarshal(s.Raw, &item) != nil || item["status"] != "completed" { |
| 76 | t.Fatal("completed raw search item missing") |
| 77 | } |
| 78 | replayed := false |
| 79 | for _, body := range requestsOnWire[1:] { |
| 80 | var request struct { |
| 81 | Input []map[string]any `json:"input"` |
| 82 | } |
| 83 | if json.Unmarshal(body, &request) != nil { |
| 84 | t.Fatal("invalid replay request") |
| 85 | } |
| 86 | for _, candidate := range request.Input { |
| 87 | replayed = replayed || reflect.DeepEqual(candidate, item) |
| 88 | } |
| 89 | } |
| 90 | if !replayed { |
| 91 | t.Fatal("search item did not replay unchanged on the next model request") |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | for _, reply := range replies { |
| 96 | for _, line := range bytes.Split(reply, []byte("\n")) { |
| 97 | if !bytes.HasPrefix(line, []byte("data:")) { |
| 98 | continue |
| 99 | } |
| 100 | var frame map[string]any |
| 101 | if json.Unmarshal(bytes.TrimSpace(bytes.TrimPrefix(line, []byte("data:"))), &frame) != nil { |
| 102 | continue |
| 103 | } |
| 104 | if frame["type"] == "response.completed" { |
| 105 | shape, _ := json.Marshal(liveJSONShape(frame["response"])) |
| 106 | t.Logf("terminal_response_shape=%s", shape) |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | requests, prompt, output := 0, 0, 0 |
| 112 | unknown := false |
| 113 | for _, e := range sink.kinds(event.Usage) { |
| 114 | if u := e.Usage; u != nil { |
| 115 | requests += u.RequestCount |
| 116 | prompt += u.PromptTokens |
| 117 | output += u.CompletionTokens |
| 118 | unknown = unknown || u.Unknown |
| 119 | } |
| 120 | } |
| 121 | t.Logf("provider=%s model=%s protocol=%s requests=%d searches=%d sources=%d tools=%d prompt=%d completion=%d unknown_usage=%t", tc.vendor, tc.model, tc.protocol, requests, searches, hits, calls.Load(), prompt, output, unknown) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | if searches == 0 || calls.Load() != 1 { |
| 126 | t.Fatal("native search metadata or exactly-once tool continuation missing") |
| 127 | } |
| 128 | t.Run("structured_sources", func(t *testing.T) { |
| 129 | if hits == 0 { |
| 130 | // A completed Responses search can omit action.sources and all |
| 131 | // message annotations. That proves search/replay, not citations. |
| 132 | t.Log("search completed without structured sources; availability explicitly not_provided") |
| 133 | } |
| 134 | }) |
| 135 | }) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // Preserve JSON field/array structure for diagnostics, never content, opaque |
| 140 | // reasoning, identifiers or URLs. Replies themselves remain in process memory. |
| 141 | func liveJSONShape(v any) any { |
| 142 | switch x := v.(type) { |
| 143 | case map[string]any: |
| 144 | out := map[string]any{} |
| 145 | for k, child := range x { |
| 146 | out[k] = liveJSONShape(child) |
| 147 | } |
| 148 | return out |
| 149 | case []any: |
| 150 | out := make([]any, len(x)) |
| 151 | for i, child := range x { |
| 152 | out[i] = liveJSONShape(child) |
| 153 | } |
| 154 | return out |
| 155 | case string: |
| 156 | return "string" |
| 157 | case nil: |
| 158 | return nil |
| 159 | default: |
| 160 | return "scalar" |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // A generated, non-sensitive image tests actual multimodal wire input. No user |
| 165 | // image, file reader or external image URL is sent to any provider. |
| 166 | func TestLiveMultiProviderVision(t *testing.T) { |
| 167 | fixture := image.NewRGBA(image.Rect(0, 0, 256, 128)) |
| 168 | for y := range 128 { |
| 169 | for x := range 256 { |
| 170 | c := color.RGBA{R: 255, A: 255} |
| 171 | if x >= 128 { |
| 172 | c = color.RGBA{B: 255, A: 255} |
| 173 | } |
| 174 | fixture.SetRGBA(x, y, c) |
| 175 | } |
| 176 | } |
| 177 | var encoded bytes.Buffer |
| 178 | if err := png.Encode(&encoded, fixture); err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | url := "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) |
| 182 | for _, tc := range multiProviderCases() { |
| 183 | if os.Getenv(tc.keyEnv) == "" || tc.model != "deepseek-v4-flash-vision-exp" { |
| 184 | continue |
| 185 | } |
| 186 | t.Run(tc.vendor+"/"+tc.protocol, func(t *testing.T) { |
| 187 | p := tc.new(t, "", "vision") |
| 188 | ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) |
| 189 | defer cancel() |
| 190 | ctx = provider.WithManagedRecovery(provider.WithRequestAttemptCounter(ctx)) |
| 191 | ch, err := p.Stream(ctx, provider.Request{Messages: []provider.Message{{Role: provider.RoleUser, Content: "Identify the two solid colors, left then right. Reply exactly two lowercase English words separated by a comma.", Images: []string{url}}}, MaxTokens: 4096}) |
| 192 | if err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | var text strings.Builder |
| 196 | prompt, output := 0, 0 |
| 197 | for c := range ch { |
| 198 | if c.Err != nil { |
| 199 | t.Fatal(c.Err) |
| 200 | } |
| 201 | if c.Type == provider.ChunkText { |
| 202 | text.WriteString(c.Text) |
| 203 | } |
| 204 | if c.Usage != nil { |
| 205 | prompt = c.Usage.PromptTokens |
| 206 | output = c.Usage.CompletionTokens |
| 207 | } |
| 208 | } |
| 209 | answer := strings.ToLower(strings.TrimSpace(text.String())) |
| 210 | answer = strings.ReplaceAll(answer, " ", "") |
| 211 | t.Logf("provider=%s protocol=%s requests=%d prompt=%d completion=%d correct=%t", tc.vendor, tc.protocol, provider.RequestAttemptCount(ctx), prompt, output, answer == "red,blue") |
| 212 | if answer != "red,blue" { |
| 213 | t.Fatalf("color answer mismatch: %q", text.String()) |
| 214 | } |
| 215 | }) |
| 216 | } |
| 217 | } |
| 218 |