| 1 | package boot |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "sync/atomic" |
| 13 | "testing" |
| 14 | |
| 15 | "reasonix/internal/config" |
| 16 | "reasonix/internal/event" |
| 17 | "reasonix/internal/netclient" |
| 18 | "reasonix/internal/provider" |
| 19 | _ "reasonix/internal/provider/responses" |
| 20 | "reasonix/internal/tool" |
| 21 | ) |
| 22 | |
| 23 | func TestIndependentSearchWireAndMainReplay(t *testing.T) { |
| 24 | for _, kind := range []string{"anthropic", "responses"} { |
| 25 | t.Run(kind, func(t *testing.T) { |
| 26 | var mu sync.Mutex |
| 27 | var requests []map[string]any |
| 28 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 29 | var body map[string]any |
| 30 | if err := json.NewDecoder(r.Body).Decode(&body); err != nil { |
| 31 | t.Error(err) |
| 32 | } |
| 33 | mu.Lock() |
| 34 | requests = append(requests, body) |
| 35 | mu.Unlock() |
| 36 | w.Header().Set("Content-Type", "text/event-stream") |
| 37 | if kind == "anthropic" { |
| 38 | fmt.Fprint(w, "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":12,\"output_tokens\":0}}}\n\n") |
| 39 | fmt.Fprint(w, "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"web_search_tool_result\",\"tool_use_id\":\"s1\",\"content\":[{\"type\":\"web_search_result\",\"title\":\"Docs\",\"url\":\"https://example.com/docs\",\"encrypted_content\":\"SECRET\"}]}}\n\n") |
| 40 | fmt.Fprint(w, "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n") |
| 41 | fmt.Fprint(w, "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Search summary\"}}\n\n") |
| 42 | fmt.Fprint(w, "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":7}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") |
| 43 | } else { |
| 44 | fmt.Fprint(w, `data: {"type":"response.output_item.done","item":{"id":"s1","type":"web_search_call","status":"completed","action":{"type":"search","queries":["test"]}}} |
| 45 | |
| 46 | data: {"type":"response.output_item.done","item":{"id":"s2","type":"web_search_call","status":"completed","action":{"type":"open_page","url":"https://example.com/docs"}}} |
| 47 | |
| 48 | `) |
| 49 | fmt.Fprint(w, "data: {\"type\":\"response.output_text.delta\",\"delta\":\"Search summary\"}\n\n") |
| 50 | fmt.Fprint(w, "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp1\",\"status\":\"completed\",\"usage\":{\"input_tokens\":12,\"output_tokens\":7,\"total_tokens\":19}}}\n\n") |
| 51 | } |
| 52 | })) |
| 53 | defer srv.Close() |
| 54 | on := true |
| 55 | entry := config.ProviderEntry{Name: "search", Kind: kind, BaseURL: srv.URL, Model: "m", WebSearch: &on, Thinking: "enabled", ResponsesMode: "stateful"} |
| 56 | cfg := &config.Config{Providers: []config.ProviderEntry{entry}} |
| 57 | reg := tool.NewRegistry() |
| 58 | var usageEvents []event.Event |
| 59 | addWebSearch(reg, cfg, &entry, netclient.ProxySpec{Mode: netclient.ModeOff}, event.FuncSink(func(e event.Event) { usageEvents = append(usageEvents, e) })) |
| 60 | applyUnifiedProviderToolSurface(reg) |
| 61 | if schemas := reg.Schemas(); len(schemas) != 1 || schemas[0].Name != "web_search" { |
| 62 | t.Fatalf("search not exposed: %+v", schemas) |
| 63 | } |
| 64 | search, ok := reg.Get("web_search") |
| 65 | if !ok { |
| 66 | t.Fatal("missing search tool") |
| 67 | } |
| 68 | for _, query := range []string{"first", "second"} { |
| 69 | output, err := search.Execute(context.Background(), json.RawMessage(`{"query":"`+query+`"}`)) |
| 70 | if err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | sources := provider.ParseServerSearchOutput(output) |
| 74 | if len(sources) != 1 || sources[0].URL != "https://example.com/docs" || !strings.Contains(output, "Search summary") || strings.Contains(output, "SECRET") { |
| 75 | t.Fatalf("bad output: %s", output) |
| 76 | } |
| 77 | } |
| 78 | if len(usageEvents) != 2 || usageEvents[0].UsageSource != "web-search" || usageEvents[0].Usage.RequestCount != 1 { |
| 79 | t.Fatalf("usage not accounted: %+v", usageEvents) |
| 80 | } |
| 81 | main, err := NewProviderWithProxy(&entry, netclient.ProxySpec{Mode: netclient.ModeOff}) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | old := provider.Message{Role: provider.RoleAssistant, Content: "old search"} |
| 86 | if kind == "anthropic" { |
| 87 | old.ServerSearch = []provider.ServerSearchCall{{ID: "old", Query: "legacy", Raw: json.RawMessage(`[]`)}} |
| 88 | } else { |
| 89 | old.ResponsesItems = []json.RawMessage{json.RawMessage(`{"id":"old","type":"web_search_call","status":"completed","action":{"type":"search","query":"legacy"}}`)} |
| 90 | } |
| 91 | stream, err := main.Stream(context.Background(), provider.Request{Messages: []provider.Message{{Role: provider.RoleUser, Content: "MAIN HISTORY"}, old}, Tools: reg.Schemas()}) |
| 92 | if err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | for range stream { |
| 96 | } |
| 97 | mu.Lock() |
| 98 | defer mu.Unlock() |
| 99 | if len(requests) != 3 { |
| 100 | t.Fatalf("got %d requests", len(requests)) |
| 101 | } |
| 102 | for i, req := range requests { |
| 103 | tools := req["tools"].([]any) |
| 104 | if len(tools) != 1 { |
| 105 | t.Fatalf("duplicate or missing tools: %+v", tools) |
| 106 | } |
| 107 | wireTool := tools[0].(map[string]any) |
| 108 | if i < 2 { |
| 109 | if wireTool["type"] != "web_search_20250305" && wireTool["type"] != "web_search" { |
| 110 | t.Fatalf("search lacks native tool: %+v", wireTool) |
| 111 | } |
| 112 | b, _ := json.Marshal(req) |
| 113 | if strings.Contains(string(b), "MAIN HISTORY") || strings.Contains(string(b), "previous_response_id") || (i == 1 && strings.Contains(string(b), "first")) { |
| 114 | t.Fatalf("search inherited history: %s", b) |
| 115 | } |
| 116 | } else { |
| 117 | if wireTool["type"] == "web_search" || wireTool["type"] == "web_search_20250305" { |
| 118 | t.Fatal("main request still contains native search") |
| 119 | } |
| 120 | b, _ := json.Marshal(req) |
| 121 | if !strings.Contains(string(b), `"old"`) || !strings.Contains(string(b), `legacy`) { |
| 122 | t.Fatalf("legacy search replay was lost: %s", b) |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | }) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestIndependentSearchDoesNotFollowRedirects(t *testing.T) { |
| 131 | var redirected atomic.Int32 |
| 132 | target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { redirected.Add(1) })) |
| 133 | defer target.Close() |
| 134 | source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 135 | http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) |
| 136 | })) |
| 137 | defer source.Close() |
| 138 | for _, kind := range []string{"anthropic", "responses"} { |
| 139 | on := true |
| 140 | entry := config.ProviderEntry{Name: "search", Kind: kind, BaseURL: source.URL, Model: "m", WebSearch: &on} |
| 141 | reg := tool.NewRegistry() |
| 142 | addWebSearch(reg, &config.Config{}, &entry, netclient.ProxySpec{Mode: netclient.ModeOff}, event.Discard) |
| 143 | search, _ := reg.Get("web_search") |
| 144 | if _, err := search.Execute(context.Background(), json.RawMessage(`{"query":"test"}`)); err == nil { |
| 145 | t.Fatalf("%s redirect accepted", kind) |
| 146 | } |
| 147 | } |
| 148 | if redirected.Load() != 0 { |
| 149 | t.Fatal("search followed a credential-bearing redirect") |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | func TestBuildExposesIndependentSearch(t *testing.T) { |
| 154 | isolateConfigHome(t) |
| 155 | dir := robustTempDir(t) |
| 156 | t.Chdir(dir) |
| 157 | writeFile(t, dir, "reasonix.toml", `default_model = "local/m" |
| 158 | [[providers]] |
| 159 | name = "local" |
| 160 | kind = "anthropic" |
| 161 | base_url = "http://localhost:12345" |
| 162 | model = "m" |
| 163 | web_search = true |
| 164 | `) |
| 165 | ctrl, err := Build(context.Background(), Options{Stderr: io.Discard}) |
| 166 | if err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | defer ctrl.Close() |
| 170 | // The assembled provider-visible inventory is part of the runtime snapshot. |
| 171 | for _, entry := range ctrl.ToolContractEntries() { |
| 172 | if entry.Name == "web_search" { |
| 173 | return |
| 174 | } |
| 175 | } |
| 176 | t.Fatal("Build did not install search") |
| 177 | } |
| 178 | |
| 179 | func TestIndependentSearchHonorsOfflineAndToolAllowlist(t *testing.T) { |
| 180 | for _, tc := range []struct { |
| 181 | name string |
| 182 | offline bool |
| 183 | enabled []string |
| 184 | want bool |
| 185 | }{ |
| 186 | {"default", false, nil, true}, |
| 187 | {"offline", true, nil, false}, |
| 188 | {"excluded", false, []string{"bash"}, false}, |
| 189 | {"included", false, []string{"web_search"}, true}, |
| 190 | } { |
| 191 | t.Run(tc.name, func(t *testing.T) { |
| 192 | cfg := &config.Config{} |
| 193 | cfg.Environment.Offline = tc.offline |
| 194 | cfg.Tools.Enabled = tc.enabled |
| 195 | on := true |
| 196 | entry := &config.ProviderEntry{Name: "search", Kind: "anthropic", BaseURL: "http://localhost:8080", Model: "m", WebSearch: &on} |
| 197 | reg := tool.NewRegistry() |
| 198 | addWebSearch(reg, cfg, entry, netclient.ProxySpec{Mode: netclient.ModeOff}, event.Discard) |
| 199 | _, found := reg.Get("web_search") |
| 200 | if found != tc.want { |
| 201 | t.Fatalf("registered=%v want %v", found, tc.want) |
| 202 | } |
| 203 | }) |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | func TestAssignedSearchRuntimeSnapshotAndStableSchema(t *testing.T) { |
| 208 | var mu sync.Mutex |
| 209 | var models []string |
| 210 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 211 | var body map[string]any |
| 212 | if err := json.NewDecoder(r.Body).Decode(&body); err != nil { |
| 213 | t.Error(err) |
| 214 | return |
| 215 | } |
| 216 | mu.Lock() |
| 217 | models = append(models, body["model"].(string)) |
| 218 | mu.Unlock() |
| 219 | messages := body["input"].([]any) |
| 220 | if len(messages) != 1 { |
| 221 | t.Error("search inherited conversation") |
| 222 | } |
| 223 | w.Header().Set("Content-Type", "text/event-stream") |
| 224 | fmt.Fprint(w, "data: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"s\",\"type\":\"web_search_call\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"query\":\"test\"}}}\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"summary\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r\",\"status\":\"completed\",\"usage\":{\"input_tokens\":2,\"output_tokens\":3}}}\n\n") |
| 225 | })) |
| 226 | defer srv.Close() |
| 227 | on, off := true, false |
| 228 | c := &config.Config{Providers: []config.ProviderEntry{{Name: "search", Kind: "responses", BaseURL: srv.URL, Models: []string{"first", "second"}, Default: "first", WebSearch: &on}}, Agent: config.AgentConfig{WebSearchModel: "search/first"}} |
| 229 | current := config.ProviderEntry{Name: "chat", Kind: "responses", Model: "main", WebSearch: &off} |
| 230 | var events []event.Event |
| 231 | sink := event.FuncSink(func(e event.Event) { mu.Lock(); defer mu.Unlock(); events = append(events, e) }) |
| 232 | old := tool.NewRegistry() |
| 233 | addWebSearch(old, c, ¤t, netclient.ProxySpec{Mode: netclient.ModeOff}, sink) |
| 234 | c.Agent.WebSearchModel = "search/second" |
| 235 | next := tool.NewRegistry() |
| 236 | addWebSearch(next, c, ¤t, netclient.ProxySpec{Mode: netclient.ModeOff}, sink) |
| 237 | firstSchema, _ := json.Marshal(old.Schemas()) |
| 238 | secondSchema, _ := json.Marshal(next.Schemas()) |
| 239 | if string(firstSchema) != string(secondSchema) { |
| 240 | t.Fatal("assignment changed main tool prefix") |
| 241 | } |
| 242 | var wg sync.WaitGroup |
| 243 | for range 2 { |
| 244 | wg.Go(func() { |
| 245 | search, _ := old.Get("web_search") |
| 246 | if _, err := search.Execute(context.Background(), json.RawMessage(`{"query":"test"}`)); err != nil { |
| 247 | t.Error(err) |
| 248 | } |
| 249 | }) |
| 250 | } |
| 251 | wg.Wait() |
| 252 | search, _ := next.Get("web_search") |
| 253 | if _, err := search.Execute(context.Background(), json.RawMessage(`{"query":"test"}`)); err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | mu.Lock() |
| 257 | defer mu.Unlock() |
| 258 | if len(models) != 3 || models[0] != "first" || models[1] != "first" || models[2] != "second" { |
| 259 | t.Fatalf("runtime selection changed: %v", models) |
| 260 | } |
| 261 | var refs []string |
| 262 | for _, e := range events { |
| 263 | if e.Kind == event.Usage { |
| 264 | if e.UsageSource != "web-search" { |
| 265 | t.Fatal("wrong usage source") |
| 266 | } |
| 267 | refs = append(refs, e.ModelRef) |
| 268 | } |
| 269 | } |
| 270 | if len(refs) != 3 || refs[0] != "search/first" || refs[2] != "search/second" { |
| 271 | t.Fatalf("wrong usage models: %v", refs) |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | func TestInvalidAssignedSearchNotifiesWithoutFallback(t *testing.T) { |
| 276 | on := true |
| 277 | c := &config.Config{Providers: []config.ProviderEntry{{Name: "fallback", Kind: "responses", BaseURL: "http://localhost:1234", Model: "m", WebSearch: &on}}, Agent: config.AgentConfig{WebSearchModel: "removed/m"}} |
| 278 | reg := tool.NewRegistry() |
| 279 | var notices []event.Event |
| 280 | addWebSearch(reg, c, nil, netclient.ProxySpec{}, event.FuncSink(func(e event.Event) { notices = append(notices, e) })) |
| 281 | if _, found := reg.Get("web_search"); found { |
| 282 | t.Fatal("registered fallback") |
| 283 | } |
| 284 | if len(notices) != 1 || notices[0].Code != "web_search_model_unavailable" { |
| 285 | t.Fatalf("notices: %+v", notices) |
| 286 | } |
| 287 | } |
| 288 |