| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "encoding/json" |
| 9 | "os" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "reasonix/internal/event" |
| 16 | "reasonix/internal/provider" |
| 17 | "reasonix/internal/provider/openai" |
| 18 | "reasonix/internal/tool" |
| 19 | ) |
| 20 | |
| 21 | type liveLargeResultTool struct{} |
| 22 | |
| 23 | func (liveLargeResultTool) Name() string { return "live_large_result" } |
| 24 | func (liveLargeResultTool) Description() string { |
| 25 | return "Return the fixed large cache-regression fixture. Call only when the user explicitly requests it." |
| 26 | } |
| 27 | func (liveLargeResultTool) Schema() json.RawMessage { |
| 28 | return json.RawMessage(`{"type":"object","properties":{},"additionalProperties":false}`) |
| 29 | } |
| 30 | func (liveLargeResultTool) ReadOnly() bool { return true } |
| 31 | func (liveLargeResultTool) Execute(context.Context, json.RawMessage) (string, error) { |
| 32 | const sentinel = "LIVE-RAW-MIDDLE-SENTINEL" |
| 33 | return strings.Repeat("R", 128<<10) + sentinel + strings.Repeat("R", (128<<10)-len(sentinel)), nil |
| 34 | } |
| 35 | |
| 36 | type liveCacheCaptureProvider struct { |
| 37 | provider.Provider |
| 38 | mu sync.Mutex |
| 39 | requests []provider.Request |
| 40 | } |
| 41 | |
| 42 | func (p *liveCacheCaptureProvider) Stream(ctx context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 43 | p.mu.Lock() |
| 44 | copyReq := req |
| 45 | copyReq.Messages = append([]provider.Message(nil), req.Messages...) |
| 46 | copyReq.Tools = append([]provider.ToolSchema(nil), req.Tools...) |
| 47 | p.requests = append(p.requests, copyReq) |
| 48 | p.mu.Unlock() |
| 49 | return p.Provider.Stream(ctx, req) |
| 50 | } |
| 51 | |
| 52 | func (p *liveCacheCaptureProvider) snapshot() []provider.Request { |
| 53 | p.mu.Lock() |
| 54 | defer p.mu.Unlock() |
| 55 | return append([]provider.Request(nil), p.requests...) |
| 56 | } |
| 57 | |
| 58 | func TestLiveDeepSeekLargeToolResultCacheTrend(t *testing.T) { |
| 59 | key := os.Getenv("DEEPSEEK_API_KEY") |
| 60 | if key == "" { |
| 61 | t.Skip("DEEPSEEK_API_KEY not set") |
| 62 | } |
| 63 | baseURL := strings.TrimRight(os.Getenv("DEEPSEEK_BASE_URL"), "/") |
| 64 | if baseURL == "" { |
| 65 | baseURL = "https://api.deepseek.com" |
| 66 | } |
| 67 | base, err := openai.New(provider.Config{ |
| 68 | Name: "live-tool-cache", BaseURL: baseURL, Model: "deepseek-v4-flash", APIKey: key, |
| 69 | Extra: map[string]any{"api_key_env": "DEEPSEEK_API_KEY"}, |
| 70 | }) |
| 71 | if err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | if closer, ok := base.(interface{ CloseIdleConnections() }); ok { |
| 75 | t.Cleanup(closer.CloseIdleConnections) |
| 76 | } |
| 77 | capture := &liveCacheCaptureProvider{Provider: base} |
| 78 | reg := tool.NewRegistry() |
| 79 | reg.Add(liveLargeResultTool{}) |
| 80 | var usages []*provider.Usage |
| 81 | sink := event.FuncSink(func(e event.Event) { |
| 82 | if e.Kind == event.Usage && e.Usage != nil { |
| 83 | copyUsage := *e.Usage |
| 84 | usages = append(usages, ©Usage) |
| 85 | } |
| 86 | }) |
| 87 | system := "You are a concise cache-regression agent. Follow explicit tool instructions. " + |
| 88 | strings.Repeat("Keep this stable provider prefix byte-identical across every request. ", 80) |
| 89 | a := New(capture, reg, NewSession(system), Options{ContextWindow: 1_000_000, MaxOutputTokens: 256, MaxSteps: 5}, sink) |
| 90 | |
| 91 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) |
| 92 | defer cancel() |
| 93 | turns := []string{ |
| 94 | "Call live_large_result exactly once, then answer only: received.", |
| 95 | "Do not call tools. Reply only: second.", |
| 96 | "Do not call tools. Reply only: third.", |
| 97 | } |
| 98 | for i, prompt := range turns { |
| 99 | if err := a.Run(ctx, prompt); err != nil { |
| 100 | t.Fatalf("turn %d: %v", i+1, err) |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | requests := capture.snapshot() |
| 105 | if len(requests) < 4 { |
| 106 | t.Fatalf("captured requests=%d, want tool loop plus two ordinary turns", len(requests)) |
| 107 | } |
| 108 | seenBoundedTool := false |
| 109 | for i, req := range requests { |
| 110 | wire, err := json.Marshal(req) |
| 111 | if err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | if bytes.Contains(wire, []byte("LIVE-RAW-MIDDLE-SENTINEL")) { |
| 115 | t.Fatalf("request %d leaked local RawContent sentinel", i+1) |
| 116 | } |
| 117 | for _, msg := range req.Messages { |
| 118 | if msg.Role == provider.RoleTool && msg.Name == "live_large_result" { |
| 119 | seenBoundedTool = true |
| 120 | if msg.RawContent != "" || len(msg.Content) > maxToolOutputBytes { |
| 121 | t.Fatalf("request %d tool bytes: content=%d raw=%d", i+1, len(msg.Content), len(msg.RawContent)) |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | if i > 0 { |
| 126 | previous := requests[i-1].Messages |
| 127 | current := req.Messages |
| 128 | if len(current) >= len(previous) { |
| 129 | for j := range previous { |
| 130 | before, _ := json.Marshal(previous[j]) |
| 131 | after, _ := json.Marshal(current[j]) |
| 132 | if !bytes.Equal(before, after) { |
| 133 | t.Fatalf("request %d changed prior provider message %d", i+1, j) |
| 134 | } |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | } |
| 139 | if !seenBoundedTool { |
| 140 | t.Fatal("model did not execute the large-result fixture") |
| 141 | } |
| 142 | for i, usage := range usages { |
| 143 | t.Logf("live cache turn=%d prompt=%d hit=%d miss=%d", i+1, usage.PromptTokens, usage.CacheHitTokens, usage.CacheMissTokens) |
| 144 | } |
| 145 | } |
| 146 |