| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "os" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "reasonix/internal/provider" |
| 13 | "reasonix/internal/provider/openai" |
| 14 | ) |
| 15 | |
| 16 | type liveOutputFieldCacheResult struct { |
| 17 | prompt int |
| 18 | hit int |
| 19 | miss int |
| 20 | } |
| 21 | |
| 22 | // Opt-in A/B: same prefix+tools, compare cache-hit when the output field is |
| 23 | // omitted vs proactively limited. Skips when credentials or cache telemetry |
| 24 | // are unavailable. |
| 25 | func TestLiveSharedWindowOutputFieldCache(t *testing.T) { |
| 26 | key := os.Getenv("DEEPSEEK_API_KEY") |
| 27 | if key == "" { |
| 28 | t.Skip("DEEPSEEK_API_KEY not set") |
| 29 | } |
| 30 | baseURL := strings.TrimRight(os.Getenv("DEEPSEEK_BASE_URL"), "/") |
| 31 | if baseURL == "" { |
| 32 | baseURL = "https://api.deepseek.com" |
| 33 | } |
| 34 | chat, err := openai.New(provider.Config{Name: "live-ds", BaseURL: baseURL, Model: "deepseek-v4-flash", APIKey: key}) |
| 35 | if err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | policy := provider.ResolveContextBudgetPolicy(chat) |
| 39 | if policy.LimitMode != provider.OutputLimitOmitWhenSafe { |
| 40 | t.Fatalf("live DeepSeek policy = %+v", policy) |
| 41 | } |
| 42 | if closer, ok := chat.(interface{ CloseIdleConnections() }); ok { |
| 43 | t.Cleanup(closer.CloseIdleConnections) |
| 44 | } |
| 45 | stablePrefix := "You are a coding agent. Keep this cache-test prefix byte-identical. " + |
| 46 | strings.Repeat("Preserve the stable prefix and answer the final request concisely. ", 70) |
| 47 | baseReq := provider.Request{ |
| 48 | Messages: []provider.Message{ |
| 49 | {Role: provider.RoleSystem, Content: stablePrefix}, |
| 50 | {Role: provider.RoleUser, Content: "Do not call tools. Reply with the single word ok."}, |
| 51 | }, |
| 52 | Tools: []provider.ToolSchema{{ |
| 53 | Name: "live_cache_marker", Description: "Return a fixed cache marker only when explicitly requested.", |
| 54 | Parameters: []byte(`{"type":"object","properties":{},"additionalProperties":false}`), |
| 55 | }}, |
| 56 | Temperature: provider.TemperaturePtr(0), |
| 57 | } |
| 58 | |
| 59 | collect := func(label string, req provider.Request) liveOutputFieldCacheResult { |
| 60 | t.Helper() |
| 61 | ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) |
| 62 | defer cancel() |
| 63 | chunks, err := chat.Stream(ctx, req) |
| 64 | if err != nil { |
| 65 | t.Fatalf("%s stream: %v", label, err) |
| 66 | } |
| 67 | var result liveOutputFieldCacheResult |
| 68 | for chunk := range chunks { |
| 69 | switch chunk.Type { |
| 70 | case provider.ChunkUsage: |
| 71 | if chunk.Usage != nil { |
| 72 | result.prompt = chunk.Usage.PromptTokens |
| 73 | result.hit = chunk.Usage.CacheHitTokens |
| 74 | result.miss = chunk.Usage.CacheMissTokens |
| 75 | } |
| 76 | case provider.ChunkError: |
| 77 | t.Fatalf("%s chunk: %v", label, chunk.Err) |
| 78 | } |
| 79 | } |
| 80 | return result |
| 81 | } |
| 82 | |
| 83 | _ = collect("warm", baseReq) |
| 84 | time.Sleep(3 * time.Second) |
| 85 | omitted := collect("omitted", baseReq) |
| 86 | limitedReq := baseReq |
| 87 | limitedReq.MaxTokens = 64 |
| 88 | limited := collect("limited", limitedReq) |
| 89 | if omitted.prompt != limited.prompt { |
| 90 | t.Fatalf("output field changed prompt tokens: omitted=%d limited=%d", omitted.prompt, limited.prompt) |
| 91 | } |
| 92 | if omitted.hit+omitted.miss == 0 || limited.hit+limited.miss == 0 { |
| 93 | t.Skipf("provider returned no cache telemetry: omitted=%+v limited=%+v", omitted, limited) |
| 94 | } |
| 95 | if omitted.hit == 0 { |
| 96 | t.Skipf("provider cache did not warm: omitted=%+v", omitted) |
| 97 | } |
| 98 | if limited.hit*100 < omitted.hit*90 { |
| 99 | t.Fatalf("limited cache hit regressed by more than 10%%: omitted=%+v limited=%+v", omitted, limited) |
| 100 | } |
| 101 | t.Logf("output-field A/B: omitted prompt=%d hit=%d miss=%d; limited prompt=%d hit=%d miss=%d", |
| 102 | omitted.prompt, omitted.hit, omitted.miss, limited.prompt, limited.hit, limited.miss) |
| 103 | } |
| 104 |