| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "reasonix/internal/event" |
| 15 | "reasonix/internal/provider" |
| 16 | "reasonix/internal/provider/anthropic" |
| 17 | "reasonix/internal/provider/openai" |
| 18 | "reasonix/internal/sessioncontext" |
| 19 | "reasonix/internal/tool" |
| 20 | ) |
| 21 | |
| 22 | type liveSessionContextProvider struct { |
| 23 | name, keyEnv, kind, baseURL, model string |
| 24 | extra map[string]any |
| 25 | } |
| 26 | |
| 27 | type liveSessionContextResult struct { |
| 28 | prompt, hit, miss int |
| 29 | systemHash string |
| 30 | } |
| 31 | |
| 32 | func TestLiveSessionContextFirstTurnMatrix(t *testing.T) { |
| 33 | if os.Getenv("REASONIX_LIVE_SESSION_CONTEXT_MATRIX") == "" { |
| 34 | t.Skip("set REASONIX_LIVE_SESSION_CONTEXT_MATRIX=1 to run the paid live matrix") |
| 35 | } |
| 36 | providers := []liveSessionContextProvider{ |
| 37 | {name: "deepseek", keyEnv: "DEEPSEEK_API_KEY", kind: "anthropic", baseURL: "https://api.deepseek.com/anthropic", model: "deepseek-v4-flash", extra: map[string]any{"thinking": "disabled", "effort": "disabled"}}, |
| 38 | {name: "longcat", keyEnv: "LONGCAT_API_KEY", kind: "openai", baseURL: "https://api.longcat.chat/openai/v1", model: "LongCat-2.0", extra: map[string]any{"thinking": "disabled"}}, |
| 39 | {name: "zhipu-coding", keyEnv: "ZHIPU_CODING_API_KEY", kind: "openai", baseURL: "https://api.z.ai/api/coding/paas/v4", model: "glm-5.1", extra: map[string]any{"thinking": "disabled"}}, |
| 40 | {name: "opencode-go", keyEnv: "OPENCODE_GO_API_KEY", kind: "openai", baseURL: "https://opencode.ai/zen/go/v1", model: "glm-5.3", extra: map[string]any{"reasoning_protocol": "openai", "effort": "low"}}, |
| 41 | } |
| 42 | stages := []struct { |
| 43 | name string |
| 44 | sections sessioncontext.Sections |
| 45 | }{ |
| 46 | {name: "same-project", sections: liveContextSections("workspace-a", "memory-v1", "alpha — initial skill")}, |
| 47 | {name: "workspace-changed", sections: liveContextSections("workspace-b", "memory-v1", "alpha — initial skill")}, |
| 48 | {name: "memory-changed", sections: liveContextSections("workspace-b", "memory-v2", "alpha — initial skill")}, |
| 49 | {name: "skill-added", sections: liveContextSections("workspace-b", "memory-v2", "alpha — initial skill\nbeta — added skill")}, |
| 50 | {name: "skill-edited", sections: liveContextSections("workspace-b", "memory-v2", "alpha — initial skill\nbeta — edited skill")}, |
| 51 | {name: "skill-deleted", sections: liveContextSections("workspace-b", "memory-v2", "beta — edited skill")}, |
| 52 | } |
| 53 | |
| 54 | for _, providerCase := range providers { |
| 55 | providerCase := providerCase |
| 56 | t.Run(providerCase.name, func(t *testing.T) { |
| 57 | key := strings.TrimSpace(os.Getenv(providerCase.keyEnv)) |
| 58 | if key == "" { |
| 59 | t.Skip(providerCase.keyEnv + " not set") |
| 60 | } |
| 61 | prov := newLiveSessionContextProvider(t, providerCase, key) |
| 62 | if closer, ok := prov.(interface{ CloseIdleConnections() }); ok { |
| 63 | t.Cleanup(closer.CloseIdleConnections) |
| 64 | } |
| 65 | var stableSystemHash string |
| 66 | call := 0 |
| 67 | for _, stage := range stages { |
| 68 | snapshot := sessioncontext.Build(stage.sections) |
| 69 | for repeat := 1; repeat <= 3; repeat++ { |
| 70 | call++ |
| 71 | result := runLiveSessionContextFirstTurn(t, prov, snapshot, call) |
| 72 | if stableSystemHash == "" { |
| 73 | stableSystemHash = result.systemHash |
| 74 | } else if result.systemHash != stableSystemHash { |
| 75 | t.Fatalf("stage=%s repeat=%d changed system hash", stage.name, repeat) |
| 76 | } |
| 77 | t.Logf("provider=%s stage=%s repeat=%d prompt=%d hit=%d miss=%d digest=%s", |
| 78 | providerCase.name, stage.name, repeat, result.prompt, result.hit, result.miss, snapshot.Digest[:12]) |
| 79 | time.Sleep(750 * time.Millisecond) |
| 80 | } |
| 81 | } |
| 82 | }) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func liveContextSections(workspace, memory, skills string) sessioncontext.Sections { |
| 87 | return sessioncontext.Sections{ |
| 88 | Environment: "runtime: live-provider-matrix\noffline: false", |
| 89 | Workspace: "Current workspace: " + workspace, |
| 90 | BackgroundMemory: memory, |
| 91 | SkillsCatalog: skills, |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | func newLiveSessionContextProvider(t *testing.T, cfg liveSessionContextProvider, key string) provider.Provider { |
| 96 | t.Helper() |
| 97 | extra := make(map[string]any, len(cfg.extra)+1) |
| 98 | for name, value := range cfg.extra { |
| 99 | extra[name] = value |
| 100 | } |
| 101 | extra["api_key_env"] = cfg.keyEnv |
| 102 | providerConfig := provider.Config{Name: cfg.name, BaseURL: cfg.baseURL, Model: cfg.model, APIKey: key, Extra: extra} |
| 103 | var ( |
| 104 | prov provider.Provider |
| 105 | err error |
| 106 | ) |
| 107 | if cfg.kind == "anthropic" { |
| 108 | prov, err = anthropic.New(providerConfig) |
| 109 | } else { |
| 110 | prov, err = openai.New(providerConfig) |
| 111 | } |
| 112 | if err != nil { |
| 113 | t.Fatalf("new %s provider: %v", cfg.name, err) |
| 114 | } |
| 115 | return prov |
| 116 | } |
| 117 | |
| 118 | func runLiveSessionContextFirstTurn(t *testing.T, prov provider.Provider, snapshot sessioncontext.Snapshot, call int) liveSessionContextResult { |
| 119 | t.Helper() |
| 120 | system := "You are a concise cache verification assistant. " + |
| 121 | strings.Repeat("Keep this stable policy prefix byte-identical and answer the final request briefly. ", 90) |
| 122 | var usage *provider.Usage |
| 123 | var diagnostics *event.CacheDiagnostics |
| 124 | sink := event.FuncSink(func(e event.Event) { |
| 125 | if e.Kind != event.Usage { |
| 126 | return |
| 127 | } |
| 128 | if e.Usage != nil { |
| 129 | copyUsage := *e.Usage |
| 130 | usage = ©Usage |
| 131 | } |
| 132 | if e.CacheDiagnostics != nil { |
| 133 | copyDiagnostics := *e.CacheDiagnostics |
| 134 | diagnostics = ©Diagnostics |
| 135 | } |
| 136 | }) |
| 137 | session := NewSession(system) |
| 138 | agent := New(prov, tool.NewRegistry(), session, Options{MaxSteps: 2, MaxOutputTokens: 64}, sink) |
| 139 | ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) |
| 140 | defer cancel() |
| 141 | ctx = WithTurnContextBundle(ctx, TurnContextBundle{Executor: snapshot}) |
| 142 | if err := agent.Run(ctx, fmt.Sprintf("Reply with exactly OK. Matrix call %d.", call)); err != nil { |
| 143 | t.Fatalf("live first turn %d: %v", call, err) |
| 144 | } |
| 145 | messages := session.Snapshot() |
| 146 | if len(messages) < 4 || messages[0].Role != provider.RoleSystem || messages[1].Origin != provider.MessageOriginHost || messages[2].Origin != provider.MessageOriginUser { |
| 147 | t.Fatalf("live first turn %d history order = %+v", call, messages) |
| 148 | } |
| 149 | parsed, ok := sessioncontext.Parse(messages[1].Content) |
| 150 | if !ok || parsed.Digest != snapshot.Digest { |
| 151 | t.Fatalf("live first turn %d persisted invalid context", call) |
| 152 | } |
| 153 | if strings.TrimSpace(messages[len(messages)-1].Content) == "" { |
| 154 | t.Fatalf("live first turn %d returned no assistant text", call) |
| 155 | } |
| 156 | if usage == nil || usage.PromptTokens == 0 { |
| 157 | t.Fatalf("live first turn %d returned no usage", call) |
| 158 | } |
| 159 | if diagnostics == nil || diagnostics.SessionContext == nil || diagnostics.SessionContext.Digest != snapshot.Digest || |
| 160 | diagnostics.SessionContext.TargetRole != "executor" || !slices.Contains(diagnostics.SessionContext.Reasons, "first_seen") { |
| 161 | t.Fatalf("live first turn %d diagnostics = %+v", call, diagnostics) |
| 162 | } |
| 163 | return liveSessionContextResult{ |
| 164 | prompt: usage.PromptTokens, hit: usage.CacheHitTokens, miss: usage.CacheMissTokens, |
| 165 | systemHash: diagnostics.SystemHash, |
| 166 | } |
| 167 | } |
| 168 |