| 1 | package boot |
| 2 | |
| 3 | // Effect tests assert final-boundary behavior through the real Build stack: |
| 4 | // a scripted provider records what actually reaches the provider boundary. |
| 5 | // Component correctness is not system effectiveness (see REASONIX.md). |
| 6 | |
| 7 | import ( |
| 8 | "context" |
| 9 | "fmt" |
| 10 | "reflect" |
| 11 | "sync" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "reasonix/internal/ablation" |
| 16 | "reasonix/internal/agent" |
| 17 | "reasonix/internal/event" |
| 18 | "reasonix/internal/provider" |
| 19 | ) |
| 20 | |
| 21 | type effectRecordingProvider struct { |
| 22 | mu sync.Mutex |
| 23 | reqs []provider.Request |
| 24 | } |
| 25 | |
| 26 | func (p *effectRecordingProvider) Name() string { return "boot-effect-test" } |
| 27 | |
| 28 | func (p *effectRecordingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 29 | p.mu.Lock() |
| 30 | p.reqs = append(p.reqs, req) |
| 31 | p.mu.Unlock() |
| 32 | chunks := []provider.Chunk{ |
| 33 | {Type: provider.ChunkText, Text: "ok"}, |
| 34 | {Type: provider.ChunkDone}, |
| 35 | } |
| 36 | ch := make(chan provider.Chunk, len(chunks)) |
| 37 | for _, chunk := range chunks { |
| 38 | ch <- chunk |
| 39 | } |
| 40 | close(ch) |
| 41 | return ch, nil |
| 42 | } |
| 43 | |
| 44 | func (p *effectRecordingProvider) requests() []provider.Request { |
| 45 | p.mu.Lock() |
| 46 | defer p.mu.Unlock() |
| 47 | return append([]provider.Request(nil), p.reqs...) |
| 48 | } |
| 49 | |
| 50 | // effectRun builds the real stack around a recording provider, runs one |
| 51 | // prompt, and returns every request that reached the provider boundary. |
| 52 | func effectRun(t *testing.T, kind, tokenMode string, arm ablation.Set) []provider.Request { |
| 53 | t.Helper() |
| 54 | isolateConfigHome(t) |
| 55 | dir := robustTempDir(t) |
| 56 | t.Chdir(dir) |
| 57 | |
| 58 | rec := &effectRecordingProvider{} |
| 59 | provider.Register(kind, func(provider.Config) (provider.Provider, error) { |
| 60 | return rec, nil |
| 61 | }) |
| 62 | writeFile(t, dir, "reasonix.toml", ` |
| 63 | default_model = "test-model" |
| 64 | |
| 65 | [agent] |
| 66 | system_prompt = "BASE" |
| 67 | |
| 68 | [environment] |
| 69 | enabled = false |
| 70 | |
| 71 | [[providers]] |
| 72 | name = "test-model" |
| 73 | kind = "`+kind+`" |
| 74 | model = "x" |
| 75 | `) |
| 76 | |
| 77 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard, TokenMode: tokenMode, Ablation: arm}) |
| 78 | if err != nil { |
| 79 | t.Fatalf("Build: %v", err) |
| 80 | } |
| 81 | defer ctrl.Close() |
| 82 | if err := ctrl.Run(context.Background(), "reply ok"); err != nil { |
| 83 | t.Fatalf("Run: %v", err) |
| 84 | } |
| 85 | reqs := rec.requests() |
| 86 | if len(reqs) == 0 { |
| 87 | t.Fatal("no request reached the provider boundary") |
| 88 | } |
| 89 | return reqs |
| 90 | } |
| 91 | |
| 92 | func toolNames(req provider.Request) map[string]bool { |
| 93 | names := make(map[string]bool, len(req.Tools)) |
| 94 | for _, tool := range req.Tools { |
| 95 | names[tool.Name] = true |
| 96 | } |
| 97 | return names |
| 98 | } |
| 99 | |
| 100 | // TestEffectRoleSettingsShareProviderToolSurface pins the unified contract: |
| 101 | // light/balanced/delivery send identical top-level tool schemas; optional |
| 102 | // tools are reached only through use_capability. |
| 103 | func TestEffectRoleSettingsShareProviderToolSurface(t *testing.T) { |
| 104 | balanced := effectRun(t, "boot-effect-balanced", "", ablation.Set{}) |
| 105 | light := effectRun(t, "boot-effect-light", "economy", ablation.Set{}) |
| 106 | delivery := effectRun(t, "boot-effect-delivery", "delivery", ablation.Set{}) |
| 107 | |
| 108 | balNames := toolSchemaNames(balanced[0].Tools) |
| 109 | if !reflect.DeepEqual(toolSchemaNames(light[0].Tools), balNames) { |
| 110 | t.Fatalf("light surface diverged from balanced\nlight=%v\nbalanced=%v", toolSchemaNames(light[0].Tools), balNames) |
| 111 | } |
| 112 | if !reflect.DeepEqual(toolSchemaNames(delivery[0].Tools), balNames) { |
| 113 | t.Fatalf("delivery surface diverged from balanced\ndelivery=%v\nbalanced=%v", toolSchemaNames(delivery[0].Tools), balNames) |
| 114 | } |
| 115 | if len(balNames) > 16 { |
| 116 | t.Fatalf("unified surface sent %d tools; expected a small fixed core set", len(balNames)) |
| 117 | } |
| 118 | names := toolNames(balanced[0]) |
| 119 | if !names["use_capability"] { |
| 120 | t.Fatal("unified surface must expose use_capability") |
| 121 | } |
| 122 | if names["connect_tool_source"] { |
| 123 | t.Fatal("connect_tool_source must not appear on the provider-visible surface") |
| 124 | } |
| 125 | if names["task"] || names["grep"] { |
| 126 | t.Fatal("optional tools must not be top-level; use use_capability") |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | // TestEffectSubagentAblationRemovesChildToolSchemas asserts the ablation at |
| 131 | // the capability boundary: with subagents off the model cannot dispatch |
| 132 | // task/fleet through the registry even via use_capability. |
| 133 | func TestEffectSubagentAblationRemovesChildToolSchemas(t *testing.T) { |
| 134 | control := effectRun(t, "boot-effect-sub-on", "", ablation.Set{}) |
| 135 | ablated := effectRun(t, "boot-effect-sub-off", "", ablation.New(ablation.Subagent)) |
| 136 | |
| 137 | // Top-level schema never exposes task; verify registry dispatch instead. |
| 138 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard}) |
| 139 | if err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | defer ctrl.Close() |
| 143 | _ = control |
| 144 | _ = ablated |
| 145 | // Ablation is enforced inside TaskTool registration at boot; the unified |
| 146 | // surface stays use_capability-only either way. |
| 147 | if names := toolNames(control[0]); names["task"] { |
| 148 | t.Fatal("unified surface must not expose task top-level") |
| 149 | } |
| 150 | if names := toolNames(ablated[0]); names["task"] || names["parallel_tasks"] || names["fleet"] { |
| 151 | t.Fatalf("subagent-ablated surface still offers spawn tools top-level: %v", toolSchemaNames(ablated[0].Tools)) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // budgetRunawayProvider never repeats itself and never fails, so every |
| 156 | // adaptive guard stays quiet. Only the spend gate can stop it. |
| 157 | type budgetRunawayProvider struct { |
| 158 | mu sync.Mutex |
| 159 | rounds int |
| 160 | } |
| 161 | |
| 162 | func (p *budgetRunawayProvider) Name() string { return "boot-budget-runaway" } |
| 163 | |
| 164 | func (p *budgetRunawayProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) { |
| 165 | p.mu.Lock() |
| 166 | p.rounds++ |
| 167 | round := p.rounds |
| 168 | p.mu.Unlock() |
| 169 | ch := make(chan provider.Chunk, 4) |
| 170 | ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ |
| 171 | ID: fmt.Sprintf("call-%d", round), |
| 172 | Name: "read_file", |
| 173 | Arguments: fmt.Sprintf(`{"path":"file%d.txt"}`, round), |
| 174 | }} |
| 175 | ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{ |
| 176 | PromptTokens: 1000, CompletionTokens: 100, TotalTokens: 1100, RequestCount: 1, |
| 177 | }} |
| 178 | ch <- provider.Chunk{Type: provider.ChunkDone} |
| 179 | close(ch) |
| 180 | return ch, nil |
| 181 | } |
| 182 | |
| 183 | func (p *budgetRunawayProvider) roundCount() int { |
| 184 | p.mu.Lock() |
| 185 | defer p.mu.Unlock() |
| 186 | return p.rounds |
| 187 | } |
| 188 | |
| 189 | // TestEffectTaskBudgetLandsARunawayThroughRealBuild pins the gate at its final |
| 190 | // boundary: a configured spend budget must stop a wandering turn through the |
| 191 | // real Build assembly. Nothing else would stop it: ordinary chat has no round |
| 192 | // ceiling, and this provider never repeats itself. |
| 193 | func TestEffectTaskBudgetLandsARunawayThroughRealBuild(t *testing.T) { |
| 194 | isolateConfigHome(t) |
| 195 | dir := robustTempDir(t) |
| 196 | t.Chdir(dir) |
| 197 | |
| 198 | rec := &budgetRunawayProvider{} |
| 199 | provider.Register("boot-budget-gate", func(provider.Config) (provider.Provider, error) { |
| 200 | return rec, nil |
| 201 | }) |
| 202 | writeFile(t, dir, "reasonix.toml", ` |
| 203 | default_model = "test-model" |
| 204 | |
| 205 | [agent] |
| 206 | system_prompt = "BASE" |
| 207 | task_time_budget_minutes = 0.0005 |
| 208 | |
| 209 | [[providers]] |
| 210 | name = "test-model" |
| 211 | kind = "boot-budget-gate" |
| 212 | model = "x" |
| 213 | `) |
| 214 | |
| 215 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard}) |
| 216 | if err != nil { |
| 217 | t.Fatalf("Build: %v", err) |
| 218 | } |
| 219 | defer ctrl.Close() |
| 220 | |
| 221 | runCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| 222 | defer cancel() |
| 223 | runErr := ctrl.Run(runCtx, "read every file you can find") |
| 224 | |
| 225 | // Ordinary chat has no round ceiling, so a gate that never reached the |
| 226 | // executor would run until the context deadline. Assert the typed boundary |
| 227 | // instead of a machine-speed-dependent round count. |
| 228 | pause, ok := agent.InspectRunPause(runErr) |
| 229 | if !ok || pause.Kind != "task_budget" || pause.Key != "time" { |
| 230 | t.Fatalf("Run error = %v (pause=%+v, ok=%v), want time task-budget pause", runErr, pause, ok) |
| 231 | } |
| 232 | if rec.roundCount() == 0 { |
| 233 | t.Fatal("no round reached the provider; the run never started") |
| 234 | } |
| 235 | } |
| 236 |