返回 DeepSeek-Reasonix
compact_summary_limit_test.go
根目录 / internal / agent / compact_summary_limit_test.go
1 package agent
2
3 import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "strings"
8 "sync"
9 "testing"
10
11 "reasonix/internal/event"
12 "reasonix/internal/provider"
13 "reasonix/internal/tool"
14 )
15
16 const deepSeekOverflowBody = `{"error":{"message":"This model's maximum context length is %d tokens. However, you requested %d tokens (%d in the messages, %d in the completion). Please reduce the length of the messages or completion.","type":"invalid_request_error","param":null,"code":"invalid_request_error"}}`
17
18 // denseTokenizerProvider counts three characters per token where the agent's
19 // cold estimate assumes four, so a fold the estimator believes fits overflows
20 // on the wire exactly as #9818 reported. Its overflow reply is the parsed
21 // DeepSeek 400 body, so the feedback path sees what production sees.
22 // alwaysOverflow reports every prompt as at least the window, so each reply
23 // still justifies its rejection while no summary form can ever land.
24 type denseTokenizerProvider struct {
25 mu sync.Mutex
26 window int
27 alwaysOverflow bool
28 // unnumberedReplay rejects replay-form summaries with a bare overflow that
29 // carries no token numbers, the shape GLM reports (#9878).
30 unnumberedReplay bool
31 requests []provider.Request
32 overflows int
33 }
34
35 func (p *denseTokenizerProvider) Name() string { return "dense-tokenizer" }
36
37 func (p *denseTokenizerProvider) ContextBudgetPolicy() provider.ContextBudgetPolicy {
38 return provider.ContextBudgetPolicy{
39 WindowMode: provider.ContextWindowShared, AutoOutputTokens: 8192, MaxOutputTokens: 8192,
40 LimitMode: provider.OutputLimitOmitWhenSafe,
41 }
42 }
43
44 func denseTokens(req provider.Request) int {
45 chars, _, _ := requestCalibrationTextShape(req, provider.SharedWindowInputPolicy{})
46 return int(chars) / 3
47 }
48
49 func isSummaryRequest(req provider.Request) bool {
50 return len(req.Messages) > 0 && strings.Contains(req.Messages[len(req.Messages)-1].Content, "Compact the preceding conversation prefix")
51 }
52
53 func (p *denseTokenizerProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
54 p.mu.Lock()
55 defer p.mu.Unlock()
56 req.Messages = append([]provider.Message(nil), req.Messages...)
57 p.requests = append(p.requests, req)
58 if p.unnumberedReplay && isSummaryRequest(req) && req.Messages[0].Content != slimSummarySystemPrompt {
59 p.overflows++
60 return nil, &provider.ContextLimitError{APIError: &provider.APIError{
61 Provider: p.Name(), Status: 400, Body: `{"error":{"code":"1261","message":"Prompt exceeds max length"}}`,
62 }}
63 }
64 prompt := denseTokens(req)
65 if p.unnumberedReplay {
66 prompt = prompt * 3 / 4 // an ordinary tokenizer: only the replay form was rejected
67 }
68 if p.alwaysOverflow {
69 prompt = max(prompt, p.window)
70 }
71 completion := req.MaxTokens
72 if completion <= 0 {
73 completion = 8192
74 }
75 if prompt+completion > p.window {
76 p.overflows++
77 body := fmt.Sprintf(deepSeekOverflowBody, p.window, prompt+completion, prompt, completion)
78 limit := provider.ParseContextLimitError(&provider.APIError{Provider: p.Name(), Status: 400, Body: body})
79 if limit == nil {
80 return nil, fmt.Errorf("test body did not parse as a context limit: %s", body)
81 }
82 return nil, limit
83 }
84 text := "ok"
85 if isSummaryRequest(req) {
86 text = "- goal: keep going\n- pending: continue"
87 }
88 return chunks(
89 provider.Chunk{Type: provider.ChunkText, Text: text},
90 provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: prompt, CompletionTokens: 8, TotalTokens: prompt + 8, RequestCount: 1}},
91 provider.Chunk{Type: provider.ChunkDone},
92 ), nil
93 }
94
95 func (p *denseTokenizerProvider) summaryRequests() []provider.Request {
96 p.mu.Lock()
97 defer p.mu.Unlock()
98 var out []provider.Request
99 for _, req := range p.requests {
100 if isSummaryRequest(req) {
101 out = append(out, req)
102 }
103 }
104 return out
105 }
106
107 func requestFingerprints(reqs []provider.Request) map[string]int {
108 seen := map[string]int{}
109 for _, req := range reqs {
110 seen[providerVisibleFingerprint(req.Messages)]++
111 }
112 return seen
113 }
114
115 func longASCIISession(turns int) *Session {
116 big := strings.Repeat("alpha beta gamma delta ", 200)
117 sess := NewSession("sys")
118 sess.Add(provider.Message{Role: provider.RoleUser, Content: "standing constraint: keep the public API stable"})
119 for i := range turns {
120 sess.Add(provider.Message{Role: provider.RoleAssistant, Content: fmt.Sprintf("step %d: %s", i, big)})
121 sess.Add(provider.Message{Role: provider.RoleUser, Content: "continue"})
122 }
123 return sess
124 }
125
126 // The estimator plans the largest prefix it believes fits; the provider counts
127 // denser and rejects it. The overflow must recalibrate the estimator and the
128 // re-planned request must be strictly smaller, landing a real digest without
129 // the fragment path and without ever repeating the rejected request.
130 func TestSummaryOverflowRecalibratesAndReplansSmaller(t *testing.T) {
131 prov := &denseTokenizerProvider{window: 20_000}
132 sess := longASCIISession(16)
133 a := New(prov, tool.NewRegistry(), sess, Options{ContextWindow: 20_000, CompactRatio: 0.8}, event.Discard)
134 if est, fold, hard := a.ContextUsedTokens(), a.compactTrigger(), a.hardInputCeiling(); est < fold || est >= hard {
135 t.Fatalf("fixture estimates %d tokens; want between the trigger %d and the ceiling %d", est, fold, hard)
136 }
137
138 if err := prepareContext(context.Background(), a, CompactionTriggerPressure); err != nil {
139 t.Fatalf("prepare = %v", err)
140 }
141 summaries := prov.summaryRequests()
142 if prov.overflows != 1 || len(summaries) != 2 {
143 t.Fatalf("overflows=%d summaries=%d, want one rejected replay and one re-planned success", prov.overflows, len(summaries))
144 }
145 if first, second := denseTokens(summaries[0]), denseTokens(summaries[1]); second >= first {
146 t.Fatalf("re-planned summary request %d tokens is not smaller than the rejected %d", second, first)
147 }
148 for fp, n := range requestFingerprints(summaries) {
149 if n > 1 {
150 t.Fatalf("summary request %s was sent %d times", fp, n)
151 }
152 }
153 if ratio := a.tokPerChar(); ratio < 0.3 {
154 t.Fatalf("calibration ratio %.3f did not learn the provider's denser tokenizer", ratio)
155 }
156 r := a.sess.compactionState.LastReceipt
157 if r == nil || r.Status != "applied" || r.Action != "summary" || latestDigest(a.sess.compactionState.Projection.Messages) == "" {
158 t.Fatalf("receipt = %+v, want an applied summary with a digest", r)
159 }
160 }
161
162 // An overflow reply without token numbers cannot recalibrate anything, so a
163 // re-plan would resend the same bytes. The ladder must skip straight to the
164 // transcript form and must not learn a ratio or window from zero fields.
165 func TestUnnumberedSummaryOverflowSkipsReplanToTranscript(t *testing.T) {
166 prov := &denseTokenizerProvider{window: 20_000, unnumberedReplay: true}
167 reg := tool.NewRegistry()
168 reg.Add(schemaTool{})
169 sess := longASCIISession(16)
170 a := New(prov, reg, sess, Options{ContextWindow: 20_000, CompactRatio: 0.8}, event.Discard)
171
172 if err := prepareContext(context.Background(), a, CompactionTriggerPressure); err != nil {
173 t.Fatalf("prepare = %v", err)
174 }
175 summaries := prov.summaryRequests()
176 if prov.overflows != 1 || len(summaries) != 2 {
177 t.Fatalf("overflows=%d summaries=%d, want one rejected replay and one transcript-form success", prov.overflows, len(summaries))
178 }
179 if slim := summaries[1]; len(slim.Tools) != 0 || len(slim.Messages) != 2 {
180 t.Fatalf("second request = %d tools, %d messages; want the transcript form, not a re-planned replay", len(slim.Tools), len(slim.Messages))
181 }
182 if ratio := a.tokPerChar(); ratio != fallbackTokPerChar {
183 t.Fatalf("calibration ratio %.3f changed on an overflow without token numbers", ratio)
184 }
185 if window := a.effectiveContextWindow(); window != 20_000 {
186 t.Fatalf("effective window %d changed on an overflow without token numbers", window)
187 }
188 r := a.sess.compactionState.LastReceipt
189 if r == nil || r.Status != "applied" || r.Action != "summary" || latestDigest(a.sess.compactionState.Projection.Messages) == "" {
190 t.Fatalf("receipt = %+v, want an applied summary with a digest", r)
191 }
192 }
193
194 // A provider that rejects every summary form must not trap /compact in a loop
195 // of identical requests: replay re-plans, then the transcript form, then the
196 // fragment path, and at the ceiling the truncation rescue finally lands.
197 func TestManualCompactOverCeilingRescuesWithoutRepeatingRequests(t *testing.T) {
198 prov := &denseTokenizerProvider{window: 20_000, alwaysOverflow: true}
199 sess := longASCIISession(30)
200 a := New(prov, tool.NewRegistry(), sess, Options{ContextWindow: 20_000, CompactRatio: 0.8}, event.Discard)
201 if est, hard := a.ContextUsedTokens(), a.hardInputCeiling(); est < hard {
202 t.Fatalf("fixture estimates %d tokens against a %d ceiling; it is not over it", est, hard)
203 }
204
205 if err := a.CompactNow(context.Background(), ""); err != nil {
206 t.Fatalf("CompactNow = %v, want the truncation rescue", err)
207 }
208 if !truncatedRescue(a) {
209 t.Fatalf("receipt = %+v, want an applied truncation without a digest", a.sess.compactionState.LastReceipt)
210 }
211 if after, hard := a.ContextUsedTokens(), a.hardInputCeiling(); after >= hard {
212 t.Fatalf("rescued view estimates %d tokens against a %d ceiling", after, hard)
213 }
214 summaries := prov.summaryRequests()
215 if len(summaries) < 3 {
216 t.Fatalf("summary requests = %d, want replay re-plans and the transcript form before the rescue", len(summaries))
217 }
218 for fp, n := range requestFingerprints(summaries) {
219 if n > 1 {
220 t.Fatalf("summary request %s was sent %d times", fp, n)
221 }
222 }
223 slim := 0
224 for _, req := range summaries {
225 if len(req.Tools) == 0 && len(req.Messages) == 2 {
226 slim++
227 }
228 }
229 if slim != 1 {
230 t.Fatalf("transcript-form summary requests = %d, want exactly one rung", slim)
231 }
232 }
233
234 type schemaTool struct{}
235
236 func (schemaTool) Name() string { return "read_file" }
237 func (schemaTool) Description() string { return "Read a file." }
238 func (schemaTool) Schema() json.RawMessage {
239 return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string"}}}`)
240 }
241 func (schemaTool) ReadOnly() bool { return true }
242 func (schemaTool) Execute(context.Context, json.RawMessage) (string, error) {
243 return "", nil
244 }
245
246 func TestSlimSummaryRequestIsBoundedAndToolFree(t *testing.T) {
247 reg := tool.NewRegistry()
248 reg.Add(schemaTool{})
249 a := New(&denseTokenizerProvider{window: 1 << 20}, reg, NewSession("sys"), Options{ContextWindow: 1 << 20}, event.Discard)
250 body := strings.Repeat("0123456789", 3000)
251 fold := []provider.Message{
252 {Role: provider.RoleUser, Content: "read it"},
253 {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: "c1", Name: "read_file", Arguments: `{"path":"x"}`}}},
254 {Role: provider.RoleTool, ToolCallID: "c1", Name: "read_file", Content: body, Images: []string{"data:image/png;base64,AAAA"}},
255 }
256
257 replay := a.summaryRequest(fold, "")
258 if len(replay.Tools) == 0 {
259 t.Fatal("replay form must carry the tool schemas the sampling request uses")
260 }
261 slim := a.slimSummaryRequest(fold, "")
262 if len(slim.Tools) != 0 || len(slim.Messages) != 2 {
263 t.Fatalf("slim form = %d tools, %d messages; want no schemas and one transcript turn", len(slim.Tools), len(slim.Messages))
264 }
265 text := slim.Messages[1].Content
266 if !strings.Contains(text, "tool result truncated for summarization") || strings.Contains(text, "base64") {
267 t.Fatal("slim transcript must cut the tool body and drop images")
268 }
269 if len(text) > slimToolResultRunes+2000 {
270 t.Fatalf("slim transcript is %d bytes; the tool body should be bounded by %d runes", len(text), slimToolResultRunes)
271 }
272 if got, want := a.estimatedRequestTokens(slim), a.estimatedRequestTokens(replay); got >= want {
273 t.Fatalf("slim request estimates %d tokens, not smaller than the replay's %d", got, want)
274 }
275 }
276
277 func TestChunkedFallbackAppliesOnlyAfterTranscriptForm(t *testing.T) {
278 overflow := &provider.ContextLimitError{WindowTokens: 10, PromptTokens: 20}
279 if chunkedFallbackApplies(overflow, SummaryInputCachePrefix) {
280 t.Fatal("a replay overflow should be re-planned, not fragmented")
281 }
282 if !chunkedFallbackApplies(overflow, SummaryInputSlim) {
283 t.Fatal("an overflow of the transcript form has no cheaper rung left")
284 }
285 if !chunkedFallbackApplies(errSummaryOutputTruncated, SummaryInputCachePrefix) || !chunkedFallbackApplies(ErrCompactionRequired, SummaryInputCachePrefix) {
286 t.Fatal("output truncation and local admission keep their direct fragment path")
287 }
288 }
289
290 func TestActiveTurnFoldBoundaryKeepsNewestRounds(t *testing.T) {
291 round := func(i int) []provider.Message {
292 id := fmt.Sprintf("c%d", i)
293 return []provider.Message{
294 {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: "read_file", Arguments: "{}"}}},
295 {Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: "body"},
296 }
297 }
298 msgs := []provider.Message{{Role: provider.RoleSystem, Content: "sys"}, {Role: provider.RoleUser, Content: "task", CreatedAt: 7}}
299 for i := range 4 {
300 msgs = append(msgs, round(i)...)
301 }
302 // Rounds occupy [2,4) [4,6) [6,8) [8,10); the newest two stay verbatim.
303 if got := activeTurnFoldBoundary(msgs, 1, len(msgs)); got != 6 {
304 t.Fatalf("boundary = %d, want 6 (fold prompt + two oldest rounds)", got)
305 }
306 short := msgs[:6]
307 if got := activeTurnFoldBoundary(short, 1, len(short)); got != 1 {
308 t.Fatalf("boundary = %d, want the turn kept whole when it has only the rounds to keep", got)
309 }
310 if got := activeTurnFoldBoundary(msgs, 1, 7); got != 4 {
311 t.Fatalf("boundary = %d, want 4 when the fold end cuts the newest rounds off", got)
312 }
313 }
314
315 func toolLoopSession(rounds int) *Session {
316 sess := NewSession("sys")
317 sess.Add(provider.Message{Role: provider.RoleUser, Content: "read everything"})
318 body := strings.Repeat("tool output line\n", 120)
319 for i := range rounds {
320 id := fmt.Sprintf("c%d", i)
321 sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: "read_file", Arguments: "{}"}}})
322 sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: body})
323 }
324 sess.Add(provider.Message{Role: provider.RoleUser, Content: "now summarize"})
325 sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "tail"})
326 return sess
327 }
328
329 func TestTruncateViewElidesOldestToolResultsBeforeDropping(t *testing.T) {
330 sess := toolLoopSession(10)
331 a := New(&denseTokenizerProvider{window: 1 << 20}, tool.NewRegistry(), sess, Options{ContextWindow: 10_000, CompactRatio: 0.5}, event.Discard)
332 visible := sess.Snapshot()
333 total := a.estimatedVisibleRequestTokens(visible)
334 if total < 3000 {
335 t.Fatalf("fixture estimates only %d tokens", total)
336 }
337
338 projected, affected := a.truncateView(visible, total*2/3)
339 if affected == 0 || len(projected) != len(visible) {
340 t.Fatalf("elision changed %d messages and %d->%d length; want in-place elision only", affected, len(visible), len(projected))
341 }
342 if !strings.HasPrefix(projected[3].Content, elidedToolResultPrefix) {
343 t.Fatalf("oldest tool result was not elided: %q", projected[3].Content[:40])
344 }
345 newest := len(visible) - 3
346 if strings.HasPrefix(projected[newest].Content, elidedToolResultPrefix) {
347 t.Fatal("the protected tail's tool result must stay verbatim")
348 }
349 if after := a.estimatedVisibleRequestTokens(projected); after >= total*2/3 {
350 t.Fatalf("elision left %d tokens, want under the %d target", after, total*2/3)
351 }
352
353 }
354
355 // Text-only history gives elision nothing to cut, so the drop stage must
356 // remove the oldest replay units behind a marker while an earlier digest and
357 // the protected tail survive.
358 func TestTruncateViewDropsOldestUnitsWhenElisionCannotReach(t *testing.T) {
359 sess := longASCIISession(6)
360 a := New(&denseTokenizerProvider{window: 1 << 20}, tool.NewRegistry(), sess, Options{ContextWindow: 10_000, CompactRatio: 0.5}, event.Discard)
361 visible := sess.Snapshot()
362 digest := formatSummaryMessage("- earlier digest")
363 withDigest := append([]provider.Message{visible[0], visible[1], digest}, visible[2:]...)
364 total := a.estimatedVisibleRequestTokens(withDigest)
365
366 dropped, affected := a.truncateView(withDigest, total/3)
367 if affected == 0 || len(dropped) >= len(withDigest) {
368 t.Fatalf("drop stage changed %d messages and %d->%d length; want oldest units removed", affected, len(withDigest), len(dropped))
369 }
370 if !strings.Contains(dropped[1].Content, "truncated to fit the context window") {
371 t.Fatalf("drop stage left no marker: %q", dropped[1].Content)
372 }
373 if latestDigest(dropped) == "" {
374 t.Fatal("an earlier compaction digest must survive truncation")
375 }
376 if last := dropped[len(dropped)-1]; last.Content != visible[len(visible)-1].Content {
377 t.Fatal("the protected tail must stay verbatim")
378 }
379 if after := a.estimatedVisibleRequestTokens(dropped); after >= total/3 {
380 t.Fatalf("drop stage left %d tokens, want under the %d target", after, total/3)
381 }
382 }
383
384 func TestFailedReceiptLiftsWhenViewOutgrowsFailure(t *testing.T) {
385 const window = 10_000
386 sess := &Session{Messages: []provider.Message{
387 {Role: provider.RoleSystem, Content: "system"},
388 {Role: provider.RoleUser, Content: "task"},
389 {Role: provider.RoleAssistant, Content: strings.Repeat("old work ", 500)},
390 {Role: provider.RoleUser, Content: "current"},
391 {Role: provider.RoleAssistant, Content: "tail"},
392 }}
393 prov := &failingSummaryProvider{}
394 a := New(prov, tool.NewRegistry(), sess, Options{ContextWindow: window, CompactRatio: 0.85, RecentKeep: 2}, event.Discard)
395 policy := ContextPreparePolicy{Trigger: CompactionTriggerPressure, ObservedInputTokens: 8600}
396 a.activeTurnCreatedAt.Store(11)
397
398 if _, err := a.contextManager().Prepare(context.Background(), policy); err != nil {
399 t.Fatal(err)
400 }
401 sess.Add(provider.Message{Role: provider.RoleTool, Content: strings.Repeat("small output ", 40)})
402 if _, err := a.contextManager().Prepare(context.Background(), policy); err != nil {
403 t.Fatal(err)
404 }
405 if prov.calls != 1 {
406 t.Fatalf("a small same-turn change made %d summary calls, want the backoff to hold", prov.calls)
407 }
408 sess.Add(provider.Message{Role: provider.RoleTool, Content: strings.Repeat("large output ", 400)})
409 if _, err := a.contextManager().Prepare(context.Background(), policy); err != nil {
410 t.Fatal(err)
411 }
412 if prov.calls != 2 {
413 t.Fatalf("a view grown by over 5%% of the window made %d summary calls, want the backoff lifted", prov.calls)
414 }
415 }
416
416 lines GO