返回 DeepSeek-Reasonix
compact.go
根目录 / internal / agent / compact.go
1 package agent
2
3 import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "sort"
9 "strings"
10 "unicode/utf8"
11
12 "reasonix/internal/ablation"
13 "reasonix/internal/event"
14 "reasonix/internal/provider"
15 )
16
17 // Compaction is a low-frequency cache-reset point: the prompt grows append-only
18 // until compactRatio of the window is crossed, then pressure-time tool pruning
19 // and up to two cache-aligned summary checkpoints restore headroom.
20 const (
21 defaultCompactRatio = 0.80 // sole automatic maintenance trigger (new configs)
22 recentTailBudgetRatio = 0.16 // recent verbatim tail as a fraction of the window
23 summaryOutputMaxTokens = 8192 // max digest output; further clipped by remaining candidate space
24
25 // summaryReasoningMaxBytes clamps a surfaced reasoning-only summary
26 // (~8k tokens of bytes), matching the summaryOutputMaxTokens envelope.
27 summaryReasoningMaxBytes = 32768
28
29 minRecentKeep = 2 // never keep fewer recent messages than this
30 minCompactMessages = 2 // skip compaction below this many compactable messages
31 fallbackTokPerChar = 0.25 // ~4 chars/token, used before any usage is available to calibrate
32 protocolReserveTokens = 256 // provider framing and control fields not represented by message estimates
33 )
34
35 var (
36 errSummaryOutputTruncated = errors.New("summarizer output truncated")
37 errCheckpointRejected = errors.New("checkpoint candidate rejected")
38 )
39
40 // summaryTag wraps the compaction summary so the model can distinguish it from
41 // live user input and later strip or skip it when reasoning about the current turn.
42 const (
43 summaryTagOpen = "<compaction-summary>"
44 summaryTagClose = "</compaction-summary>"
45 )
46
47 // compactionInstruction is appended as the only new message after an otherwise
48 // byte-stable sampling prefix. This lets providers reuse the ordinary request's
49 // system, tools and message-prefix KV cache.
50 const compactionInstruction = `Compact the preceding conversation prefix into a durable resume briefing.
51 Write under these exact headings, omitting a heading only if it has no content:
52
53 ## Standing facts & constraints
54 Everything the user stated that still governs the work — names, paths, IDs, versions, tokens, preferences, and hard "never do X" rules — in their own words. Be exhaustive; this is the durable contract, so prefer over- to under-including.
55
56 ## Goal
57 The user's request and intent.
58
59 ## Decisions & rationale
60 Key choices made so far and why — so they are not re-litigated or reversed.
61
62 ## Files & code
63 Files read or modified, with the specific facts that matter: signatures, line locations, data shapes, and exact edits applied. Be concrete; this is what lets the agent act without re-reading everything.
64
65 ## Commands & outcomes
66 Commands run (builds, tests, git) and their relevant results — what passed, what failed, and the error text that matters.
67
68 ## Errors & fixes
69 Problems hit and how they were resolved (or not), so the same dead ends are not repeated.
70
71 ## Pending & next step
72 What is still in progress or unstarted, and the single most concrete next action to take.
73
74 Rules: be terse — bullet points and fragments, not prose. Preserve identifiers, paths, and numbers exactly. Merge valid facts from any existing <compaction-summary> and remove facts superseded by later messages. Do NOT invent anything not present in the messages; if something is unknown, leave it out rather than guessing. Output only the structured Markdown briefing. Do not call tools. Do not output reasoning.`
75
76 // compactTrigger is the sole automatic context-maintenance boundary. Output
77 // budgets are intentionally absent: they are clipped against the final request
78 // at send time and must never make compaction happen earlier than the user's
79 // configured compact_ratio.
80 func (a *Agent) compact(ctx context.Context, trigger, instructions string, force bool) error {
81 _, err := a.compactToProjectionWithChunked(ctx, trigger, instructions, foldRequest{
82 force: force, allowChunked: trigger == CompactionTriggerManual,
83 })
84 return err
85 }
86
87 func (a *Agent) compactToProjection(ctx context.Context, trigger, instructions string, force, mustFree bool) (CompactionOutcome, error) {
88 return a.compactToProjectionWithChunked(ctx, trigger, instructions, foldRequest{force: force, mustFree: mustFree})
89 }
90
91 func (a *Agent) compactToProjectionWithChunked(ctx context.Context, trigger, instructions string, req foldRequest) (CompactionOutcome, error) {
92 a.sess.compactionRunMu.Lock()
93 defer a.sess.compactionRunMu.Unlock()
94 return a.compactToProjectionLocked(ctx, trigger, instructions, req)
95 }
96
97 func (a *Agent) compactTrigger() int {
98 window := a.effectiveContextWindow()
99 if a == nil || window <= 0 {
100 return 0
101 }
102 ratio := a.compactRatio
103 if ratio <= 0 {
104 ratio = defaultCompactRatio
105 }
106 if a.ablation.Off(ablation.Compaction) {
107 ratio = 0.5
108 }
109 return max(1, int(float64(window)*ratio))
110 }
111
112 // hardInputCeiling is a physical input-safety boundary, not another user
113 // compaction threshold. Reply budgets are resolved independently at send time.
114 func (a *Agent) hardInputCeiling() int {
115 window := a.effectiveContextWindow()
116 if a == nil || window <= 0 {
117 return 0
118 }
119 return max(1, window-protocolReserveTokens)
120 }
121
122 // recentTailBudget is the content-construction budget for the recent verbatim
123 // tail. Harness-style compaction always retains 16% of the model window.
124 func (a *Agent) recentTailBudget() int {
125 window := a.effectiveContextWindow()
126 if a == nil || window <= 0 {
127 return 1
128 }
129 return max(1, int(float64(window)*recentTailBudgetRatio))
130 }
131
132 // foldEconomics estimates whether compacting the given region saves enough
133 // tokens to justify the summarization API call. It returns false when the
134 // region is too small for the savings to outweigh the extra round-trip cost
135 // and latency of calling the summarizer.
136 func foldEconomics(region []provider.Message) bool {
137 const minFoldTokens = 400
138 return estimateMessagesTokens(region) >= minFoldTokens
139 }
140
141 func estimateMessagesTokens(msgs []provider.Message) int {
142 total := 0
143 for _, m := range msgs {
144 if m.LocalOnly || IsPinnedContextRevision(m) {
145 continue
146 }
147 total += 4 // chat-message framing overhead
148 total += estimateTextTokens(m.Content)
149 total += estimateTextTokens(m.ReasoningContent)
150 total += estimateTextTokens(m.Name)
151 total += estimateTextTokens(m.ToolCallID)
152 for _, tc := range m.ToolCalls {
153 total += 8
154 total += estimateTextTokens(tc.ID)
155 total += estimateTextTokens(tc.Name)
156 total += estimateTextTokens(tc.Arguments)
157 }
158 for _, item := range m.ResponsesItems {
159 total += estimateTextTokens(string(item))
160 }
161 for _, search := range m.ServerSearch {
162 provider.WalkServerSearchEstimate(search, func(s string) {
163 total += estimateTextTokens(s)
164 })
165 }
166 }
167 return total
168 }
169
170 func estimateTextTokens(s string) int {
171 if s == "" {
172 return 0
173 }
174 // A conservative cross-language approximation: English-ish text trends near
175 // four bytes per token, while CJK-heavy text is closer to one rune per token.
176 bytes := len(s)
177 runes := utf8.RuneCountInString(s)
178 byBytes := (bytes + 3) / 4
179 if runes > byBytes {
180 return runes
181 }
182 return byBytes
183 }
184
185 // SummarizeFrom keeps the compatibility index contract while installing a
186 // projection that compresses from that user-turn boundary onward.
187 func (a *Agent) SummarizeFrom(ctx context.Context, fromIdx int) error {
188 return a.summarizeAtProjectionBoundary(ctx, fromIdx, "after")
189 }
190
191 // SummarizeUpTo keeps the compatibility index contract while installing a
192 // projection that compresses everything before that user-turn boundary.
193 func (a *Agent) SummarizeUpTo(ctx context.Context, toIdx int) error {
194 return a.summarizeAtProjectionBoundary(ctx, toIdx, "before")
195 }
196
197 func (a *Agent) summarizeAtProjectionBoundary(ctx context.Context, canonicalIndex int, direction string) error {
198 snap := a.snapshotExplicitCompression()
199 if canonicalIndex < 0 || canonicalIndex >= len(snap.canonical) {
200 return nil
201 }
202 anchor := snap.canonical[canonicalIndex]
203 if !compressAnchorCandidate(anchor) {
204 return nil
205 }
206 visibleIndex := -1
207 for i, msg := range snap.visible {
208 if !compressAnchorCandidate(msg) {
209 continue
210 }
211 if anchor.CreatedAt != 0 && msg.CreatedAt == anchor.CreatedAt {
212 visibleIndex = i
213 break
214 }
215 if anchor.CreatedAt == 0 && UserMessageText(msg) == UserMessageText(anchor) {
216 if visibleIndex >= 0 {
217 return fmt.Errorf("summarize boundary is ambiguous in the current model context")
218 }
219 visibleIndex = i
220 }
221 }
222 if visibleIndex < 0 {
223 return fmt.Errorf("context compression unavailable: selected turn is no longer present in the model context")
224 }
225 result, err := a.compressVisibleRange(ctx, snap, CompactionTriggerManual, direction, visibleIndex, anchorPreview(UserMessageText(anchor)), "")
226 if err != nil {
227 return err
228 }
229 if result.Status != "ok" {
230 reason := strings.TrimSpace(result.Reason)
231 if reason == "" {
232 reason = "selected range did not reduce the model context"
233 }
234 return fmt.Errorf("context compression skipped: %s", reason)
235 }
236 return nil
237 }
238
239 // IsCompactionSummary reports whether m is a rolling digest inserted by a
240 // prior compaction fold. Exported for session owners outside this package
241 // (e.g. the guardian) whose turn rollback must not treat a digest as a
242 // disposable user message.
243 func IsCompactionSummary(m provider.Message) bool { return isCompactionSummary(m) }
244
245 func (a *Agent) activeTurnStart(msgs []provider.Message) int {
246 createdAt := a.activeTurnCreatedAt.Load()
247 if createdAt == 0 {
248 return -1
249 }
250 for i, m := range msgs {
251 if m.Role == provider.RoleUser && m.CreatedAt == createdAt {
252 return i
253 }
254 }
255 return -1
256 }
257
258 // isCompactionSummary reports whether m is a rolling summary from a prior fold.
259 func isCompactionSummary(m provider.Message) bool {
260 return m.Role == provider.RoleUser &&
261 strings.HasPrefix(strings.TrimLeft(m.Content, "\n "), summaryTagOpen)
262 }
263
264 // pinnedPrefixLen keeps only the system message. All older user turns,
265 // failures, and [[keep]] markers enter the Harness-style summary prefix.
266 func (a *Agent) pinnedPrefixLen(msgs []provider.Message) int {
267 if len(msgs) > 0 && msgs[0].Role == provider.RoleSystem {
268 return 1
269 }
270 return 0
271 }
272
273 // planCompaction returns [head:start] to fold while retaining the newest 16%
274 // of the model window and keeping tool-call/result groups balanced.
275 func (a *Agent) planCompaction(msgs []provider.Message, min int, force bool) (head, start int, ok bool) {
276 head = a.pinnedPrefixLen(msgs)
277 if a.contextWindow > 0 {
278 budget := a.recentTailBudget()
279 if force {
280 if half := estimateMessagesTokens(modelInputMessages(msgs)) / 2; half > 0 && half < budget {
281 budget = half
282 }
283 }
284 start = tailStart(msgs, head, budget, a.tokPerChar(), a.tailFloor())
285 // Remeasure when force or non-strict roles; strict-alternating otherwise
286 // keeps a cheap tokPerChar overestimate of the tail under force.
287 floor := max(head, len(msgs)-a.tailFloor())
288 remeasure := force || !a.strictAlternatingRoles
289 for remeasure && start < floor && estimateMessagesTokens(provider.ModelMessages(msgs[start:])) > budget {
290 start++
291 for start < floor && start < len(msgs) && msgs[start].Role == provider.RoleTool {
292 start++
293 }
294 }
295 } else {
296 // No window: keep a fixed recent count, aligned off tool results.
297 start = len(msgs) - a.tailFloor()
298 for start > head && start < len(msgs) && msgs[start].Role == provider.RoleTool {
299 start--
300 }
301 }
302 start = max(start, head)
303 if start-head < min {
304 return head, start, false
305 }
306 return head, start, true
307 }
308
309 func (a *Agent) tailFloor() int {
310 return 0
311 }
312
313 // tailStart walks newest→oldest, growing the verbatim tail until the next
314 // message would push its token estimate past budgetTokens (but never below
315 // minKeep messages), then aligns the boundary back off any tool result so the
316 // tail never begins with an orphan whose assistant tool_calls were summarized
317 // away.
318 func tailStart(msgs []provider.Message, head, budgetTokens int, tokPerChar float64, minKeep int) int {
319 start := len(msgs)
320 acc := 0
321 for i := len(msgs) - 1; i > head; i-- {
322 c := int(float64(msgChars(msgs[i])) * tokPerChar)
323 if len(msgs)-i > minKeep && acc+c > budgetTokens {
324 break
325 }
326 acc += c
327 start = i
328 }
329 // start == len(msgs) when nothing fit the tail (a session too small to have a
330 // message after head); there is no msgs[start] to align off, and the caller's
331 // minCompactMessages check then no-ops the pass.
332 for start > head && start < len(msgs) && msgs[start].Role == provider.RoleTool {
333 start--
334 }
335 return start
336 }
337
338 // tokPerChar derives a tokens-per-character ratio from the last turn's real
339 // usage so per-message estimates track the provider's tokenizer without a local
340 // one. Reasoning content is excluded from the char count to match the prompt
341 // actually sent (the provider strips it). Falls back to ~4 chars/token before
342 // any usage is known, and ignores absurd ratios.
343 func (a *Agent) tokPerChar() float64 {
344 if cal := a.sess.output.promptCalibration.Load(); cal != nil && cal.compactChars > 0 {
345 if r := float64(cal.promptTokens) / float64(cal.compactChars); r > 0.05 && r < 2 {
346 return r
347 }
348 }
349 return fallbackTokPerChar
350 }
351
352 // msgChars counts the characters that ride to the provider for one message —
353 // content plus tool-call names and arguments, but not reasoning (stripped on
354 // send).
355 func msgChars(m provider.Message) int {
356 if m.LocalOnly {
357 return 0
358 }
359 n := len(m.Content)
360 for _, tc := range m.ToolCalls {
361 n += len(tc.Name) + len(tc.Arguments)
362 }
363 return n
364 }
365
366 func charsOfMessages(msgs []provider.Message) int {
367 n := 0
368 for _, m := range msgs {
369 n += msgChars(m)
370 }
371 return n
372 }
373
374 // summarize asks the executor's own provider to distill a replayed prefix into
375 // a briefing. instructions is optional /compact focus + PreCompact text.
376 // Named returns so defer can attach RequestCount and still return usage.
377 func compactionInstructionWithFocus(instructions string) string {
378 instruction := compactionInstruction
379 if strings.TrimSpace(instructions) != "" {
380 instruction += "\n\nAdditional focus for this compaction (prioritize keeping this):\n" + strings.TrimSpace(instructions)
381 }
382 return instruction
383 }
384
385 // summaryRequest builds the exact cache-aligned request shape used by
386 // summarize. Keeping planning and execution on this shared builder prevents a
387 // supposedly safe overflow fold from being rejected only after it is selected.
388 func (a *Agent) summaryRequest(region []provider.Message, instructions string) provider.Request {
389 prefix := append([]provider.Message(nil), region...)
390 for i := range prefix {
391 if !a.imageInput.native && prefix[i].VisionSummary != nil {
392 prefix[i].ImageInputs = nil
393 }
394 }
395 if len(prefix) == 0 || prefix[0].Role != provider.RoleSystem {
396 visible := a.modelVisibleMessages()
397 if len(visible) > 0 && visible[0].Role == provider.RoleSystem {
398 prefix = append([]provider.Message{visible[0]}, prefix...)
399 }
400 }
401 messages := a.normalizeModelRequestMessages(prefix)
402 messages = append(messages, HostGeneratedUserMessage(compactionInstructionWithFocus(instructions)))
403 var schemas []provider.ToolSchema
404 if a.svc.tools != nil {
405 schemas = a.providerToolSchemas()
406 }
407 return provider.Request{
408 Messages: messages,
409 Tools: schemas,
410 MaxTokens: a.summaryOutputBudget(),
411 Temperature: provider.OptionalTemperature(a.temperature),
412 }
413 }
414
415 // summarize asks the executor's own provider to distill a replayed prefix into
416 // a briefing. instructions is optional /compact focus + PreCompact text.
417 func (a *Agent) summarize(ctx context.Context, region []provider.Message, instructions string) (string, *provider.Usage, error) {
418 req := a.summaryRequest(region, instructions)
419 summary, usage, err := a.runSummaryRequest(ctx, req)
420 a.observeSummaryOutcome(req, usage, err)
421 return summary, usage, err
422 }
423
424 // runSummaryRequest admits, sends, and drains one summary request.
425 // Named returns so defer can attach RequestCount and still return usage.
426 func (a *Agent) runSummaryRequest(ctx context.Context, req provider.Request) (summary string, usage *provider.Usage, err error) {
427 req.Messages, err = a.resolveRequestImages(ctx, req.Messages)
428 if err != nil {
429 return "", nil, err
430 }
431 ctx, cancel := context.WithCancel(ctx)
432 defer cancel()
433 ctx = provider.WithRequestAttemptCounter(ctx)
434 defer func() {
435 usage = provider.UsageWithRequestAttemptCount(ctx, usage)
436 if usage != nil && (usage.TotalTokens > 0 || usage.RequestCount > 0) {
437 a.svc.sink.Emit(event.Event{Kind: event.Usage, ModelRef: a.modelRef, Usage: usage, Pricing: a.svc.pricing, UsageSource: event.UsageSourceCompaction})
438 }
439 }()
440 defer trackPublishedHostStream(ctx, cancel)()
441 if err := a.applySummaryAdmissionToRequest(&req); err != nil {
442 return "", usage, err
443 }
444 if budget := a.summaryOutputBudget(); req.MaxTokens > budget {
445 req.MaxTokens = budget
446 }
447 if req.MaxTokens < 256 {
448 return "", usage, fmt.Errorf("summary output budget too small (%d tokens)", req.MaxTokens)
449 }
450 if a.svc.prov == nil {
451 return "", usage, fmt.Errorf("summary unavailable")
452 }
453 ch, err := provider.StreamAuxiliary(provider.WithRecoverySleeper(ctx, recoverySleep), a.svc.prov, req)
454 if err != nil {
455 return "", usage, err
456 }
457
458 // Unblock on timeout if the stream stalls while open.
459 var b strings.Builder
460 var reasoning strings.Builder
461 toolCalls := 0
462 for {
463 select {
464 case <-ctx.Done():
465 return "", usage, ctx.Err()
466 case chunk, ok := <-ch:
467 if !ok {
468 if usage != nil && usage.FinishReason == "length" {
469 return "", usage, fmt.Errorf("%w: provider reached the output token limit", errSummaryOutputTruncated)
470 }
471 s := strings.TrimSpace(b.String())
472 if s == "" {
473 // Thinking providers may answer with reasoning_content only. Surface
474 // it as the briefing unless the turn also reached for tools: that
475 // reasoning is private chain-of-thought, not digest material.
476 r := strings.TrimSpace(reasoning.String())
477 if r == "" || toolCalls > 0 {
478 return "", usage, fmt.Errorf("summarizer returned empty output")
479 }
480 return truncateUTF8Bytes(r, summaryReasoningMaxBytes), usage, nil
481 }
482 return s, usage, nil
483 }
484 switch chunk.Type {
485 case provider.ChunkText:
486 b.WriteString(chunk.Text)
487 case provider.ChunkReasoning:
488 reasoning.WriteString(chunk.Text)
489 case provider.ChunkToolCall, provider.ChunkToolCallStart:
490 toolCalls++
491 case provider.ChunkUsage:
492 usage = chunk.Usage
493 case provider.ChunkError:
494 return "", usage, chunk.Err
495 }
496 }
497 }
498 }
499
500 // summarizeOnce performs exactly one application-layer summary request.
501 // Timeouts, empty results, stream errors, and output truncation all fail once
502 // with no second attempt.
503 func (a *Agent) summarizeOnce(ctx context.Context, fold []provider.Message, instructions string) (string, *provider.Usage, error) {
504 return a.summarize(ctx, fold, instructions)
505 }
506
507 // renderTranscript flattens messages into a bounded transcript for the
508 // transcript-form summary request. Tool bodies are the provider-visible
509 // Content cut to slimToolResultRunes; RawContent never enters a summary.
510 func renderTranscript(msgs []provider.Message) string {
511 var b strings.Builder
512 for _, m := range msgs {
513 if m.LocalOnly {
514 continue
515 }
516 switch m.Role {
517 case provider.RoleUser:
518 fmt.Fprintf(&b, "[user]\n%s\n\n", m.Content)
519 case provider.RoleAssistant:
520 if m.Content != "" {
521 fmt.Fprintf(&b, "[assistant]\n%s\n", m.Content)
522 }
523 for _, tc := range m.ToolCalls {
524 fmt.Fprintf(&b, "[assistant calls %s] %s\n", tc.Name, summarizeToolArgs(tc.Arguments))
525 }
526 b.WriteString("\n")
527 case provider.RoleTool:
528 fmt.Fprintf(&b, "[tool %s result]\n%s\n\n", m.Name, slimToolResult(m.Content))
529 case provider.RoleSystem:
530 fmt.Fprintf(&b, "[system]\n%s\n\n", m.Content)
531 }
532 }
533 return b.String()
534 }
535
536 // summarizeToolArgs returns a short summary of tool-call arguments instead of
537 // the full JSON. This prevents the summarizer from reproducing long argument
538 // text (like sub-agent task prompts) in the compaction summary, which would
539 // leak into the session as a user message (#4317).
540 func summarizeToolArgs(args string) string {
541 if args == "" {
542 return "(no arguments)"
543 }
544 var parsed map[string]any
545 if err := json.Unmarshal([]byte(args), &parsed); err != nil {
546 // Not valid JSON — return a length hint instead of raw text.
547 return fmt.Sprintf("(%d bytes)", len(args))
548 }
549 keys := make([]string, 0, len(parsed))
550 for k := range parsed {
551 keys = append(keys, k)
552 }
553 sort.Strings(keys)
554 return fmt.Sprintf("{%s} (%d keys)", strings.Join(keys, ", "), len(parsed))
555 }
556
556 lines GO