| 1 | package historycatalog |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | "unicode/utf8" |
| 7 | |
| 8 | "reasonix/internal/provider" |
| 9 | ) |
| 10 | |
| 11 | func TestTruncateToolText(t *testing.T) { |
| 12 | t.Parallel() |
| 13 | patterned := strings.Repeat("0123456789", 10240) // 100KB, position-varying |
| 14 | tests := []struct { |
| 15 | name string |
| 16 | text string |
| 17 | truncated bool |
| 18 | }{ |
| 19 | {"short unchanged", strings.Repeat("a", 100), false}, |
| 20 | {"at limit unchanged", patterned[:toolTextMaxBytes], false}, |
| 21 | {"one byte over truncated", patterned[:toolTextMaxBytes+1], true}, |
| 22 | {"large truncated", patterned, true}, |
| 23 | } |
| 24 | for _, tt := range tests { |
| 25 | t.Run(tt.name, func(t *testing.T) { |
| 26 | t.Parallel() |
| 27 | got := truncateToolText(tt.text) |
| 28 | if !tt.truncated { |
| 29 | if got != tt.text { |
| 30 | t.Fatalf("text changed: got %d bytes, want unchanged %d", len(got), len(tt.text)) |
| 31 | } |
| 32 | return |
| 33 | } |
| 34 | if !strings.Contains(got, toolTextTruncationMarker) { |
| 35 | t.Fatalf("missing truncation marker in %q…", got[:64]) |
| 36 | } |
| 37 | if !strings.HasPrefix(got, tt.text[:toolTextHeadBytes]) { |
| 38 | t.Fatal("head bytes altered") |
| 39 | } |
| 40 | if !strings.HasSuffix(got, tt.text[len(tt.text)-toolTextTailBytes:]) { |
| 41 | t.Fatal("tail bytes altered") |
| 42 | } |
| 43 | if len(got) > toolTextMaxBytes+len(toolTextTruncationMarker) { |
| 44 | t.Fatalf("truncated text still %d bytes", len(got)) |
| 45 | } |
| 46 | }) |
| 47 | } |
| 48 | } |
| 49 | |
| 50 | func TestTruncateToolTextKeepsRuneBoundary(t *testing.T) { |
| 51 | t.Parallel() |
| 52 | prefix := strings.Repeat("x", toolTextHeadBytes-1) |
| 53 | // Byte toolTextHeadBytes lands inside the multi-byte rune 世. |
| 54 | text := prefix + "世界" + strings.Repeat("y", 64*1024) |
| 55 | got := truncateToolText(text) |
| 56 | if !utf8.ValidString(got) { |
| 57 | t.Fatal("truncated text is not valid UTF-8") |
| 58 | } |
| 59 | if !strings.HasPrefix(got, prefix) { |
| 60 | t.Fatal("head lost bytes before the boundary rune") |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestDocumentsExcludePinnedContextRevisions(t *testing.T) { |
| 65 | docs := documents([]provider.Message{ |
| 66 | {Role: provider.RoleUser, Origin: provider.MessageOriginHost, Content: "<pinned_context_revision>private pinned body</pinned_context_revision>"}, |
| 67 | {Role: provider.RoleUser, Content: "visible question"}, |
| 68 | }) |
| 69 | if len(docs) != 1 || docs[0].kind != "user_text" || strings.Contains(docs[0].terms, "private") { |
| 70 | t.Fatalf("documents() = %#v, want only visible user text", docs) |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | func TestDocumentsTruncateToolPayloadsOnly(t *testing.T) { |
| 75 | t.Parallel() |
| 76 | filler := strings.Repeat("filler ", 4000) // 28KB |
| 77 | toolContent := filler[:10000] + " midmarkertoken " + filler[10000:] + " tailmarkertoken" |
| 78 | userContent := filler + " endusermarkertoken" |
| 79 | docs := documents([]provider.Message{ |
| 80 | {Role: provider.RoleUser, Content: userContent}, |
| 81 | {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: "1", Name: "bash", Arguments: toolContent}}}, |
| 82 | {Role: provider.RoleTool, ToolCallID: "1", Name: "bash", Content: toolContent}, |
| 83 | }) |
| 84 | byKind := map[string]indexedDocument{} |
| 85 | for _, doc := range docs { |
| 86 | byKind[doc.kind] = doc |
| 87 | } |
| 88 | user, ok := byKind["user_text"] |
| 89 | if !ok || !strings.Contains(user.terms, "endusermarkertoken") { |
| 90 | t.Fatalf("user_text must stay untruncated: %#v", user) |
| 91 | } |
| 92 | for _, kind := range []string{"tool_input", "tool_output"} { |
| 93 | doc, ok := byKind[kind] |
| 94 | if !ok { |
| 95 | t.Fatalf("missing %s document", kind) |
| 96 | } |
| 97 | if !strings.Contains(doc.terms, "tailmarkertoken") { |
| 98 | t.Fatalf("%s lost its tail token", kind) |
| 99 | } |
| 100 | if strings.Contains(doc.terms, "midmarkertoken") { |
| 101 | t.Fatalf("%s kept elided middle token", kind) |
| 102 | } |
| 103 | if !strings.Contains(doc.terms, "truncated") { |
| 104 | t.Fatalf("%s is missing the truncation marker", kind) |
| 105 | } |
| 106 | } |
| 107 | } |
| 108 |