| 1 | package historycatalog |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "unicode/utf8" |
| 6 | |
| 7 | "reasonix/internal/agent" |
| 8 | "reasonix/internal/provider" |
| 9 | "reasonix/internal/retrieval" |
| 10 | ) |
| 11 | |
| 12 | const ( |
| 13 | toolTextMaxBytes = 8 * 1024 |
| 14 | toolTextHeadBytes = 6 * 1024 |
| 15 | toolTextTailBytes = 2 * 1024 |
| 16 | ) |
| 17 | |
| 18 | // toolTextTruncationMarker makes elided middle bytes recognizable in search |
| 19 | // hits, so truncation is not mistaken for source content. |
| 20 | const toolTextTruncationMarker = "\n…[truncated]…\n" |
| 21 | |
| 22 | // truncateToolText bounds one tool payload's indexed text (#8717: tool output |
| 23 | // is the index size driver). The tail survives because tool errors and |
| 24 | // summaries typically sit at the end of the output. |
| 25 | func truncateToolText(text string) string { |
| 26 | if len(text) <= toolTextMaxBytes { |
| 27 | return text |
| 28 | } |
| 29 | head := text[:toolTextHeadBytes] |
| 30 | for len(head) > 0 && !utf8.ValidString(head) { |
| 31 | head = head[:len(head)-1] |
| 32 | } |
| 33 | tail := text[len(text)-toolTextTailBytes:] |
| 34 | for len(tail) > 0 && !utf8.ValidString(tail) { |
| 35 | tail = tail[1:] |
| 36 | } |
| 37 | return head + toolTextTruncationMarker + tail |
| 38 | } |
| 39 | |
| 40 | type indexedDocument struct { |
| 41 | message int |
| 42 | part int |
| 43 | role string |
| 44 | kind string |
| 45 | tool string |
| 46 | terms string |
| 47 | count int |
| 48 | } |
| 49 | |
| 50 | func documents(messages []provider.Message) []indexedDocument { |
| 51 | out := []indexedDocument{} |
| 52 | appendDoc := func(message, part int, role, kind, tool, text string) { |
| 53 | terms := retrieval.Tokens(strings.TrimSpace(text)) |
| 54 | if len(terms) == 0 { |
| 55 | return |
| 56 | } |
| 57 | out = append(out, indexedDocument{message: message, part: part, role: role, kind: kind, tool: tool, terms: strings.Join(terms, " "), count: len(terms)}) |
| 58 | } |
| 59 | for i, msg := range messages { |
| 60 | if agent.IsPinnedContextRevision(msg) { |
| 61 | continue |
| 62 | } |
| 63 | switch msg.Role { |
| 64 | case provider.RoleUser: |
| 65 | appendDoc(i, 0, string(msg.Role), "user_text", "", msg.Content) |
| 66 | case provider.RoleAssistant: |
| 67 | appendDoc(i, 0, string(msg.Role), "assistant_text", "", msg.Content) |
| 68 | for part, call := range msg.ToolCalls { |
| 69 | appendDoc(i, part, string(msg.Role), "tool_input", call.Name, truncateToolText(call.Name+" "+call.Arguments)) |
| 70 | } |
| 71 | case provider.RoleTool: |
| 72 | // Index both tool_error and tool_output so explicit kind filters stay |
| 73 | // honest. Default search kinds still exclude tool_output. |
| 74 | text := truncateToolText(msg.Name + " " + msg.Content) |
| 75 | lower := strings.ToLower(strings.TrimSpace(msg.Content)) |
| 76 | if strings.HasPrefix(lower, "error:") || strings.HasPrefix(lower, "blocked:") || strings.Contains(lower, "permission denied") { |
| 77 | appendDoc(i, 0, string(msg.Role), "tool_error", msg.Name, text) |
| 78 | } |
| 79 | appendDoc(i, 0, string(msg.Role), "tool_output", msg.Name, text) |
| 80 | } |
| 81 | } |
| 82 | return out |
| 83 | } |
| 84 |