| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "reasonix/internal/event" |
| 16 | "reasonix/internal/tool" |
| 17 | ) |
| 18 | |
| 19 | const liveArgumentLabel = "中文🙂\nsecond line" |
| 20 | |
| 21 | type liveArgumentTool struct { |
| 22 | mu sync.Mutex |
| 23 | calls []string |
| 24 | invalid bool |
| 25 | } |
| 26 | |
| 27 | func (*liveArgumentTool) Name() string { return "inspect_marker" } |
| 28 | func (*liveArgumentTool) Description() string { |
| 29 | return "Inspect the requested marker. The unavailable marker returns an ordinary read error; alpha and beta return their marker." |
| 30 | } |
| 31 | func (*liveArgumentTool) ReadOnly() bool { return true } |
| 32 | func (*liveArgumentTool) Schema() json.RawMessage { |
| 33 | return json.RawMessage(`{"type":"object","properties":{"marker":{"type":"string","enum":["alpha","beta","unavailable"]},"label":{"type":"string"}},"required":["marker","label"],"additionalProperties":false}`) |
| 34 | } |
| 35 | func (p *liveArgumentTool) Execute(_ context.Context, args json.RawMessage) (string, error) { |
| 36 | var v struct { |
| 37 | Marker string `json:"marker"` |
| 38 | Label string `json:"label"` |
| 39 | } |
| 40 | if err := json.Unmarshal(args, &v); err != nil { |
| 41 | return "", err |
| 42 | } |
| 43 | p.mu.Lock() |
| 44 | defer p.mu.Unlock() |
| 45 | p.calls = append(p.calls, v.Marker) |
| 46 | if v.Label != liveArgumentLabel { |
| 47 | p.invalid = true |
| 48 | return "", fmt.Errorf("label did not preserve requested Unicode/newline") |
| 49 | } |
| 50 | if v.Marker == "unavailable" { |
| 51 | return "", deterministicApplicationError("synthetic read failure: marker unavailable") |
| 52 | } |
| 53 | if v.Marker != "alpha" && v.Marker != "beta" { |
| 54 | p.invalid = true |
| 55 | return "", fmt.Errorf("unexpected marker") |
| 56 | } |
| 57 | return "marker-" + v.Marker, nil |
| 58 | } |
| 59 | |
| 60 | // Parameters contain Unicode/escapes and the model must continue after an |
| 61 | // ordinary read error. No shell, credentials reader or external side effect. |
| 62 | func TestLiveMultiProviderArgumentsAndToolError(t *testing.T) { |
| 63 | for _, tc := range multiProviderCases() { |
| 64 | if os.Getenv(tc.keyEnv) == "" { |
| 65 | continue |
| 66 | } |
| 67 | t.Run(tc.vendor+"/"+tc.model+"/"+tc.protocol, func(t *testing.T) { |
| 68 | p := tc.new(t, "", "baseline") |
| 69 | reg := tool.NewRegistry() |
| 70 | probe := &liveArgumentTool{} |
| 71 | reg.Add(probe) |
| 72 | sess := NewSession("Use inspect_marker for each requested marker exactly once. An unavailable marker is an ordinary read error; continue the remaining reads without repeating it. Preserve literal argument strings exactly.") |
| 73 | sink := &recordSink{} |
| 74 | a := New(p, reg, sess, Options{MaxSteps: 6, MaxOutputTokens: 4096, MissingReasoningWarnStateDir: t.TempDir()}, sink) |
| 75 | ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) |
| 76 | defer cancel() |
| 77 | labelJSON, _ := json.Marshal(liveArgumentLabel) |
| 78 | err := a.Run(ctx, fmt.Sprintf("Call inspect_marker for unavailable, alpha, and beta, exactly once each. Decode the JSON string %s and use its value as label in every call. Then summarize the two available markers and the unavailable result.", labelJSON)) |
| 79 | probe.mu.Lock() |
| 80 | defer probe.mu.Unlock() |
| 81 | counts := map[string]int{} |
| 82 | for _, c := range probe.calls { |
| 83 | counts[c]++ |
| 84 | } |
| 85 | requests, prompt, completion := 0, 0, 0 |
| 86 | unknown := false |
| 87 | for _, e := range sink.kinds(event.Usage) { |
| 88 | if u := e.Usage; u != nil { |
| 89 | requests += u.RequestCount |
| 90 | prompt += u.PromptTokens |
| 91 | completion += u.CompletionTokens |
| 92 | unknown = unknown || u.Unknown |
| 93 | } |
| 94 | } |
| 95 | t.Logf("provider=%s model=%s protocol=%s requests=%d prompt=%d completion=%d usage_unknown=%t calls=%v retries=%d", tc.vendor, tc.model, tc.protocol, requests, prompt, completion, unknown, counts, len(sink.kinds(event.Retrying))) |
| 96 | if err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | if probe.invalid || counts["unavailable"] != 1 || counts["alpha"] != 1 || counts["beta"] != 1 { |
| 100 | t.Fatalf("argument/read-error continuation mismatch: invalid=%t calls=%v", probe.invalid, counts) |
| 101 | } |
| 102 | m := sess.Snapshot() |
| 103 | if len(m) == 0 || !strings.Contains(m[len(m)-1].Content, "alpha") || !strings.Contains(m[len(m)-1].Content, "beta") { |
| 104 | t.Fatal("final answer lost successful read results") |
| 105 | } |
| 106 | }) |
| 107 | } |
| 108 | } |
| 109 |