| 1 | //go:build live |
| 2 | |
| 3 | package agent |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/rand" |
| 8 | "encoding/json" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "strings" |
| 14 | "sync/atomic" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "reasonix/internal/config" |
| 19 | "reasonix/internal/event" |
| 20 | "reasonix/internal/tool" |
| 21 | ) |
| 22 | |
| 23 | type kimiActionFixture struct { |
| 24 | path, action, expected string |
| 25 | proposed, executed atomic.Int32 |
| 26 | } |
| 27 | |
| 28 | func (f *kimiActionFixture) Name() string { return f.action + "_fixture" } |
| 29 | func (f *kimiActionFixture) Description() string { |
| 30 | return "Perform the requested fixture operation. read returns the file contents, edit replaces the file with value, verify runs the fixed fixture verification test. Return real results only." |
| 31 | } |
| 32 | func (f *kimiActionFixture) ReadOnly() bool { return f.action != "edit" } |
| 33 | func (f *kimiActionFixture) Schema() json.RawMessage { |
| 34 | if f.action == "edit" { |
| 35 | return json.RawMessage(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"],"additionalProperties":false}`) |
| 36 | } |
| 37 | return json.RawMessage(`{"type":"object","properties":{},"additionalProperties":false}`) |
| 38 | } |
| 39 | func (f *kimiActionFixture) Execute(ctx context.Context, raw json.RawMessage) (string, error) { |
| 40 | f.proposed.Add(1) |
| 41 | if f.action == "edit" { |
| 42 | var v struct { |
| 43 | Value string `json:"value"` |
| 44 | } |
| 45 | if err := json.Unmarshal(raw, &v); err != nil { |
| 46 | return "", err |
| 47 | } |
| 48 | if v.Value != f.expected { |
| 49 | return "", deterministicApplicationError("value did not match the requested Unicode/newline text; decode the JSON string before passing value. No write occurred.") |
| 50 | } |
| 51 | if err := os.WriteFile(f.path, []byte(v.Value), 0600); err != nil { |
| 52 | return "", err |
| 53 | } |
| 54 | } else if f.action == "verify" { |
| 55 | binary, err := os.Executable() |
| 56 | if err != nil { |
| 57 | return "", err |
| 58 | } |
| 59 | cmd := exec.CommandContext(ctx, binary, "-test.run=^TestLiveKimiFixedVerifierChild$") |
| 60 | cmd.Env = []string{"REASONIX_FIXTURE_VERIFY=" + f.path, "REASONIX_FIXTURE_EXPECTED=" + f.expected} |
| 61 | if output, err := cmd.CombinedOutput(); err != nil { |
| 62 | return string(output), err |
| 63 | } |
| 64 | } |
| 65 | content, err := os.ReadFile(f.path) |
| 66 | if err != nil { |
| 67 | return "", err |
| 68 | } |
| 69 | f.executed.Add(1) |
| 70 | return string(content), nil |
| 71 | } |
| 72 | |
| 73 | // The child executes this fixed verifier only, with no provider credentials or |
| 74 | // arbitrary model-controlled commands, paths, or executable source. |
| 75 | func TestLiveKimiFixedVerifierChild(t *testing.T) { |
| 76 | path := os.Getenv("REASONIX_FIXTURE_VERIFY") |
| 77 | if path == "" { |
| 78 | t.Skip("fixed verifier child only") |
| 79 | } |
| 80 | actual, err := os.ReadFile(path) |
| 81 | if err != nil || string(actual) != os.Getenv("REASONIX_FIXTURE_EXPECTED") { |
| 82 | t.Fatal("fixture verification failed") |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func TestLiveKimiActionComparison(t *testing.T) { |
| 87 | if os.Getenv("OPENCODE_GO_API_KEY") == "" { |
| 88 | t.Skip("Go credential unavailable") |
| 89 | } |
| 90 | var tc multiProviderCase |
| 91 | for _, candidate := range multiProviderCases() { |
| 92 | if candidate.vendor == "go" && candidate.model == "kimi-k3" && candidate.protocol == "chat" { |
| 93 | tc = candidate |
| 94 | break |
| 95 | } |
| 96 | } |
| 97 | for _, effort := range []string{"low", "max"} { |
| 98 | for _, action := range []string{"read", "edit", "verify"} { |
| 99 | for sample := 0; sample < 10; sample++ { |
| 100 | for _, candidate := range []bool{sample%2 == 0, sample%2 != 0} { |
| 101 | t.Run(fmt.Sprintf("%s/%s/%02d/candidate_%t", effort, action, sample, candidate), func(t *testing.T) { |
| 102 | t.Parallel() |
| 103 | caseConfig := tc |
| 104 | caseConfig.effort = effort |
| 105 | p := caseConfig.new(t, "", "baseline") |
| 106 | marker := "fixture-" + rand.Text() |
| 107 | f := &kimiActionFixture{path: filepath.Join(t.TempDir(), "fixture.txt"), action: action, expected: marker} |
| 108 | if action == "edit" { |
| 109 | f.expected = "中文🙂\n" + marker |
| 110 | } |
| 111 | if err := os.WriteFile(f.path, []byte(marker), 0600); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | reg := tool.NewRegistry() |
| 115 | reg.Add(f) |
| 116 | system := config.DefaultSystemPrompt + "\n\n" + config.UserDecisionPolicy + "\n\n" + config.WorkPracticePolicy + "\n\n" + config.LanguagePolicy |
| 117 | if candidate { |
| 118 | system += "\n\n" + config.KimiActionPolicy |
| 119 | } |
| 120 | session := NewSession(system) |
| 121 | sink := &recordSink{} |
| 122 | a := New(p, reg, session, Options{MaxSteps: 5, MaxOutputTokens: 2048, MissingReasoningWarnStateDir: t.TempDir()}, sink) |
| 123 | prompt := "Use " + f.Name() + " to " + action + " the fixture and report the actual returned contents. Do not invent the contents." |
| 124 | if action == "edit" { |
| 125 | value, _ := json.Marshal(f.expected) |
| 126 | prompt = "Edit the fixture to the decoded value of this JSON string: " + string(value) + ". Use edit_fixture and report the actual result. If a call is rejected before writing, correct it; do not repeat a successful write." |
| 127 | } |
| 128 | ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) |
| 129 | defer cancel() |
| 130 | start := time.Now() |
| 131 | err := a.Run(ctx, prompt) |
| 132 | requests, tokens, output := 0, 0, 0 |
| 133 | unknown := false |
| 134 | for _, e := range sink.kinds(event.Usage) { |
| 135 | if e.Usage != nil { |
| 136 | requests += e.Usage.RequestCount |
| 137 | tokens += e.Usage.PromptTokens |
| 138 | output += e.Usage.CompletionTokens |
| 139 | unknown = unknown || e.Usage.Unknown |
| 140 | } |
| 141 | } |
| 142 | messages := session.Snapshot() |
| 143 | proposals, firstValid := 0, false |
| 144 | for _, message := range messages { |
| 145 | for _, call := range message.ToolCalls { |
| 146 | proposals++ |
| 147 | if proposals == 1 { |
| 148 | var values map[string]any |
| 149 | valid := json.Unmarshal([]byte(call.Arguments), &values) == nil && values != nil && call.Name == f.Name() |
| 150 | if action == "edit" { |
| 151 | valid = valid && len(values) == 1 && values["value"] == f.expected |
| 152 | } else { |
| 153 | valid = valid && len(values) == 0 |
| 154 | } |
| 155 | firstValid = valid |
| 156 | } |
| 157 | } |
| 158 | } |
| 159 | answer := "" |
| 160 | if len(messages) > 0 { |
| 161 | answer = messages[len(messages)-1].Content |
| 162 | } |
| 163 | passed := err == nil && f.executed.Load() == 1 && strings.Contains(answer, marker) |
| 164 | metric := map[string]any{"effort": effort, "action": action, "sample": sample, "candidate": candidate, "passed": passed, "proposed": proposals, "first_arguments_correct": firstValid, "entered_execute": f.proposed.Load(), "executed": f.executed.Load(), "requests": requests, "prompt": tokens, "output": output, "unknown_usage": unknown, "elapsed_ms": time.Since(start).Milliseconds(), "error": fmt.Sprint(err)} |
| 165 | encoded, _ := json.Marshal(metric) |
| 166 | t.Logf("METRIC %s", encoded) |
| 167 | if !passed { |
| 168 | t.Fail() |
| 169 | } |
| 170 | }) |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 |