| 1 | package goaleval |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "reasonix/internal/event" |
| 11 | "reasonix/internal/provider" |
| 12 | ) |
| 13 | |
| 14 | type scriptedProvider struct { |
| 15 | turns []string // one response per call, recycled |
| 16 | err error // stream-open error |
| 17 | timeout bool // hang until ctx deadline |
| 18 | usage *provider.Usage |
| 19 | calls int |
| 20 | } |
| 21 | |
| 22 | func (s *scriptedProvider) Name() string { return "scripted" } |
| 23 | |
| 24 | func (s *scriptedProvider) Stream(ctx context.Context, _ provider.Request) (<-chan provider.Chunk, error) { |
| 25 | s.calls++ |
| 26 | if s.err != nil { |
| 27 | return nil, s.err |
| 28 | } |
| 29 | ch := make(chan provider.Chunk, 2) |
| 30 | if s.timeout { |
| 31 | <-ctx.Done() |
| 32 | close(ch) |
| 33 | return ch, nil |
| 34 | } |
| 35 | i := s.calls - 1 |
| 36 | if i >= len(s.turns) { |
| 37 | i = len(s.turns) - 1 |
| 38 | } |
| 39 | ch <- provider.Chunk{Type: provider.ChunkText, Text: s.turns[i]} |
| 40 | if s.usage != nil { |
| 41 | ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: s.usage} |
| 42 | } |
| 43 | close(ch) |
| 44 | return ch, nil |
| 45 | } |
| 46 | |
| 47 | func evaluate(t *testing.T, prov provider.Provider, evidence GoalEvidence) (Verdict, error) { |
| 48 | t.Helper() |
| 49 | s := NewSession(prov, nil) |
| 50 | return s.Evaluate(context.Background(), evidence) |
| 51 | } |
| 52 | |
| 53 | func TestEvaluateParsesVerdicts(t *testing.T) { |
| 54 | for _, tc := range []struct { |
| 55 | body string |
| 56 | outcome Outcome |
| 57 | }{ |
| 58 | {`{"outcome":"complete","reason":"the goal is done"}`, OutcomeComplete}, |
| 59 | {`{"outcome":"continue","reason":"more work remains"}`, OutcomeContinue}, |
| 60 | {`{"outcome":"blocked","reason":"needs user input"}`, OutcomeBlocked}, |
| 61 | {`{"outcome":"uncertain","reason":"cannot judge"}`, OutcomeUncertain}, |
| 62 | // Fenced / prose-wrapped JSON is tolerated. |
| 63 | {"```json\n{\"outcome\":\"complete\",\"reason\":\"done\"}\n```", OutcomeComplete}, |
| 64 | {"Here is my judgment: {\"outcome\":\"continue\",\"reason\":\"keep going\"}", OutcomeContinue}, |
| 65 | } { |
| 66 | t.Run(tc.body, func(t *testing.T) { |
| 67 | prov := &scriptedProvider{turns: []string{tc.body}} |
| 68 | verdict, err := evaluate(t, prov, GoalEvidence{GoalContract: "fix the parser"}) |
| 69 | if err != nil { |
| 70 | t.Fatalf("Evaluate() error = %v", err) |
| 71 | } |
| 72 | if verdict.Outcome != tc.outcome { |
| 73 | t.Fatalf("outcome = %q, want %q", verdict.Outcome, tc.outcome) |
| 74 | } |
| 75 | }) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | func TestEvaluateFailClosedOnBadResponses(t *testing.T) { |
| 80 | for _, tc := range []struct { |
| 81 | name string |
| 82 | body string |
| 83 | }{ |
| 84 | {"empty", ""}, |
| 85 | {"invalid json", "{not json"}, |
| 86 | {"missing outcome", `{"reason":"no outcome"}`}, |
| 87 | {"invalid outcome", `{"outcome":"maybe","reason":"x"}`}, |
| 88 | } { |
| 89 | t.Run(tc.name, func(t *testing.T) { |
| 90 | prov := &scriptedProvider{turns: []string{tc.body}} |
| 91 | if _, err := evaluate(t, prov, GoalEvidence{}); err == nil { |
| 92 | t.Fatalf("Evaluate() error = nil, want fail-closed error for %q", tc.body) |
| 93 | } |
| 94 | }) |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | func TestEvaluateFailsOnProviderErrors(t *testing.T) { |
| 99 | prov := &scriptedProvider{err: errors.New("provider exploded")} |
| 100 | if _, err := evaluate(t, prov, GoalEvidence{}); err == nil { |
| 101 | t.Fatal("Evaluate() error = nil, want provider error") |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestEvaluateTimesOut(t *testing.T) { |
| 106 | prov := &scriptedProvider{timeout: true} |
| 107 | s := NewSession(prov, nil) |
| 108 | s.timeout = 50 * time.Millisecond |
| 109 | start := time.Now() |
| 110 | _, err := s.Evaluate(context.Background(), GoalEvidence{}) |
| 111 | if err == nil { |
| 112 | t.Fatal("Evaluate() error = nil, want timeout") |
| 113 | } |
| 114 | if elapsed := time.Since(start); elapsed > 2*time.Second { |
| 115 | t.Fatalf("Evaluate() took %v, want bounded timeout", elapsed) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | func TestEvaluateOverlongOutputFailsClosed(t *testing.T) { |
| 120 | prov := &scriptedProvider{turns: []string{strings.Repeat("x", MaxOutputBytes+1024)}} |
| 121 | if _, err := evaluate(t, prov, GoalEvidence{}); err == nil || !strings.Contains(err.Error(), "exceeded") { |
| 122 | t.Fatalf("Evaluate() error = %v, want output cap error", err) |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | func TestEvaluateEmitsGoalEvaluatorUsage(t *testing.T) { |
| 127 | usage := &provider.Usage{TotalTokens: 123, PromptTokens: 100, CompletionTokens: 23} |
| 128 | prov := &scriptedProvider{turns: []string{`{"outcome":"continue","reason":"keep going"}`}, usage: usage} |
| 129 | var seen *event.Event |
| 130 | sink := event.FuncSink(func(e event.Event) { |
| 131 | if e.Kind == event.Usage { |
| 132 | seen = &e |
| 133 | } |
| 134 | }) |
| 135 | s := NewSessionWithSink(prov, nil, "test/model", sink) |
| 136 | if _, err := s.Evaluate(context.Background(), GoalEvidence{GoalContract: "x"}); err != nil { |
| 137 | t.Fatalf("Evaluate() error = %v", err) |
| 138 | } |
| 139 | if seen == nil || seen.UsageSource != event.UsageSourceGoalEvaluator || seen.Usage == nil || seen.Usage.TotalTokens != 123 { |
| 140 | t.Fatalf("usage event = %+v, want goal-evaluator attribution with tokens", seen) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | func TestEvidenceIsBoundedAndUntrusted(t *testing.T) { |
| 145 | // Oversized evidence fields must be clipped, not rejected — and the JSON |
| 146 | // stays valid. |
| 147 | prov := &scriptedProvider{turns: []string{`{"outcome":"complete","reason":"done"}`}} |
| 148 | ev := GoalEvidence{ |
| 149 | GoalContract: strings.Repeat("g", 10_000), |
| 150 | AssistantFinal: strings.Repeat("a", 10_000), |
| 151 | TodoSummary: strings.Repeat("t", 10_000), |
| 152 | } |
| 153 | _, err := evaluate(t, prov, ev) |
| 154 | if err != nil { |
| 155 | t.Fatalf("Evaluate() error = %v, want clipped evidence accepted", err) |
| 156 | } |
| 157 | if prov.calls != 1 { |
| 158 | t.Fatalf("provider calls = %d, want 1", prov.calls) |
| 159 | } |
| 160 | } |
| 161 |