| 1 | package control |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "os/exec" |
| 9 | "path/filepath" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | |
| 13 | "reasonix/internal/agent" |
| 14 | "reasonix/internal/event" |
| 15 | "reasonix/internal/provider" |
| 16 | "reasonix/internal/session" |
| 17 | "reasonix/internal/tool" |
| 18 | ) |
| 19 | |
| 20 | type crashAfterEffectTool struct{ path string } |
| 21 | |
| 22 | func (crashAfterEffectTool) Name() string { return "crash_after_effect" } |
| 23 | func (crashAfterEffectTool) Description() string { return "test fixture" } |
| 24 | func (crashAfterEffectTool) Schema() json.RawMessage { return json.RawMessage(`{"type":"object"}`) } |
| 25 | func (crashAfterEffectTool) ReadOnly() bool { return false } |
| 26 | func (t crashAfterEffectTool) Execute(context.Context, json.RawMessage) (string, error) { |
| 27 | f, err := os.OpenFile(t.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) |
| 28 | if err != nil { |
| 29 | return "", err |
| 30 | } |
| 31 | if _, err = f.WriteString("effect\n"); err != nil { |
| 32 | return "", err |
| 33 | } |
| 34 | if err = f.Sync(); err != nil { |
| 35 | return "", err |
| 36 | } |
| 37 | _ = f.Close() |
| 38 | os.Exit(73) // A committed external effect, with no local result receipt. |
| 39 | return "", nil |
| 40 | } |
| 41 | |
| 42 | func TestToolRecoveryCrashAfterEffect(t *testing.T) { |
| 43 | if root := os.Getenv("REASONIX_RECOVERY_CRASH_FIXTURE"); root != "" { |
| 44 | reg := tool.NewRegistry() |
| 45 | reg.Add(crashAfterEffectTool{path: filepath.Join(root, "effects")}) |
| 46 | p := &recordingProvider{streams: [][]provider.Chunk{{{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "crash", Name: "crash_after_effect", Arguments: `{}`}}, {Type: provider.ChunkDone}}}} |
| 47 | a := agent.New(p, reg, agent.NewSession("sys"), agent.Options{}, event.Discard) |
| 48 | c := newOwnedTestController(t, Options{Executor: a, Runner: a, SessionPath: filepath.Join(root, "session.jsonl"), SessionDir: root, Sink: event.Discard}) |
| 49 | if err := c.RunTurn(context.Background(), "perform effect"); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | t.Fatal("fixture failed to crash") |
| 53 | } |
| 54 | root := t.TempDir() |
| 55 | cmd := exec.Command(os.Args[0], "-test.run=^TestToolRecoveryCrashAfterEffect$") |
| 56 | cmd.Env = append(os.Environ(), "REASONIX_RECOVERY_CRASH_FIXTURE="+root) |
| 57 | out, err := cmd.CombinedOutput() |
| 58 | var exit *exec.ExitError |
| 59 | if !errors.As(err, &exit) || exit.ExitCode() != 73 { |
| 60 | t.Fatalf("crash helper: %v %s", err, out) |
| 61 | } |
| 62 | path := filepath.Join(root, "session.jsonl") |
| 63 | a := agent.New(nil, tool.NewRegistry(), agent.NewSession("sys"), agent.Options{}, event.Discard) |
| 64 | c := newOwnedTestController(t, Options{Executor: a, SessionPath: path, SessionDir: root, Sink: event.Discard}) |
| 65 | defer c.Close() |
| 66 | c.recoverInterruptedTurn(path) |
| 67 | view := c.ToolRecoverySnapshot() |
| 68 | if !view.Retired || view.RetryEnabled || len(view.Calls) != 0 { |
| 69 | t.Fatalf("unresolved crash effects=%+v", view) |
| 70 | } |
| 71 | req := ToolRecoveryRequest{SessionPath: view.SessionPath, RuntimeEpoch: view.RuntimeEpoch, Revision: view.Revision, AttemptID: "crash", Action: "inspect"} |
| 72 | if _, err = c.ResolveToolRecovery(context.Background(), req); err == nil || !strings.Contains(err.Error(), "tool_recovery_retired") { |
| 73 | t.Fatalf("retired recovery action err=%v", err) |
| 74 | } |
| 75 | projection := loadDurableSessionProjection(t, path) |
| 76 | if len(projection.ActiveTools) != 0 || projection.TurnStatus != event.TurnInterrupted { |
| 77 | t.Fatal("retired endpoint rewrote the historical unknown fact") |
| 78 | } |
| 79 | commits, err := session.Replay(sessionDirectory(path), nil) |
| 80 | if err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | unknown := false |
| 84 | for _, commit := range commits { |
| 85 | for _, recorded := range commit.Events { |
| 86 | if recorded.Kind != "tool/result" { |
| 87 | continue |
| 88 | } |
| 89 | var body struct { |
| 90 | ID string `json:"id"` |
| 91 | State string `json:"state"` |
| 92 | } |
| 93 | if json.Unmarshal(recorded.Payload, &body) == nil && body.ID == "crash" && body.State == "result_unknown" { |
| 94 | unknown = true |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | if !unknown { |
| 99 | t.Fatal("restart did not preserve the unknown external result as a typed v3 fact") |
| 100 | } |
| 101 | effects, err := os.ReadFile(filepath.Join(root, "effects")) |
| 102 | if err != nil || string(effects) != "effect\n" { |
| 103 | t.Fatalf("effect repeated: %q %v", effects, err) |
| 104 | } |
| 105 | } |
| 106 |