| 1 | package sidecar |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "slices" |
| 6 | "strconv" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "reasonix/internal/extension" |
| 11 | "reasonix/internal/extension/dispatch" |
| 12 | "reasonix/internal/pluginpkg" |
| 13 | ) |
| 14 | |
| 15 | // BenchmarkExtensionSidecarStartup measures a real no-op sidecar process |
| 16 | // spawn plus Extension Protocol handshake. Shutdown is kept outside the |
| 17 | // benchmark timer; sampled p50/p95 cover StartClient only. |
| 18 | func BenchmarkExtensionSidecarStartup(b *testing.B) { |
| 19 | pkg, installed := fakeSidecarPackage(b, "benchmark", nil) |
| 20 | opts := ClientOptions{Package: pkg, Installed: installed, Session: testSessionContext()} |
| 21 | b.ReportAllocs() |
| 22 | const maxSamples = 100_000 |
| 23 | samples := make([]int64, 0, maxSamples) |
| 24 | for b.Loop() { |
| 25 | start := time.Now() |
| 26 | client, err := StartClient(context.Background(), opts) |
| 27 | if err != nil { |
| 28 | b.Fatal(err) |
| 29 | } |
| 30 | elapsed := time.Since(start).Nanoseconds() |
| 31 | b.StopTimer() |
| 32 | if err := client.Close(); err != nil { |
| 33 | b.Fatal(err) |
| 34 | } |
| 35 | b.StartTimer() |
| 36 | if len(samples) < maxSamples { |
| 37 | samples = append(samples, elapsed) |
| 38 | } |
| 39 | } |
| 40 | b.StopTimer() |
| 41 | reportSidecarPercentiles(b, samples) |
| 42 | } |
| 43 | |
| 44 | // BenchmarkExtensionSidecarDispatchLatency measures actual NDJSON RPC through |
| 45 | // one and four real no-op sidecar processes on turn and tool hot paths. Calls |
| 46 | // are serial by protocol contract, so the four-sidecar case exposes additive |
| 47 | // latency instead of only the dispatcher's in-process overhead. |
| 48 | func BenchmarkExtensionSidecarDispatchLatency(b *testing.B) { |
| 49 | for _, count := range []int{1, 4} { |
| 50 | b.Run("Turn/NoopSidecars"+strconv.Itoa(count), func(b *testing.B) { |
| 51 | d := benchmarkSidecarDispatcher(b, extension.PointInputReceive, count) |
| 52 | payload := dispatch.InputPayload{Text: "hello"} |
| 53 | benchmarkSidecarLatency(b, func() error { |
| 54 | _, err := d.Intercept(context.Background(), extension.PointInputReceive, &payload) |
| 55 | return err |
| 56 | }) |
| 57 | }) |
| 58 | b.Run("Tool/NoopSidecars"+strconv.Itoa(count), func(b *testing.B) { |
| 59 | d := benchmarkSidecarDispatcher(b, extension.PointToolBefore, count) |
| 60 | payload := dispatch.ToolBeforePayload{Name: "bash", Arguments: `{"cmd":"pwd"}`} |
| 61 | benchmarkSidecarLatency(b, func() error { |
| 62 | _, err := d.Intercept(context.Background(), extension.PointToolBefore, &payload) |
| 63 | return err |
| 64 | }) |
| 65 | }) |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | func benchmarkSidecarDispatcher(b *testing.B, point extension.InterceptorPoint, count int) *dispatch.Dispatcher { |
| 70 | b.Helper() |
| 71 | chain := make([]extension.Contribution, 0, count) |
| 72 | clients := make(map[string]*Client, count) |
| 73 | for i := range count { |
| 74 | pluginID := "benchmark-" + strconv.Itoa(i) |
| 75 | client := startFakeClient(b, func(rt *pluginpkg.RuntimeSpec) { |
| 76 | rt.Intercepts = []string{string(point)} |
| 77 | }, nil) |
| 78 | clients[pluginID] = client |
| 79 | chain = append(chain, extension.Contribution{ |
| 80 | Kind: extension.KindInterceptor, |
| 81 | ID: string(point), |
| 82 | Source: extension.ContributionSource{ |
| 83 | Scope: extension.ScopePlugin, |
| 84 | PluginID: pluginID, |
| 85 | }, |
| 86 | }) |
| 87 | } |
| 88 | return dispatch.New(map[extension.InterceptorPoint][]extension.Contribution{point: chain}, nil, |
| 89 | func(pluginID string) dispatch.Client { return clients[pluginID] }, nil, dispatch.Options{}) |
| 90 | } |
| 91 | |
| 92 | func benchmarkSidecarLatency(b *testing.B, fn func() error) { |
| 93 | b.Helper() |
| 94 | b.ReportAllocs() |
| 95 | const maxSamples = 100_000 |
| 96 | samples := make([]int64, 0, maxSamples) |
| 97 | for b.Loop() { |
| 98 | start := time.Now() |
| 99 | if err := fn(); err != nil { |
| 100 | b.Fatal(err) |
| 101 | } |
| 102 | if len(samples) < maxSamples { |
| 103 | samples = append(samples, time.Since(start).Nanoseconds()) |
| 104 | } |
| 105 | } |
| 106 | b.StopTimer() |
| 107 | reportSidecarPercentiles(b, samples) |
| 108 | } |
| 109 | |
| 110 | func reportSidecarPercentiles(b *testing.B, samples []int64) { |
| 111 | b.Helper() |
| 112 | slices.Sort(samples) |
| 113 | if len(samples) == 0 { |
| 114 | return |
| 115 | } |
| 116 | b.ReportMetric(float64(samples[(len(samples)-1)*50/100]), "p50-ns/op") |
| 117 | b.ReportMetric(float64(samples[(len(samples)-1)*95/100]), "p95-ns/op") |
| 118 | } |
| 119 | |
| 120 | var _ dispatch.Client = (*Client)(nil) |
| 121 |