| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "net" |
| 8 | "net/http" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | ) |
| 14 | |
| 15 | // errEnsureHealing stands in for the transient EnsureServer failures observed |
| 16 | // while the SSH layer is still re-establishing a dropped tunnel. |
| 17 | var errEnsureHealing = errors.New("tunnel healing") |
| 18 | |
| 19 | func TestRemoteTabSnapshotReplaysAndClearsPendingPrompt(t *testing.T) { |
| 20 | fs := newFakeServe(t, "s3cret", nil) |
| 21 | kernel := &fakeRemoteKernel{ |
| 22 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 23 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 24 | } |
| 25 | seedBridgeTestHost(t, "box") |
| 26 | a := &App{remoteRuntime: kernel} |
| 27 | cleanupRemoteTabPumps(t, a) |
| 28 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 29 | a.remoteTabMu.Lock() |
| 30 | gen := a.remoteTabs[meta.ID].gen |
| 31 | a.remoteTabMu.Unlock() |
| 32 | a.cacheRemotePendingEvent(meta.ID, gen, "approval_request", json.RawMessage(`{"kind":"approval_request","approval":{"id":"approval-1","tool":"bash"}}`)) |
| 33 | snap, err := a.RemoteTabSnapshot(meta.ID) |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | if len(snap.PendingEvents) != 1 || !strings.Contains(string(snap.PendingEvents[0]), "approval-1") { |
| 38 | t.Fatalf("pending replay = %s", snap.PendingEvents) |
| 39 | } |
| 40 | if err := a.ApproveRemoteTab(meta.ID, "approval-1", "deny"); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | snap, err = a.RemoteTabSnapshot(meta.ID) |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | if len(snap.PendingEvents) != 0 { |
| 48 | t.Fatalf("resolved prompt was still replayed: %s", snap.PendingEvents) |
| 49 | } |
| 50 | |
| 51 | form := json.RawMessage(`{"kind":"extension_surface","extension":{"pluginId":"remote-plugin","surfaceId":"setup","kind":"form","form":{"title":"Remote setup","fields":[{"key":"region","label":"Region","kind":"input"}]}}}`) |
| 52 | if !a.cacheRemotePendingExtensionForm(meta.ID, gen, form) { |
| 53 | t.Fatal("actionable extension form was not retained") |
| 54 | } |
| 55 | snap, err = a.RemoteTabSnapshot(meta.ID) |
| 56 | if err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | if len(snap.PendingEvents) != 1 || !strings.Contains(string(snap.PendingEvents[0]), "remote-plugin") { |
| 60 | t.Fatalf("pending extension form replay = %s", snap.PendingEvents) |
| 61 | } |
| 62 | fs.mu.Lock() |
| 63 | fs.failNext = "form rejected" |
| 64 | fs.mu.Unlock() |
| 65 | if err := a.SubmitRemoteTabExtensionForm(meta.ID, "remote-plugin", "setup", map[string]any{"region": "us-west"}); err == nil { |
| 66 | t.Fatal("failed extension form submission succeeded") |
| 67 | } |
| 68 | snap, err = a.RemoteTabSnapshot(meta.ID) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | if len(snap.PendingEvents) != 1 { |
| 73 | t.Fatalf("failed extension form submission cleared replay: %s", snap.PendingEvents) |
| 74 | } |
| 75 | if err := a.SubmitRemoteTabExtensionForm(meta.ID, "remote-plugin", "setup", map[string]any{"region": "us-west"}); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | snap, err = a.RemoteTabSnapshot(meta.ID) |
| 79 | if err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | if len(snap.PendingEvents) != 0 { |
| 83 | t.Fatalf("submitted extension form was still replayed: %s", snap.PendingEvents) |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | func TestRemoteTabSnapshotRehydratesAndDropsPriorSessionPromptOnStatusAdoption(t *testing.T) { |
| 88 | const firstPath = "/sessions/first.jsonl" |
| 89 | const nextPath = "/sessions/next.jsonl" |
| 90 | fs := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "first", Path: firstPath, Current: true}}) |
| 91 | kernel := &fakeRemoteKernel{ |
| 92 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 93 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 94 | } |
| 95 | seedBridgeTestHost(t, "box") |
| 96 | log := &eventLog{} |
| 97 | a := &App{remoteRuntime: kernel, remoteEventHook: log.add} |
| 98 | cleanupRemoteTabPumps(t, a) |
| 99 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{SessionName: "first", SessionPath: firstPath}) |
| 100 | a.remoteTabMu.Lock() |
| 101 | gen := a.remoteTabs[meta.ID].gen |
| 102 | a.remoteTabMu.Unlock() |
| 103 | a.cacheRemotePendingEvent(meta.ID, gen, "approval_request", json.RawMessage(`{"kind":"approval_request","approval":{"id":"old-approval"}}`)) |
| 104 | readyBefore := log.count("remote-tab:" + meta.ID + ":state ") |
| 105 | fs.mu.Lock() |
| 106 | fs.statusPayload = `{"sessionName":"next","sessionPath":"` + nextPath + `","pendingPrompt":false}` |
| 107 | fs.mu.Unlock() |
| 108 | snap, err := a.RemoteTabSnapshot(meta.ID) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | if len(snap.PendingEvents) != 0 { |
| 113 | t.Fatalf("new session snapshot replayed prior prompt: %s", snap.PendingEvents) |
| 114 | } |
| 115 | a.remoteTabMu.Lock() |
| 116 | path := a.remoteTabs[meta.ID].routing.currentPath |
| 117 | a.remoteTabMu.Unlock() |
| 118 | if path != nextPath || log.count("remote-tab:"+meta.ID+":state ") != readyBefore+1 { |
| 119 | t.Fatalf("status adoption path/ready barrier = %q/%v", path, log.recorded()) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | func TestRemoteTabDoesNotPublishReadyWithoutEventStream(t *testing.T) { |
| 124 | previousDelays := remoteTabReattachDelays |
| 125 | remoteTabReattachDelays = nil |
| 126 | t.Cleanup(func() { remoteTabReattachDelays = previousDelays }) |
| 127 | |
| 128 | fs := newFakeServe(t, "s3cret", nil) |
| 129 | fs.mu.Lock() |
| 130 | fs.eventsStatus = http.StatusServiceUnavailable |
| 131 | fs.mu.Unlock() |
| 132 | kernel := &fakeRemoteKernel{ |
| 133 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 134 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 135 | } |
| 136 | seedBridgeTestHost(t, "box") |
| 137 | a := &App{remoteRuntime: kernel} |
| 138 | cleanupRemoteTabPumps(t, a) |
| 139 | meta, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{NewSession: true}) |
| 140 | if err != nil { |
| 141 | t.Fatal(err) |
| 142 | } |
| 143 | // A refusing stream no longer parks the tab in error — it retries through |
| 144 | // the reattach loop and only then parks in serve_down. Ready must never |
| 145 | // publish without a live stream either way. |
| 146 | waitForTabState(t, a, meta.ID, "serve_down") |
| 147 | time.Sleep(50 * time.Millisecond) |
| 148 | a.remoteTabMu.Lock() |
| 149 | state := a.remoteTabs[meta.ID].state |
| 150 | a.remoteTabMu.Unlock() |
| 151 | if state == "ready" { |
| 152 | t.Fatal("tab published ready after /events failed") |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | func TestRemoteTabDoesNotPublishReadyWhenEventStreamClosesDuringAttach(t *testing.T) { |
| 157 | previousDelays := remoteTabReattachDelays |
| 158 | remoteTabReattachDelays = nil |
| 159 | t.Cleanup(func() { remoteTabReattachDelays = previousDelays }) |
| 160 | |
| 161 | fs := newFakeServe(t, "s3cret", nil) |
| 162 | fs.mu.Lock() |
| 163 | fs.eventsCloseEarly = true |
| 164 | fs.enterDelay = 100 * time.Millisecond |
| 165 | fs.mu.Unlock() |
| 166 | kernel := &fakeRemoteKernel{ |
| 167 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 168 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 169 | } |
| 170 | seedBridgeTestHost(t, "box") |
| 171 | log := &eventLog{} |
| 172 | a := &App{remoteRuntime: kernel, remoteEventHook: log.add} |
| 173 | cleanupRemoteTabPumps(t, a) |
| 174 | meta, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{NewSession: true}) |
| 175 | if err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | waitForTabState(t, a, meta.ID, "serve_down") |
| 179 | for _, event := range log.recorded() { |
| 180 | if strings.HasPrefix(event, "remote-tab:"+meta.ID+":state ") && strings.Contains(event, `"state":"ready"`) { |
| 181 | t.Fatalf("closed event stream published ready: %v", log.recorded()) |
| 182 | } |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | func TestRemoteTabReviveAppliesRequestedNamedSession(t *testing.T) { |
| 187 | fs := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "saved", Path: "/saved.jsonl", Title: "Saved"}}) |
| 188 | feed := make(chan string, 1) |
| 189 | kernel := &fakeRemoteKernel{ |
| 190 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 191 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 192 | } |
| 193 | seedBridgeTestHost(t, "box") |
| 194 | log := &eventLog{} |
| 195 | a := &App{remoteRuntime: kernel, remoteEventHook: log.add} |
| 196 | cleanupRemoteTabPumps(t, a) |
| 197 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 198 | a.remoteTabMu.Lock() |
| 199 | tab := a.remoteTabs[meta.ID] |
| 200 | if tab.cancel != nil { |
| 201 | tab.cancel() |
| 202 | } |
| 203 | tab.gen++ |
| 204 | tab.cancel, tab.client, tab.base, tab.token = nil, nil, "", "" |
| 205 | tab.state = "disconnected" |
| 206 | tab.session = remoteTabSessionState{newSession: true} |
| 207 | a.remoteTabMu.Unlock() |
| 208 | fs.mu.Lock() |
| 209 | fs.eventFeed = feed |
| 210 | fs.resumeStarted = make(chan string, 1) |
| 211 | fs.resumeRelease = make(chan struct{}) |
| 212 | started, release := fs.resumeStarted, fs.resumeRelease |
| 213 | fs.mu.Unlock() |
| 214 | t.Cleanup(func() { |
| 215 | select { |
| 216 | case <-release: |
| 217 | default: |
| 218 | close(release) |
| 219 | } |
| 220 | }) |
| 221 | if _, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{SessionName: "saved"}); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | select { |
| 225 | case <-started: |
| 226 | case <-time.After(time.Second): |
| 227 | t.Fatal("revived resume request did not start") |
| 228 | } |
| 229 | feed <- `{"kind":"notice","text":"revived output","sessionPath":"/saved.jsonl"}` |
| 230 | deadline := time.Now().Add(time.Second) |
| 231 | for log.count("remote-tab:"+meta.ID+":event") < 3 { |
| 232 | if time.Now().After(deadline) { |
| 233 | t.Fatalf("revived target frame was dropped while /resume was pending: %v", log.recorded()) |
| 234 | } |
| 235 | time.Sleep(time.Millisecond) |
| 236 | } |
| 237 | close(release) |
| 238 | waitForTabState(t, a, meta.ID, "ready") |
| 239 | _, resumed, _ := fs.snapshot() |
| 240 | if resumed != "/saved.jsonl" { |
| 241 | t.Fatalf("revived shell resumed %q, want the selected session", resumed) |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | func TestRemoteTabServeDownCanRetry(t *testing.T) { |
| 246 | fs := newFakeServe(t, "s3cret", nil) |
| 247 | kernel := &fakeRemoteKernel{ |
| 248 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 249 | ensureView: RemoteServerView{HostID: "box", State: "error", Error: "temporary"}, ensureToken: "s3cret", |
| 250 | } |
| 251 | seedBridgeTestHost(t, "box") |
| 252 | a := &App{remoteRuntime: kernel} |
| 253 | cleanupRemoteTabPumps(t, a) |
| 254 | meta, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{NewSession: true}) |
| 255 | if err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | waitForTabState(t, a, meta.ID, "serve_down") |
| 259 | kernel.ensureView = RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL} |
| 260 | if _, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{NewSession: true}); err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | waitForTabState(t, a, meta.ID, "ready") |
| 264 | } |
| 265 | |
| 266 | func TestRemoteTabServeDownNewSessionClearsPendingBeforeDelayedMarker(t *testing.T) { |
| 267 | const oldPath = "/sessions/old.jsonl" |
| 268 | const freshPath = "/sessions/fresh.jsonl" |
| 269 | feed := make(chan string, 1) |
| 270 | fs := newFakeServe(t, "s3cret", nil) |
| 271 | kernel := &fakeRemoteKernel{ |
| 272 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 273 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 274 | } |
| 275 | seedBridgeTestHost(t, "box") |
| 276 | log := &eventLog{} |
| 277 | a := &App{remoteRuntime: kernel, remoteEventHook: log.add} |
| 278 | cleanupRemoteTabPumps(t, a) |
| 279 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 280 | |
| 281 | a.remoteTabMu.Lock() |
| 282 | tab := a.remoteTabs[meta.ID] |
| 283 | if tab.cancel != nil { |
| 284 | tab.cancel() |
| 285 | } |
| 286 | tab.gen++ |
| 287 | tab.cancel, tab.client, tab.base, tab.token = nil, nil, "", "" |
| 288 | tab.state = "serve_down" |
| 289 | tab.session = remoteTabSessionState{name: "old", path: oldPath} |
| 290 | tab.routing.currentPath = oldPath |
| 291 | tab.pendingEvents = map[string]json.RawMessage{ |
| 292 | "approval_request:old": json.RawMessage(`{"kind":"approval_request","approval":{"id":"old"}}`), |
| 293 | } |
| 294 | tab.runtime = remoteTabRuntimeState{pendingPrompt: true, cancellable: true} |
| 295 | a.remoteTabMu.Unlock() |
| 296 | fs.mu.Lock() |
| 297 | fs.newSessionPath = freshPath |
| 298 | fs.eventFeed = feed |
| 299 | fs.mu.Unlock() |
| 300 | |
| 301 | if _, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{NewSession: true}); err != nil { |
| 302 | t.Fatal(err) |
| 303 | } |
| 304 | waitForTabState(t, a, meta.ID, "ready") |
| 305 | a.remoteTabMu.Lock() |
| 306 | path, pending, prompt := tab.routing.currentPath, len(tab.pendingEvents), tab.runtime.pendingPrompt |
| 307 | a.remoteTabMu.Unlock() |
| 308 | if path != freshPath || pending != 0 || prompt { |
| 309 | t.Fatalf("fresh attach route/pending/prompt = %q/%d/%v, want %q/0/false", path, pending, prompt, freshPath) |
| 310 | } |
| 311 | |
| 312 | eventPrefix := "remote-tab:" + meta.ID + ":event" |
| 313 | before := log.count(eventPrefix) |
| 314 | feed <- `{"kind":"session_changed","sessionPath":"/sessions/fresh.jsonl","sessionCurrent":true,"sessionReset":true}` |
| 315 | waitForRemoteEventCount(t, log, eventPrefix, before+1) |
| 316 | a.remoteTabMu.Lock() |
| 317 | pending, prompt = len(tab.pendingEvents), tab.runtime.pendingPrompt |
| 318 | a.remoteTabMu.Unlock() |
| 319 | if pending != 0 || prompt { |
| 320 | t.Fatalf("delayed reset marker restored stale prompt: pending=%d prompt=%v", pending, prompt) |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | func TestRemoteTabFocusOnlyAttachPreservesCurrentServeSession(t *testing.T) { |
| 325 | fs := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "current", Path: "/current.jsonl", Title: "Current", Current: true}}) |
| 326 | kernel := &fakeRemoteKernel{statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret"} |
| 327 | seedBridgeTestHost(t, "box") |
| 328 | a := &App{remoteRuntime: kernel} |
| 329 | cleanupRemoteTabPumps(t, a) |
| 330 | meta, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{}) |
| 331 | if err != nil { |
| 332 | t.Fatal(err) |
| 333 | } |
| 334 | waitForTabState(t, a, meta.ID, "ready") |
| 335 | newCalled, resumed, _ := fs.snapshot() |
| 336 | if newCalled != 0 || resumed != "" { |
| 337 | t.Fatalf("focus-only attach changed Serve session: new=%d resume=%q", newCalled, resumed) |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | func TestRemoteSavedSessionLookupCannotReviveDisconnectedGeneration(t *testing.T) { |
| 342 | fs := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "saved", Path: "/saved.jsonl", Title: "Saved"}}) |
| 343 | fs.sessionsStarted = make(chan struct{}, 1) |
| 344 | fs.sessionsRelease = make(chan struct{}) |
| 345 | kernel := &fakeRemoteKernel{statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret"} |
| 346 | seedBridgeTestHost(t, "box") |
| 347 | a := &App{remoteRuntime: kernel} |
| 348 | cleanupRemoteTabPumps(t, a) |
| 349 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 350 | |
| 351 | done := make(chan struct{}) |
| 352 | go func() { a.resumeRemoteTabSession(meta.ID, "saved"); close(done) }() |
| 353 | select { |
| 354 | case <-fs.sessionsStarted: |
| 355 | case <-time.After(time.Second): |
| 356 | t.Fatal("saved-session lookup did not start") |
| 357 | } |
| 358 | a.suspendRemoteTabPumps("box", "reconnecting", "") |
| 359 | close(fs.sessionsRelease) |
| 360 | select { |
| 361 | case <-done: |
| 362 | case <-time.After(time.Second): |
| 363 | t.Fatal("saved-session lookup did not finish") |
| 364 | } |
| 365 | a.remoteTabMu.Lock() |
| 366 | state := a.remoteTabs[meta.ID].state |
| 367 | a.remoteTabMu.Unlock() |
| 368 | if state != "reconnecting" { |
| 369 | t.Fatalf("stale session lookup changed state to %q", state) |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | func TestRemoteTabReplacementServeReentersLearnedSessionBeforeReady(t *testing.T) { |
| 374 | oldServe := newFakeServe(t, "s3cret", nil) |
| 375 | newServe := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "generated", Path: "/generated.jsonl", Title: "Generated"}}) |
| 376 | kernel := &fakeRemoteKernel{ |
| 377 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 378 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: oldServe.server.URL, InstanceID: "serve-old"}, ensureToken: "s3cret", |
| 379 | } |
| 380 | seedBridgeTestHost(t, "box") |
| 381 | a := &App{remoteRuntime: kernel} |
| 382 | cleanupRemoteTabPumps(t, a) |
| 383 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 384 | oldServe.mu.Lock() |
| 385 | oldServe.statusPayload = `{"running":false,"sessionName":"generated"}` |
| 386 | oldServe.mu.Unlock() |
| 387 | if _, err := a.RemoteTabStatus(meta.ID); err != nil { |
| 388 | t.Fatal(err) |
| 389 | } |
| 390 | |
| 391 | a.remoteTabMu.Lock() |
| 392 | tab := a.remoteTabs[meta.ID] |
| 393 | if tab.cancel != nil { |
| 394 | tab.cancel() |
| 395 | } |
| 396 | tab.gen++ |
| 397 | tab.cancel, tab.client, tab.base, tab.token = nil, nil, "", "" |
| 398 | tab.state = "reconnecting" |
| 399 | // Restored tabs have no handshake metadata; stale capabilities from an |
| 400 | // earlier service must also be replaced, not merged into the new binding. |
| 401 | tab.capabilities = map[string]bool{"retired-capability": true} |
| 402 | a.remoteTabMu.Unlock() |
| 403 | kernel.ensureView = RemoteServerView{HostID: "box", State: "ready", LocalURL: newServe.server.URL, InstanceID: "serve-new"} |
| 404 | |
| 405 | if !a.reattachRemoteTabOnce(meta.ID) { |
| 406 | t.Fatal("replacement serve reattach failed") |
| 407 | } |
| 408 | _, resumed, _ := newServe.snapshot() |
| 409 | if resumed != "/generated.jsonl" { |
| 410 | t.Fatalf("replacement serve resumed %q, want /generated.jsonl", resumed) |
| 411 | } |
| 412 | a.remoteTabMu.Lock() |
| 413 | state, instanceID := a.remoteTabs[meta.ID].state, a.remoteTabs[meta.ID].session.instanceID |
| 414 | a.remoteTabMu.Unlock() |
| 415 | if state != "ready" || instanceID != "serve-new" { |
| 416 | t.Fatalf("reattached state/instance = %q/%q", state, instanceID) |
| 417 | } |
| 418 | if err := a.requireRemoteExecutionProtocol(meta.ID); err != nil { |
| 419 | t.Fatalf("reconnected service lost its execution capabilities: %v", err) |
| 420 | } |
| 421 | a.remoteTabMu.Lock() |
| 422 | staleCapability := a.remoteTabs[meta.ID].capabilities["retired-capability"] |
| 423 | a.remoteTabMu.Unlock() |
| 424 | if staleCapability { |
| 425 | t.Fatal("reconnect retained a capability absent from the new handshake") |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | func TestRemoteTabServeDownRetryPreservesNamedSession(t *testing.T) { |
| 430 | fs := newFakeServe(t, "s3cret", []serveSessionEntry{{Name: "saved", Path: "/saved.jsonl", Title: "Saved"}}) |
| 431 | kernel := &fakeRemoteKernel{ |
| 432 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 433 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 434 | } |
| 435 | seedBridgeTestHost(t, "box") |
| 436 | a := &App{remoteRuntime: kernel} |
| 437 | cleanupRemoteTabPumps(t, a) |
| 438 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{SessionName: "saved"}) |
| 439 | a.parkRemoteTabsForServer("box", "~/app", "serve_down", "stopped") |
| 440 | if _, err := a.OpenRemoteProjectTab("box", "~/app", RemoteTabOpenOptions{}); err != nil { |
| 441 | t.Fatal(err) |
| 442 | } |
| 443 | waitForTabState(t, a, meta.ID, "ready") |
| 444 | _, resumed, _ := fs.snapshot() |
| 445 | if resumed != "/saved.jsonl" { |
| 446 | t.Fatalf("retry resumed %q, want the parked named session", resumed) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | func TestRemoteSnapshotRejectsChangedGeneration(t *testing.T) { |
| 451 | fs := newFakeServe(t, "s3cret", nil) |
| 452 | kernel := &fakeRemoteKernel{ |
| 453 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 454 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 455 | } |
| 456 | seedBridgeTestHost(t, "box") |
| 457 | a := &App{remoteRuntime: kernel} |
| 458 | cleanupRemoteTabPumps(t, a) |
| 459 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 460 | fs.mu.Lock() |
| 461 | fs.historyStarted = make(chan struct{}, 1) |
| 462 | fs.historyRelease = make(chan struct{}) |
| 463 | started, release := fs.historyStarted, fs.historyRelease |
| 464 | fs.mu.Unlock() |
| 465 | errCh := make(chan error, 1) |
| 466 | go func() { |
| 467 | _, err := a.RemoteTabSnapshot(meta.ID) |
| 468 | errCh <- err |
| 469 | }() |
| 470 | <-started |
| 471 | a.suspendRemoteTabPumps("box", "reconnecting", "") |
| 472 | close(release) |
| 473 | if err := <-errCh; err == nil || !strings.Contains(err.Error(), "changed while loading snapshot") { |
| 474 | t.Fatalf("snapshot error = %v, want generation fence", err) |
| 475 | } |
| 476 | } |
| 477 | |
| 478 | func TestRemoteStopAndCloseCancelsBeforeRemovingTab(t *testing.T) { |
| 479 | fs := newFakeServe(t, "s3cret", nil) |
| 480 | fs.mu.Lock() |
| 481 | fs.statusPayload = `{"running":true,"pendingPrompt":false,"backgroundJobs":1,"cancellable":true,"jobs":[{"id":"job-remote","kind":"task","label":"verify","status":"running","startedAt":1}]}` |
| 482 | fs.statusAfterCancel = `{"running":false,"pendingPrompt":false,"backgroundJobs":0,"cancellable":false}` |
| 483 | fs.mu.Unlock() |
| 484 | kernel := &fakeRemoteKernel{ |
| 485 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 486 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: fs.server.URL}, ensureToken: "s3cret", |
| 487 | } |
| 488 | seedBridgeTestHost(t, "box") |
| 489 | a := &App{remoteRuntime: kernel} |
| 490 | cleanupRemoteTabPumps(t, a) |
| 491 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 492 | work := a.ActiveWorkForTab(meta.ID) |
| 493 | if !work.Running || !work.Cancellable { |
| 494 | t.Fatalf("remote active work = %+v", work) |
| 495 | } |
| 496 | // The one-surface policy refuses to remove the sole visible surface. What |
| 497 | // "stop and close" promises regardless is the stop, so that is what this |
| 498 | // pins: both cancels land, and the tab stays because the close was refused. |
| 499 | err := a.CloseTabWithPolicy(meta.ID, "stop_and_close") |
| 500 | if err == nil || !strings.Contains(err.Error(), "cannot close the last tab") { |
| 501 | t.Fatalf("stop-and-close on the sole surface = %v, want the last-surface refusal", err) |
| 502 | } |
| 503 | if !slices.ContainsFunc(fs.recorded(), func(call string) bool { return strings.HasPrefix(call, "POST /cancel") }) { |
| 504 | t.Fatalf("stop-and-close did not cancel remote work: %v", fs.recorded()) |
| 505 | } |
| 506 | if !slices.Contains(fs.recorded(), `POST /jobs/cancel {"ids":["job-remote"]}`) { |
| 507 | t.Fatalf("stop-and-close did not cancel remote background jobs: %v", fs.recorded()) |
| 508 | } |
| 509 | a.remoteTabMu.Lock() |
| 510 | _, present := a.remoteTabs[meta.ID] |
| 511 | a.remoteTabMu.Unlock() |
| 512 | if !present { |
| 513 | t.Fatal("a refused stop-and-close must leave the tab registered") |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // The observed tunnel-drop failure: the stream dies mid-turn and the first |
| 518 | // EnsureServer calls race the SSH layer's own recovery. The reattach loop |
| 519 | // must keep retrying across that window instead of parking a healthy tab in |
| 520 | // serve_down after half a second. |
| 521 | func TestRemoteTabReattachRetriesThroughTransientEnsureServerFailure(t *testing.T) { |
| 522 | serve := newFakeServe(t, "s3cret", nil) |
| 523 | kernel := &fakeRemoteKernel{ |
| 524 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 525 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: serve.server.URL, InstanceID: "serve-1"}, ensureToken: "s3cret", |
| 526 | } |
| 527 | seedBridgeTestHost(t, "box") |
| 528 | a := &App{remoteRuntime: kernel} |
| 529 | cleanupRemoteTabPumps(t, a) |
| 530 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 531 | // The transient failures start after the tab is open: they stand in for |
| 532 | // the tunnel dropping mid-session, not for a broken bootstrap. |
| 533 | kernel.ensureErrs = []error{errEnsureHealing, errEnsureHealing} |
| 534 | |
| 535 | previousDelays := remoteTabReattachDelays |
| 536 | remoteTabReattachDelays = []time.Duration{time.Millisecond, time.Millisecond} |
| 537 | t.Cleanup(func() { remoteTabReattachDelays = previousDelays }) |
| 538 | |
| 539 | a.remoteTabMu.Lock() |
| 540 | tab := a.remoteTabs[meta.ID] |
| 541 | if tab.cancel != nil { |
| 542 | tab.cancel() |
| 543 | } |
| 544 | tab.gen++ |
| 545 | tab.cancel, tab.client, tab.base, tab.token = nil, nil, "", "" |
| 546 | tab.state = "reconnecting" |
| 547 | a.remoteTabMu.Unlock() |
| 548 | |
| 549 | a.reattachRemoteTab(meta.ID) |
| 550 | a.remoteTabMu.Lock() |
| 551 | state := a.remoteTabs[meta.ID].state |
| 552 | a.remoteTabMu.Unlock() |
| 553 | if state != "ready" { |
| 554 | t.Fatalf("transient EnsureServer failures parked the tab in %q", state) |
| 555 | } |
| 556 | if len(kernel.ensureErrs) != 0 || kernel.ensureCalls != 4 { |
| 557 | t.Fatalf("reattach did not retry through both transient failures: calls=%d remaining=%d", kernel.ensureCalls, len(kernel.ensureErrs)) |
| 558 | } |
| 559 | } |
| 560 | |
| 561 | // serve_down tabs parked by reattach exhaustion must revive when the host |
| 562 | // connection recovers — the tunnel healing is exactly what they were waiting |
| 563 | // for, and nothing else revisits them. |
| 564 | func TestResumeRemoteTabsRevivesServeDownTabs(t *testing.T) { |
| 565 | serve := newFakeServe(t, "s3cret", nil) |
| 566 | kernel := &fakeRemoteKernel{ |
| 567 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 568 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: serve.server.URL, InstanceID: "serve-1"}, ensureToken: "s3cret", |
| 569 | } |
| 570 | seedBridgeTestHost(t, "box") |
| 571 | a := &App{remoteRuntime: kernel} |
| 572 | cleanupRemoteTabPumps(t, a) |
| 573 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 574 | |
| 575 | previousDelays := remoteTabReattachDelays |
| 576 | remoteTabReattachDelays = nil |
| 577 | t.Cleanup(func() { remoteTabReattachDelays = previousDelays }) |
| 578 | |
| 579 | a.remoteTabMu.Lock() |
| 580 | tab := a.remoteTabs[meta.ID] |
| 581 | if tab.cancel != nil { |
| 582 | tab.cancel() |
| 583 | } |
| 584 | tab.gen++ |
| 585 | tab.cancel, tab.client, tab.base, tab.token = nil, nil, "", "" |
| 586 | tab.state = "serve_down" |
| 587 | tab.err = "Remote session reconnect failed. Retry to restart the server." |
| 588 | a.remoteTabMu.Unlock() |
| 589 | |
| 590 | a.resumeRemoteTabs("box") |
| 591 | deadline := time.Now().Add(2 * time.Second) |
| 592 | for { |
| 593 | a.remoteTabMu.Lock() |
| 594 | state := a.remoteTabs[meta.ID].state |
| 595 | a.remoteTabMu.Unlock() |
| 596 | if state == "ready" { |
| 597 | break |
| 598 | } |
| 599 | if time.Now().After(deadline) { |
| 600 | t.Fatalf("host recovery left the serve_down tab in %q", state) |
| 601 | } |
| 602 | time.Sleep(5 * time.Millisecond) |
| 603 | } |
| 604 | } |
| 605 | |
| 606 | // A pump whose /events connection is refused (tunnel just dropped, serve |
| 607 | // restarting) must route through the reattach loop rather than parking the |
| 608 | // tab in a terminal error state — HTTP sends still work at that point, so a |
| 609 | // stranded pump means replies silently never render. |
| 610 | func TestRemoteTabPumpConnectionFailureReattachesInsteadOfParking(t *testing.T) { |
| 611 | listener, err := net.Listen("tcp", "127.0.0.1:0") |
| 612 | if err != nil { |
| 613 | t.Fatal(err) |
| 614 | } |
| 615 | deadBase := "http://" + listener.Addr().String() |
| 616 | listener.Close() |
| 617 | |
| 618 | serve := newFakeServe(t, "s3cret", nil) |
| 619 | kernel := &fakeRemoteKernel{ |
| 620 | statuses: []RemoteConnectionStatusView{{HostID: "box", State: "connected"}}, |
| 621 | ensureView: RemoteServerView{HostID: "box", State: "ready", LocalURL: serve.server.URL, InstanceID: "serve-1"}, ensureToken: "s3cret", |
| 622 | } |
| 623 | seedBridgeTestHost(t, "box") |
| 624 | a := &App{remoteRuntime: kernel} |
| 625 | cleanupRemoteTabPumps(t, a) |
| 626 | meta := openReadyRemoteTab(t, a, RemoteTabOpenOptions{NewSession: true}) |
| 627 | |
| 628 | previousDelays := remoteTabReattachDelays |
| 629 | remoteTabReattachDelays = nil |
| 630 | t.Cleanup(func() { remoteTabReattachDelays = previousDelays }) |
| 631 | |
| 632 | a.remoteTabMu.Lock() |
| 633 | tab := a.remoteTabs[meta.ID] |
| 634 | if tab.cancel != nil { |
| 635 | tab.cancel() |
| 636 | } |
| 637 | pumpCtx, cancelPump := context.WithCancel(context.Background()) |
| 638 | tab.gen++ |
| 639 | tab.cancel = cancelPump |
| 640 | tab.client = serve.server.Client() |
| 641 | tab.base = deadBase |
| 642 | tab.state = "connecting" |
| 643 | gen := tab.gen |
| 644 | a.remoteTabMu.Unlock() |
| 645 | |
| 646 | opened := make(chan error, 1) |
| 647 | go a.remoteTabPump(pumpCtx, meta.ID, gen, opened) |
| 648 | if err := <-opened; err == nil { |
| 649 | t.Fatal("the refused stream should be reported to the opener") |
| 650 | } |
| 651 | deadline := time.Now().Add(2 * time.Second) |
| 652 | for { |
| 653 | a.remoteTabMu.Lock() |
| 654 | state := a.remoteTabs[meta.ID].state |
| 655 | a.remoteTabMu.Unlock() |
| 656 | if state == "ready" { |
| 657 | break |
| 658 | } |
| 659 | if state == "error" { |
| 660 | t.Fatal("a transiently refused stream parked the tab in error") |
| 661 | } |
| 662 | if time.Now().After(deadline) { |
| 663 | t.Fatalf("reattach loop did not recover the refused stream, state=%q", state) |
| 664 | } |
| 665 | time.Sleep(5 * time.Millisecond) |
| 666 | } |
| 667 | } |
| 668 |