| 1 | //! Process-wide retry-state surface (#499). |
| 2 | //! |
| 3 | //! Read-side caveat (0.9.4): the renderer this module was written for was |
| 4 | //! the legacy footer's retry banner, which went with `FooterWidget`. The |
| 5 | //! *producer* — `client::send_with_retry` — is still live and still records |
| 6 | //! every retry, and `client`'s own tests read it back through [`snapshot`]. |
| 7 | //! The read surface (`snapshot`, the countdown, the banner fields) is |
| 8 | //! therefore test-gated (`#[cfg(test)]` where possible, |
| 9 | //! `cfg_attr(not(test))` allows where prod constructs); a renderer |
| 10 | //! restores it by dropping the gates. Give the banner a renderer, or |
| 11 | //! delete the producer too — but not half of it. |
| 12 | //! |
| 13 | //! The HTTP retry path in `client::send_with_retry` already times its |
| 14 | //! waits and knows the error category. This module gives the TUI a way |
| 15 | //! to observe that state — `start`, `succeeded`, and `failed` flip a |
| 16 | //! global `RetryState` that the footer / status panel reads each frame. |
| 17 | //! |
| 18 | //! Why a process-wide global: the user-facing TUI runs as one engine |
| 19 | //! per process, and the only retry state we want to surface is the one |
| 20 | //! the user is staring at. Sub-agent retries in background tasks |
| 21 | //! deliberately do **not** light up the foreground banner — they're |
| 22 | //! supposed to be invisible. If a future feature ever needs per-engine |
| 23 | //! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the |
| 24 | //! `EngineHandle`; the public API stays the same. |
| 25 | |
| 26 | use std::sync::{Mutex, OnceLock}; |
| 27 | use std::time::{Duration, Instant}; |
| 28 | |
| 29 | /// One in-flight retry attempt. `deadline` is the wall-clock time the |
| 30 | /// next request will fire — the UI subtracts `Instant::now()` from it |
| 31 | /// to render a live countdown. |
| 32 | #[derive(Debug, Clone)] |
| 33 | #[cfg_attr(not(test), allow(dead_code))] |
| 34 | pub struct RetryBanner { |
| 35 | /// 1-indexed retry attempt number (the first retry is attempt 1). |
| 36 | pub attempt: u32, |
| 37 | /// Time at which the next request will be sent. |
| 38 | pub deadline: Instant, |
| 39 | /// Short human-readable reason ("rate limited", "server error", …). |
| 40 | pub reason: String, |
| 41 | } |
| 42 | |
| 43 | /// Snapshot of the retry surface for the UI to render. |
| 44 | #[derive(Debug, Clone, Default)] |
| 45 | pub enum RetryState { |
| 46 | /// No retry in flight. Banner hidden. |
| 47 | #[default] |
| 48 | Idle, |
| 49 | /// A request is sleeping before retrying. Show countdown banner. |
| 50 | Active(#[cfg_attr(not(test), allow(dead_code))] RetryBanner), |
| 51 | /// All retries exhausted; show failure row until the next turn |
| 52 | /// starts. `since` records when the row was set so a future polish |
| 53 | /// pass can age it out automatically; today the engine clears it on |
| 54 | /// `TurnStarted`. |
| 55 | Failed { |
| 56 | #[cfg_attr(not(test), expect(dead_code))] |
| 57 | reason: String, |
| 58 | #[expect(dead_code)] |
| 59 | since: Instant, |
| 60 | }, |
| 61 | } |
| 62 | |
| 63 | impl RetryState { |
| 64 | /// Wall-clock seconds remaining on the active banner, or `None` if |
| 65 | /// not active. Saturates at zero — the renderer should treat any |
| 66 | /// negative remaining as "firing now". |
| 67 | #[cfg(test)] |
| 68 | #[must_use] |
| 69 | pub fn seconds_remaining(&self) -> Option<u64> { |
| 70 | match self { |
| 71 | Self::Active(banner) => Some( |
| 72 | banner |
| 73 | .deadline |
| 74 | .saturating_duration_since(Instant::now()) |
| 75 | .as_secs(), |
| 76 | ), |
| 77 | _ => None, |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | /// Whether the failure row should still be shown. Mirrors the |
| 82 | /// "until next turn" rule in the issue spec; the engine clears it |
| 83 | /// explicitly via [`clear`] on `TurnStarted`. |
| 84 | #[cfg(test)] |
| 85 | #[must_use] |
| 86 | pub fn is_failed(&self) -> bool { |
| 87 | matches!(self, Self::Failed { .. }) |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | /// Lazy-init the cell on first read so callers don't have to initialize |
| 92 | /// process-wide state at boot. |
| 93 | #[cfg(not(test))] |
| 94 | fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R { |
| 95 | static STATE: OnceLock<Mutex<RetryState>> = OnceLock::new(); |
| 96 | let mut state = STATE |
| 97 | .get_or_init(|| Mutex::new(RetryState::Idle)) |
| 98 | .lock() |
| 99 | .unwrap_or_else(|error| error.into_inner()); |
| 100 | f(&mut state) |
| 101 | } |
| 102 | |
| 103 | #[cfg(not(test))] |
| 104 | fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R { |
| 105 | static STATE: OnceLock<Mutex<Option<Instant>>> = OnceLock::new(); |
| 106 | let mut state = STATE |
| 107 | .get_or_init(|| Mutex::new(None)) |
| 108 | .lock() |
| 109 | .unwrap_or_else(|error| error.into_inner()); |
| 110 | f(&mut state) |
| 111 | } |
| 112 | |
| 113 | /// Under test, this state is per-thread. |
| 114 | /// |
| 115 | /// Production has exactly one foreground engine per process, so a global is the |
| 116 | /// right model there. The test harness does not: retry state is written as a |
| 117 | /// side effect of *any* client request, by production code that has no test |
| 118 | /// guard to take, so a real request in one test could publish a banner or a |
| 119 | /// provider pause into another test's assertions. Scoping by thread removes the |
| 120 | /// race at its source rather than asking every future test that happens to |
| 121 | /// perform HTTP to remember a lock. |
| 122 | #[cfg(test)] |
| 123 | fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R { |
| 124 | #[allow(clippy::type_complexity)] |
| 125 | static STATE: OnceLock<Mutex<std::collections::HashMap<std::thread::ThreadId, RetryState>>> = |
| 126 | OnceLock::new(); |
| 127 | let mut by_thread = STATE |
| 128 | .get_or_init(|| Mutex::new(std::collections::HashMap::new())) |
| 129 | .lock() |
| 130 | .unwrap_or_else(|error| error.into_inner()); |
| 131 | f(by_thread |
| 132 | .entry(std::thread::current().id()) |
| 133 | .or_insert(RetryState::Idle)) |
| 134 | } |
| 135 | |
| 136 | #[cfg(test)] |
| 137 | fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R { |
| 138 | #[allow(clippy::type_complexity)] |
| 139 | static STATE: OnceLock< |
| 140 | Mutex<std::collections::HashMap<std::thread::ThreadId, Option<Instant>>>, |
| 141 | > = OnceLock::new(); |
| 142 | let mut by_thread = STATE |
| 143 | .get_or_init(|| Mutex::new(std::collections::HashMap::new())) |
| 144 | .lock() |
| 145 | .unwrap_or_else(|error| error.into_inner()); |
| 146 | f(by_thread.entry(std::thread::current().id()).or_default()) |
| 147 | } |
| 148 | |
| 149 | /// Read snapshot for renderers. No production renderer exists since the |
| 150 | /// legacy footer went away; `client` retry tests are the only readers. |
| 151 | #[cfg(test)] |
| 152 | #[must_use] |
| 153 | pub fn snapshot() -> RetryState { |
| 154 | with_state(|state| state.clone()) |
| 155 | } |
| 156 | |
| 157 | /// Extend the provider-wide rate-limit pause window. This is separate from |
| 158 | /// the footer banner so one successful concurrent request cannot clear another |
| 159 | /// request's active `Retry-After` window. |
| 160 | pub fn note_rate_limit(delay: Duration) { |
| 161 | let deadline = Instant::now() + delay; |
| 162 | with_rate_limit(|current| { |
| 163 | if current.is_none_or(|existing| existing < deadline) { |
| 164 | *current = Some(deadline); |
| 165 | } |
| 166 | }); |
| 167 | } |
| 168 | |
| 169 | /// Remaining provider-wide rate-limit pause, if any. |
| 170 | #[must_use] |
| 171 | pub fn rate_limit_remaining() -> Option<Duration> { |
| 172 | let now = Instant::now(); |
| 173 | with_rate_limit(|current| match *current { |
| 174 | Some(deadline) if deadline > now => Some(deadline.duration_since(now)), |
| 175 | Some(_) => { |
| 176 | *current = None; |
| 177 | None |
| 178 | } |
| 179 | None => None, |
| 180 | }) |
| 181 | } |
| 182 | |
| 183 | /// Mark an in-flight retry. `attempt` is the number of the *upcoming* |
| 184 | /// retry (1 for the first); `delay` is how long the client will sleep |
| 185 | /// before firing. |
| 186 | pub fn start(attempt: u32, delay: Duration, reason: impl Into<String>) { |
| 187 | let banner = RetryBanner { |
| 188 | attempt, |
| 189 | deadline: Instant::now() + delay, |
| 190 | reason: reason.into(), |
| 191 | }; |
| 192 | with_state(|state| *state = RetryState::Active(banner)); |
| 193 | } |
| 194 | |
| 195 | /// Mark the retry chain as having succeeded. Hides the banner. |
| 196 | pub fn succeeded() { |
| 197 | with_state(|state| *state = RetryState::Idle); |
| 198 | } |
| 199 | |
| 200 | /// Mark the retry chain as having exhausted retries. The renderer keeps |
| 201 | /// the failure row until [`clear`] (typically called on `TurnStarted`). |
| 202 | pub fn failed(reason: impl Into<String>) { |
| 203 | with_state(|state| { |
| 204 | *state = RetryState::Failed { |
| 205 | reason: reason.into(), |
| 206 | since: Instant::now(), |
| 207 | }; |
| 208 | }); |
| 209 | } |
| 210 | |
| 211 | /// Reset to idle. Called on `TurnStarted` so the previous turn's |
| 212 | /// failure row doesn't bleed into the next turn. |
| 213 | pub fn clear() { |
| 214 | with_state(|state| *state = RetryState::Idle); |
| 215 | } |
| 216 | |
| 217 | #[cfg(test)] |
| 218 | pub fn clear_rate_limit() { |
| 219 | with_rate_limit(|current| *current = None); |
| 220 | } |
| 221 | |
| 222 | /// Test helper: serialize tests that touch the global state so cargo's |
| 223 | /// parallel runner can't observe a torn read. The guard is exported so |
| 224 | /// tests in *other* modules (e.g. footer rendering tests) can hold the |
| 225 | /// same lock as the ones in `retry_status::tests`. |
| 226 | #[cfg(test)] |
| 227 | pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { |
| 228 | static GUARD: Mutex<()> = Mutex::new(()); |
| 229 | GUARD.lock().unwrap_or_else(|e| e.into_inner()) |
| 230 | } |
| 231 | |
| 232 | #[cfg(test)] |
| 233 | mod tests { |
| 234 | use super::*; |
| 235 | |
| 236 | /// Acquire the cross-module test guard from [`super::test_guard`] and |
| 237 | /// reset state to `Idle` before yielding to the test body. |
| 238 | fn setup() -> std::sync::MutexGuard<'static, ()> { |
| 239 | let g = test_guard(); |
| 240 | clear(); |
| 241 | clear_rate_limit(); |
| 242 | g |
| 243 | } |
| 244 | |
| 245 | #[test] |
| 246 | fn idle_by_default_after_clear() { |
| 247 | let _g = setup(); |
| 248 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 249 | assert_eq!(snapshot().seconds_remaining(), None); |
| 250 | } |
| 251 | |
| 252 | #[test] |
| 253 | fn start_then_succeeded_returns_to_idle() { |
| 254 | let _g = setup(); |
| 255 | start(1, Duration::from_secs(5), "rate limited"); |
| 256 | let s = snapshot(); |
| 257 | assert!(matches!(s, RetryState::Active(_))); |
| 258 | let remaining = s.seconds_remaining().unwrap(); |
| 259 | assert!(remaining <= 5, "{remaining}"); |
| 260 | succeeded(); |
| 261 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 262 | } |
| 263 | |
| 264 | #[test] |
| 265 | fn failed_persists_until_clear() { |
| 266 | let _g = setup(); |
| 267 | failed("upstream 500"); |
| 268 | let s = snapshot(); |
| 269 | assert!(s.is_failed()); |
| 270 | if let RetryState::Failed { reason, .. } = s { |
| 271 | assert_eq!(reason, "upstream 500"); |
| 272 | } else { |
| 273 | panic!("expected Failed"); |
| 274 | } |
| 275 | clear(); |
| 276 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 277 | } |
| 278 | |
| 279 | #[test] |
| 280 | fn deadline_in_past_yields_zero_remaining() { |
| 281 | let _g = setup(); |
| 282 | // Bypass `start` so we can plant a deadline already in the past. |
| 283 | with_state(|state| { |
| 284 | *state = RetryState::Active(RetryBanner { |
| 285 | attempt: 2, |
| 286 | deadline: Instant::now() - Duration::from_secs(1), |
| 287 | reason: "test".into(), |
| 288 | }); |
| 289 | }); |
| 290 | assert_eq!(snapshot().seconds_remaining(), Some(0)); |
| 291 | clear(); |
| 292 | } |
| 293 | |
| 294 | #[test] |
| 295 | fn rate_limit_deadline_survives_banner_clear() { |
| 296 | let _g = setup(); |
| 297 | note_rate_limit(Duration::from_secs(5)); |
| 298 | start(1, Duration::from_secs(5), "rate limited"); |
| 299 | succeeded(); |
| 300 | assert!( |
| 301 | rate_limit_remaining().is_some(), |
| 302 | "provider-wide rate limit pause must not be cleared by an unrelated success" |
| 303 | ); |
| 304 | clear_rate_limit(); |
| 305 | } |
| 306 | } |
| 307 |