| 1 | //! Process-wide retry-state surface (#499). |
| 2 | //! |
| 3 | //! Read-side caveat (0.9.4): the renderer this module was written for was |
| 4 | //! the legacy footer's retry banner, which went with `FooterWidget`. The |
| 5 | //! *producer* — `client::send_with_retry` — is still live and still records |
| 6 | //! every retry, and `client`'s own tests read it back through [`snapshot`]. |
| 7 | //! The read surface below therefore carries `#[allow(dead_code)]` rather |
| 8 | //! than being deleted: removing it would mean changing `start`/`failed`'s |
| 9 | //! signatures at their live call sites in `client.rs`. Give the banner a |
| 10 | //! renderer, or delete the producer too — but not half of it. |
| 11 | //! |
| 12 | //! The HTTP retry path in `client::send_with_retry` already times its |
| 13 | //! waits and knows the error category. This module gives the TUI a way |
| 14 | //! to observe that state — `start`, `succeeded`, and `failed` flip a |
| 15 | //! global `RetryState` that the footer / status panel reads each frame. |
| 16 | //! |
| 17 | //! Why a process-wide global: the user-facing TUI runs as one engine |
| 18 | //! per process, and the only retry state we want to surface is the one |
| 19 | //! the user is staring at. Sub-agent retries in background tasks |
| 20 | //! deliberately do **not** light up the foreground banner — they're |
| 21 | //! supposed to be invisible. If a future feature ever needs per-engine |
| 22 | //! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the |
| 23 | //! `EngineHandle`; the public API stays the same. |
| 24 | |
| 25 | use std::sync::{Mutex, OnceLock}; |
| 26 | use std::time::{Duration, Instant}; |
| 27 | |
| 28 | /// One in-flight retry attempt. `deadline` is the wall-clock time the |
| 29 | /// next request will fire — the UI subtracts `Instant::now()` from it |
| 30 | /// to render a live countdown. |
| 31 | #[derive(Debug, Clone)] |
| 32 | #[allow(dead_code)] // written by client::send_with_retry; see the read-side caveat above |
| 33 | pub struct RetryBanner { |
| 34 | /// 1-indexed retry attempt number (the first retry is attempt 1). |
| 35 | pub attempt: u32, |
| 36 | /// Time at which the next request will be sent. |
| 37 | pub deadline: Instant, |
| 38 | /// Short human-readable reason ("rate limited", "server error", …). |
| 39 | pub reason: String, |
| 40 | } |
| 41 | |
| 42 | /// Snapshot of the retry surface for the UI to render. |
| 43 | #[derive(Debug, Clone, Default)] |
| 44 | pub enum RetryState { |
| 45 | /// No retry in flight. Banner hidden. |
| 46 | #[default] |
| 47 | Idle, |
| 48 | /// A request is sleeping before retrying. Show countdown banner. |
| 49 | Active(#[allow(dead_code)] RetryBanner), |
| 50 | /// All retries exhausted; show failure row until the next turn |
| 51 | /// starts. `since` records when the row was set so a future polish |
| 52 | /// pass can age it out automatically; today the engine clears it on |
| 53 | /// `TurnStarted`. |
| 54 | Failed { |
| 55 | #[allow(dead_code)] |
| 56 | reason: String, |
| 57 | #[allow(dead_code)] |
| 58 | since: Instant, |
| 59 | }, |
| 60 | } |
| 61 | |
| 62 | impl RetryState { |
| 63 | /// Wall-clock seconds remaining on the active banner, or `None` if |
| 64 | /// not active. Saturates at zero — the renderer should treat any |
| 65 | /// negative remaining as "firing now". |
| 66 | #[must_use] |
| 67 | #[allow(dead_code)] // no renderer since the legacy footer banner went |
| 68 | pub fn seconds_remaining(&self) -> Option<u64> { |
| 69 | match self { |
| 70 | Self::Active(banner) => Some( |
| 71 | banner |
| 72 | .deadline |
| 73 | .saturating_duration_since(Instant::now()) |
| 74 | .as_secs(), |
| 75 | ), |
| 76 | _ => None, |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | /// Whether the failure row should still be shown. Mirrors the |
| 81 | /// "until next turn" rule in the issue spec; the engine clears it |
| 82 | /// explicitly via [`clear`] on `TurnStarted`. |
| 83 | #[cfg(test)] |
| 84 | #[must_use] |
| 85 | pub fn is_failed(&self) -> bool { |
| 86 | matches!(self, Self::Failed { .. }) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | /// Lazy-init the cell on first read so callers don't have to initialize |
| 91 | /// process-wide state at boot. |
| 92 | #[cfg(not(test))] |
| 93 | fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R { |
| 94 | static STATE: OnceLock<Mutex<RetryState>> = OnceLock::new(); |
| 95 | let mut state = STATE |
| 96 | .get_or_init(|| Mutex::new(RetryState::Idle)) |
| 97 | .lock() |
| 98 | .unwrap_or_else(|error| error.into_inner()); |
| 99 | f(&mut state) |
| 100 | } |
| 101 | |
| 102 | #[cfg(not(test))] |
| 103 | fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R { |
| 104 | static STATE: OnceLock<Mutex<Option<Instant>>> = OnceLock::new(); |
| 105 | let mut state = STATE |
| 106 | .get_or_init(|| Mutex::new(None)) |
| 107 | .lock() |
| 108 | .unwrap_or_else(|error| error.into_inner()); |
| 109 | f(&mut state) |
| 110 | } |
| 111 | |
| 112 | /// Under test, this state is per-thread. |
| 113 | /// |
| 114 | /// Production has exactly one foreground engine per process, so a global is the |
| 115 | /// right model there. The test harness does not: retry state is written as a |
| 116 | /// side effect of *any* client request, by production code that has no test |
| 117 | /// guard to take, so a real request in one test could publish a banner or a |
| 118 | /// provider pause into another test's assertions. Scoping by thread removes the |
| 119 | /// race at its source rather than asking every future test that happens to |
| 120 | /// perform HTTP to remember a lock. |
| 121 | #[cfg(test)] |
| 122 | fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R { |
| 123 | #[allow(clippy::type_complexity)] |
| 124 | static STATE: OnceLock<Mutex<std::collections::HashMap<std::thread::ThreadId, RetryState>>> = |
| 125 | OnceLock::new(); |
| 126 | let mut by_thread = STATE |
| 127 | .get_or_init(|| Mutex::new(std::collections::HashMap::new())) |
| 128 | .lock() |
| 129 | .unwrap_or_else(|error| error.into_inner()); |
| 130 | f(by_thread |
| 131 | .entry(std::thread::current().id()) |
| 132 | .or_insert(RetryState::Idle)) |
| 133 | } |
| 134 | |
| 135 | #[cfg(test)] |
| 136 | fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R { |
| 137 | #[allow(clippy::type_complexity)] |
| 138 | static STATE: OnceLock< |
| 139 | Mutex<std::collections::HashMap<std::thread::ThreadId, Option<Instant>>>, |
| 140 | > = OnceLock::new(); |
| 141 | let mut by_thread = STATE |
| 142 | .get_or_init(|| Mutex::new(std::collections::HashMap::new())) |
| 143 | .lock() |
| 144 | .unwrap_or_else(|error| error.into_inner()); |
| 145 | f(by_thread.entry(std::thread::current().id()).or_default()) |
| 146 | } |
| 147 | |
| 148 | /// Public read snapshot for renderers. |
| 149 | #[must_use] |
| 150 | #[allow(dead_code)] // read by client.rs's retry tests; no production renderer today |
| 151 | pub fn snapshot() -> RetryState { |
| 152 | with_state(|state| state.clone()) |
| 153 | } |
| 154 | |
| 155 | /// Extend the provider-wide rate-limit pause window. This is separate from |
| 156 | /// the footer banner so one successful concurrent request cannot clear another |
| 157 | /// request's active `Retry-After` window. |
| 158 | pub fn note_rate_limit(delay: Duration) { |
| 159 | let deadline = Instant::now() + delay; |
| 160 | with_rate_limit(|current| { |
| 161 | if current.is_none_or(|existing| existing < deadline) { |
| 162 | *current = Some(deadline); |
| 163 | } |
| 164 | }); |
| 165 | } |
| 166 | |
| 167 | /// Remaining provider-wide rate-limit pause, if any. |
| 168 | #[must_use] |
| 169 | pub fn rate_limit_remaining() -> Option<Duration> { |
| 170 | let now = Instant::now(); |
| 171 | with_rate_limit(|current| match *current { |
| 172 | Some(deadline) if deadline > now => Some(deadline.duration_since(now)), |
| 173 | Some(_) => { |
| 174 | *current = None; |
| 175 | None |
| 176 | } |
| 177 | None => None, |
| 178 | }) |
| 179 | } |
| 180 | |
| 181 | /// Mark an in-flight retry. `attempt` is the number of the *upcoming* |
| 182 | /// retry (1 for the first); `delay` is how long the client will sleep |
| 183 | /// before firing. |
| 184 | pub fn start(attempt: u32, delay: Duration, reason: impl Into<String>) { |
| 185 | let banner = RetryBanner { |
| 186 | attempt, |
| 187 | deadline: Instant::now() + delay, |
| 188 | reason: reason.into(), |
| 189 | }; |
| 190 | with_state(|state| *state = RetryState::Active(banner)); |
| 191 | } |
| 192 | |
| 193 | /// Mark the retry chain as having succeeded. Hides the banner. |
| 194 | pub fn succeeded() { |
| 195 | with_state(|state| *state = RetryState::Idle); |
| 196 | } |
| 197 | |
| 198 | /// Mark the retry chain as having exhausted retries. The renderer keeps |
| 199 | /// the failure row until [`clear`] (typically called on `TurnStarted`). |
| 200 | pub fn failed(reason: impl Into<String>) { |
| 201 | with_state(|state| { |
| 202 | *state = RetryState::Failed { |
| 203 | reason: reason.into(), |
| 204 | since: Instant::now(), |
| 205 | }; |
| 206 | }); |
| 207 | } |
| 208 | |
| 209 | /// Reset to idle. Called on `TurnStarted` so the previous turn's |
| 210 | /// failure row doesn't bleed into the next turn. |
| 211 | pub fn clear() { |
| 212 | with_state(|state| *state = RetryState::Idle); |
| 213 | } |
| 214 | |
| 215 | #[cfg(test)] |
| 216 | pub fn clear_rate_limit() { |
| 217 | with_rate_limit(|current| *current = None); |
| 218 | } |
| 219 | |
| 220 | /// Test helper: serialize tests that touch the global state so cargo's |
| 221 | /// parallel runner can't observe a torn read. The guard is exported so |
| 222 | /// tests in *other* modules (e.g. footer rendering tests) can hold the |
| 223 | /// same lock as the ones in `retry_status::tests`. |
| 224 | #[cfg(test)] |
| 225 | pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { |
| 226 | static GUARD: Mutex<()> = Mutex::new(()); |
| 227 | GUARD.lock().unwrap_or_else(|e| e.into_inner()) |
| 228 | } |
| 229 | |
| 230 | #[cfg(test)] |
| 231 | mod tests { |
| 232 | use super::*; |
| 233 | |
| 234 | /// Acquire the cross-module test guard from [`super::test_guard`] and |
| 235 | /// reset state to `Idle` before yielding to the test body. |
| 236 | fn setup() -> std::sync::MutexGuard<'static, ()> { |
| 237 | let g = test_guard(); |
| 238 | clear(); |
| 239 | clear_rate_limit(); |
| 240 | g |
| 241 | } |
| 242 | |
| 243 | #[test] |
| 244 | fn idle_by_default_after_clear() { |
| 245 | let _g = setup(); |
| 246 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 247 | assert_eq!(snapshot().seconds_remaining(), None); |
| 248 | } |
| 249 | |
| 250 | #[test] |
| 251 | fn start_then_succeeded_returns_to_idle() { |
| 252 | let _g = setup(); |
| 253 | start(1, Duration::from_secs(5), "rate limited"); |
| 254 | let s = snapshot(); |
| 255 | assert!(matches!(s, RetryState::Active(_))); |
| 256 | let remaining = s.seconds_remaining().unwrap(); |
| 257 | assert!(remaining <= 5, "{remaining}"); |
| 258 | succeeded(); |
| 259 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 260 | } |
| 261 | |
| 262 | #[test] |
| 263 | fn failed_persists_until_clear() { |
| 264 | let _g = setup(); |
| 265 | failed("upstream 500"); |
| 266 | let s = snapshot(); |
| 267 | assert!(s.is_failed()); |
| 268 | if let RetryState::Failed { reason, .. } = s { |
| 269 | assert_eq!(reason, "upstream 500"); |
| 270 | } else { |
| 271 | panic!("expected Failed"); |
| 272 | } |
| 273 | clear(); |
| 274 | assert!(matches!(snapshot(), RetryState::Idle)); |
| 275 | } |
| 276 | |
| 277 | #[test] |
| 278 | fn deadline_in_past_yields_zero_remaining() { |
| 279 | let _g = setup(); |
| 280 | // Bypass `start` so we can plant a deadline already in the past. |
| 281 | with_state(|state| { |
| 282 | *state = RetryState::Active(RetryBanner { |
| 283 | attempt: 2, |
| 284 | deadline: Instant::now() - Duration::from_secs(1), |
| 285 | reason: "test".into(), |
| 286 | }); |
| 287 | }); |
| 288 | assert_eq!(snapshot().seconds_remaining(), Some(0)); |
| 289 | clear(); |
| 290 | } |
| 291 | |
| 292 | #[test] |
| 293 | fn rate_limit_deadline_survives_banner_clear() { |
| 294 | let _g = setup(); |
| 295 | note_rate_limit(Duration::from_secs(5)); |
| 296 | start(1, Duration::from_secs(5), "rate limited"); |
| 297 | succeeded(); |
| 298 | assert!( |
| 299 | rate_limit_remaining().is_some(), |
| 300 | "provider-wide rate limit pause must not be cleared by an unrelated success" |
| 301 | ); |
| 302 | clear_rate_limit(); |
| 303 | } |
| 304 | } |
| 305 |