| 1 | //! Unified guarded fetch pipeline for `fetch_url` and `web.run`. |
| 2 | |
| 3 | use std::collections::BTreeMap; |
| 4 | use std::sync::Arc; |
| 5 | use std::time::{Duration, Instant}; |
| 6 | #[cfg(not(test))] |
| 7 | use std::time::{SystemTime, UNIX_EPOCH}; |
| 8 | |
| 9 | use futures_util::StreamExt; |
| 10 | use serde::Serialize; |
| 11 | |
| 12 | use super::cache::{self, CachedFetch}; |
| 13 | use super::extract::is_js_shell_error; |
| 14 | use super::guard::{ |
| 15 | DnsPin, guarded_reqwest_client_builder, validate_fetch_target, validate_network_policy, |
| 16 | }; |
| 17 | use crate::features::Feature; |
| 18 | use crate::tools::spec::{ToolContext, ToolError}; |
| 19 | use crate::worker_profile::ShellPolicy; |
| 20 | |
| 21 | pub(crate) const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15); |
| 22 | pub(crate) const HARD_MAX_TIMEOUT: Duration = Duration::from_secs(60); |
| 23 | pub(crate) const DEFAULT_MAX_BYTES: usize = 1_000_000; |
| 24 | pub(crate) const HARD_MAX_BYTES: usize = 10 * 1024 * 1024; |
| 25 | const MAX_REDIRECTS: usize = 5; |
| 26 | const USER_AGENT: &str = concat!( |
| 27 | "Mozilla/5.0 (compatible; codewhale/", |
| 28 | env!("CARGO_PKG_VERSION"), |
| 29 | "; +https://github.com/Hmbown/CodeWhale)" |
| 30 | ); |
| 31 | |
| 32 | #[derive(Debug, Clone)] |
| 33 | pub(crate) struct FetchOptions { |
| 34 | pub(crate) timeout: Duration, |
| 35 | pub(crate) max_bytes: usize, |
| 36 | pub(crate) accept: &'static str, |
| 37 | } |
| 38 | |
| 39 | impl FetchOptions { |
| 40 | pub(crate) fn new(timeout: Duration, max_bytes: usize, accept: &'static str) -> Self { |
| 41 | Self { |
| 42 | timeout: timeout.min(HARD_MAX_TIMEOUT), |
| 43 | max_bytes: max_bytes.clamp(1, HARD_MAX_BYTES), |
| 44 | accept, |
| 45 | } |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | #[derive(Debug, Clone)] |
| 50 | pub(crate) struct FetchedPayload { |
| 51 | pub(crate) url: String, |
| 52 | pub(crate) status: u16, |
| 53 | pub(crate) headers: BTreeMap<String, String>, |
| 54 | pub(crate) content_type: String, |
| 55 | pub(crate) bytes: Arc<Vec<u8>>, |
| 56 | pub(crate) truncated: bool, |
| 57 | pub(crate) cache_hit: bool, |
| 58 | pub(crate) retries: usize, |
| 59 | pub(crate) redirects: usize, |
| 60 | } |
| 61 | |
| 62 | /// Whether one request may be answered from a cache, or must revalidate. |
| 63 | /// |
| 64 | /// `Revalidate` bypasses the session fetch cache *and* asks every intermediary |
| 65 | /// to revalidate. An edge cache can hold a prerendered variant while an origin |
| 66 | /// MISS serves the client-side shell, so the same URL alternates between |
| 67 | /// readable and unreadable depending on which variant answered (#5904). |
| 68 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 69 | enum CacheMode { |
| 70 | Default, |
| 71 | Revalidate, |
| 72 | } |
| 73 | |
| 74 | impl CacheMode { |
| 75 | const fn is_revalidate(self) -> bool { |
| 76 | matches!(self, Self::Revalidate) |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | /// Response headers that explain the cache state behind a 200. |
| 81 | /// |
| 82 | /// These are the four that distinguish "the edge served a prerendered page" |
| 83 | /// from "the origin served the JavaScript shell", so both the success and the |
| 84 | /// failure receipt carry whichever of them the response actually had. |
| 85 | const CACHE_STATE_HEADERS: [&str; 4] = [ |
| 86 | "age", |
| 87 | "cf-cache-status", |
| 88 | "x-nextjs-prerender", |
| 89 | "x-vercel-cache", |
| 90 | ]; |
| 91 | |
| 92 | /// One request inside a readable-fetch sequence, as it appears on the receipt. |
| 93 | #[derive(Debug, Clone, Serialize)] |
| 94 | pub(crate) struct FetchAttempt { |
| 95 | /// 1-based position in the sequence. |
| 96 | pub(crate) attempt: usize, |
| 97 | pub(crate) status: u16, |
| 98 | /// Whether the session fetch cache answered this attempt. |
| 99 | pub(crate) cache_hit: bool, |
| 100 | /// Whether this attempt sent `Cache-Control: no-cache` / `Pragma: no-cache` |
| 101 | /// and skipped the session cache. |
| 102 | pub(crate) cache_busted: bool, |
| 103 | /// Whether this attempt is the one that yielded a readable document. |
| 104 | pub(crate) produced_content: bool, |
| 105 | /// `age`, `cf-cache-status`, `x-nextjs-prerender`, `x-vercel-cache` — only |
| 106 | /// those the response actually carried. |
| 107 | #[serde(skip_serializing_if = "BTreeMap::is_empty")] |
| 108 | pub(crate) cache_headers: BTreeMap<String, String>, |
| 109 | } |
| 110 | |
| 111 | impl FetchAttempt { |
| 112 | fn record(payload: &FetchedPayload, attempt: usize, mode: CacheMode) -> Self { |
| 113 | Self { |
| 114 | attempt, |
| 115 | status: payload.status, |
| 116 | cache_hit: payload.cache_hit, |
| 117 | cache_busted: mode.is_revalidate(), |
| 118 | produced_content: false, |
| 119 | cache_headers: cache_state_headers(&payload.headers), |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | fn summarize(&self) -> String { |
| 124 | let mut facts = vec![format!("HTTP {}", self.status)]; |
| 125 | if self.cache_hit { |
| 126 | facts.push("session cache hit".to_string()); |
| 127 | } |
| 128 | for (name, value) in &self.cache_headers { |
| 129 | facts.push(format!("{name}={value}")); |
| 130 | } |
| 131 | let label = if self.cache_busted { |
| 132 | format!("attempt {} (Cache-Control: no-cache)", self.attempt) |
| 133 | } else { |
| 134 | format!("attempt {}", self.attempt) |
| 135 | }; |
| 136 | format!("{label}: {}", facts.join(", ")) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | fn cache_state_headers(headers: &BTreeMap<String, String>) -> BTreeMap<String, String> { |
| 141 | CACHE_STATE_HEADERS |
| 142 | .iter() |
| 143 | .filter_map(|name| { |
| 144 | headers |
| 145 | .get(*name) |
| 146 | .map(|value| ((*name).to_string(), value.clone())) |
| 147 | }) |
| 148 | .collect() |
| 149 | } |
| 150 | |
| 151 | /// The extraction step [`fetch_readable`] may run against either attempt. |
| 152 | /// |
| 153 | /// Spelled as an explicit boxed future over an *owned* payload rather than as |
| 154 | /// an `AsyncFn` over a borrowed one: the callers live inside |
| 155 | /// `async fn execute(&self, .., &ToolContext)` futures that must stay `Send`, |
| 156 | /// and a higher-ranked borrow of the payload would force the returned future |
| 157 | /// to outlive the tool context it reads. |
| 158 | pub(crate) type ExtractFuture<'a, T> = |
| 159 | std::pin::Pin<Box<dyn Future<Output = Result<T, ToolError>> + Send + 'a>>; |
| 160 | |
| 161 | /// A fetch that produced a readable document, plus the attempts it took. |
| 162 | #[derive(Debug)] |
| 163 | pub(crate) struct ReadableFetch<T> { |
| 164 | pub(crate) payload: FetchedPayload, |
| 165 | pub(crate) document: T, |
| 166 | pub(crate) attempts: Vec<FetchAttempt>, |
| 167 | } |
| 168 | |
| 169 | /// Fetch `url` and extract it, re-fetching once past every cache when a 2xx |
| 170 | /// response yields no readable content. |
| 171 | /// |
| 172 | /// This is the single place that turns the JS-shell case into either a second |
| 173 | /// chance or an error the model can act on. `extract` runs against the fetched |
| 174 | /// payload; only [`is_js_shell_error`] failures earn the second request, so |
| 175 | /// transport failures keep the existing single-retry behavior of one request. |
| 176 | pub(crate) async fn fetch_readable<'e, T, F>( |
| 177 | url: &str, |
| 178 | options: &FetchOptions, |
| 179 | context: &ToolContext, |
| 180 | tool_label: &str, |
| 181 | extract: F, |
| 182 | ) -> Result<ReadableFetch<T>, ToolError> |
| 183 | where |
| 184 | F: Fn(FetchedPayload) -> ExtractFuture<'e, T>, |
| 185 | { |
| 186 | fetch_readable_inner(url, options, context, tool_label, None, extract).await |
| 187 | } |
| 188 | |
| 189 | #[cfg(test)] |
| 190 | pub(crate) async fn fetch_readable_with_initial_pin<'e, T, F>( |
| 191 | url: &str, |
| 192 | options: &FetchOptions, |
| 193 | context: &ToolContext, |
| 194 | tool_label: &str, |
| 195 | initial_pin: DnsPin, |
| 196 | extract: F, |
| 197 | ) -> Result<ReadableFetch<T>, ToolError> |
| 198 | where |
| 199 | F: Fn(FetchedPayload) -> ExtractFuture<'e, T>, |
| 200 | { |
| 201 | fetch_readable_inner( |
| 202 | url, |
| 203 | options, |
| 204 | context, |
| 205 | tool_label, |
| 206 | Some(initial_pin), |
| 207 | extract, |
| 208 | ) |
| 209 | .await |
| 210 | } |
| 211 | |
| 212 | async fn fetch_readable_inner<'e, T, F>( |
| 213 | url: &str, |
| 214 | options: &FetchOptions, |
| 215 | context: &ToolContext, |
| 216 | tool_label: &str, |
| 217 | test_initial_pin: Option<DnsPin>, |
| 218 | extract: F, |
| 219 | ) -> Result<ReadableFetch<T>, ToolError> |
| 220 | where |
| 221 | F: Fn(FetchedPayload) -> ExtractFuture<'e, T>, |
| 222 | { |
| 223 | let mut attempts: Vec<FetchAttempt> = Vec::with_capacity(2); |
| 224 | for mode in [CacheMode::Default, CacheMode::Revalidate] { |
| 225 | let payload = fetch_inner( |
| 226 | url, |
| 227 | options, |
| 228 | context, |
| 229 | tool_label, |
| 230 | test_initial_pin.clone(), |
| 231 | mode, |
| 232 | ) |
| 233 | .await?; |
| 234 | let mut record = FetchAttempt::record(&payload, attempts.len() + 1, mode); |
| 235 | let final_url = payload.url.clone(); |
| 236 | match extract(payload.clone()).await { |
| 237 | Ok(document) => { |
| 238 | record.produced_content = true; |
| 239 | attempts.push(record); |
| 240 | return Ok(ReadableFetch { |
| 241 | payload, |
| 242 | document, |
| 243 | attempts, |
| 244 | }); |
| 245 | } |
| 246 | // A 2xx whose body held no readable content is the one failure a |
| 247 | // second request can fix: the first response may have been a |
| 248 | // cached client-side shell. |
| 249 | Err(error) |
| 250 | if is_js_shell_error(&error) |
| 251 | && (200..300).contains(&payload.status) |
| 252 | && mode == CacheMode::Default => |
| 253 | { |
| 254 | attempts.push(record); |
| 255 | } |
| 256 | Err(error) => { |
| 257 | attempts.push(record); |
| 258 | return Err(if is_js_shell_error(&error) { |
| 259 | js_shell_failure(&final_url, &attempts, context, tool_label) |
| 260 | } else { |
| 261 | error |
| 262 | }); |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | unreachable!("the revalidate pass either returns a document or an error"); |
| 267 | } |
| 268 | |
| 269 | /// The terminal JS-shell error, carrying the failure receipt and the recovery |
| 270 | /// the *calling role* actually owns. |
| 271 | fn js_shell_failure( |
| 272 | url: &str, |
| 273 | attempts: &[FetchAttempt], |
| 274 | context: &ToolContext, |
| 275 | tool_label: &str, |
| 276 | ) -> ToolError { |
| 277 | let receipt = attempts |
| 278 | .iter() |
| 279 | .map(FetchAttempt::summarize) |
| 280 | .collect::<Vec<_>>() |
| 281 | .join("; "); |
| 282 | ToolError::execution_failed(format!( |
| 283 | "{marker} {url} after {count} attempts, the second past every cache ({receipt}). The response parsed but held no readable body, which usually means the page renders its content with JavaScript. Recovery: {recovery}", |
| 284 | marker = super::extract::JS_SHELL_MARKER, |
| 285 | count = attempts.len(), |
| 286 | recovery = js_shell_recovery(context, tool_label), |
| 287 | )) |
| 288 | } |
| 289 | |
| 290 | /// Whether the `web.run` browse surface is reachable from this context. |
| 291 | /// |
| 292 | /// Both facts already exist: the web family is feature-gated, and a |
| 293 | /// network-denied Fleet worker carries `network_access: Some(false)` on the |
| 294 | /// authority envelope that also removes `web.run` from its registry |
| 295 | /// (`fleet::role::NETWORK_TOOL_DENYLIST`). Nothing new is registered here. |
| 296 | fn browser_surface_available(context: &ToolContext) -> bool { |
| 297 | context.features.enabled(Feature::WebSearch) && network_authorized(context) |
| 298 | } |
| 299 | |
| 300 | /// Whether this role could shell out to `curl` as a last resort. Read-only and |
| 301 | /// shell-less roles cannot: the read-only grammar rejects a network fetch. |
| 302 | fn shell_fallback_available(context: &ToolContext) -> bool { |
| 303 | context.shell_policy == ShellPolicy::Full && network_authorized(context) |
| 304 | } |
| 305 | |
| 306 | fn network_authorized(context: &ToolContext) -> bool { |
| 307 | context |
| 308 | .tool_authority |
| 309 | .as_deref() |
| 310 | .is_none_or(|authority| authority.network_access != Some(false)) |
| 311 | } |
| 312 | |
| 313 | fn js_shell_recovery(context: &ToolContext, tool_label: &str) -> String { |
| 314 | // `web.run` is itself the escalation, so it never names itself. |
| 315 | if tool_label != "web_run" && browser_surface_available(context) { |
| 316 | return "open this URL with the `web.run` browse surface (`web.run {\"open\": {\"url\": ...}}`), which requests it with a browser user-agent and a ten-megabyte budget and usually receives the prerendered variant.".to_string(); |
| 317 | } |
| 318 | let unavailable = if tool_label == "web_run" { |
| 319 | "this is already the `web.run` browse surface, so there is no further web escalation." |
| 320 | } else { |
| 321 | "the `web.run` browse surface is not available to this role." |
| 322 | }; |
| 323 | if shell_fallback_available(context) { |
| 324 | format!("{unavailable} Fall back to a shell fetch (`curl -sSL`) or a rendering tool.") |
| 325 | } else { |
| 326 | format!( |
| 327 | "{unavailable} This role is read-only and cannot fall back to a shell fetch, so report this URL as unreadable rather than substituting another source." |
| 328 | ) |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | #[cfg(test)] |
| 333 | pub(crate) async fn fetch_with_initial_pin( |
| 334 | url: &str, |
| 335 | options: &FetchOptions, |
| 336 | context: &ToolContext, |
| 337 | tool_label: &str, |
| 338 | initial_pin: DnsPin, |
| 339 | ) -> Result<FetchedPayload, ToolError> { |
| 340 | fetch_inner( |
| 341 | url, |
| 342 | options, |
| 343 | context, |
| 344 | tool_label, |
| 345 | Some(initial_pin), |
| 346 | CacheMode::Default, |
| 347 | ) |
| 348 | .await |
| 349 | } |
| 350 | |
| 351 | async fn fetch_inner( |
| 352 | url: &str, |
| 353 | options: &FetchOptions, |
| 354 | context: &ToolContext, |
| 355 | tool_label: &str, |
| 356 | test_initial_pin: Option<DnsPin>, |
| 357 | cache_mode: CacheMode, |
| 358 | ) -> Result<FetchedPayload, ToolError> { |
| 359 | let initial_url = reqwest::Url::parse(url) |
| 360 | .map_err(|err| ToolError::invalid_input(format!("invalid URL: {err}")))?; |
| 361 | if !matches!(initial_url.scheme(), "http" | "https") { |
| 362 | return Err(ToolError::invalid_input( |
| 363 | "only http:// and https:// URLs are supported", |
| 364 | )); |
| 365 | } |
| 366 | |
| 367 | // Validation precedes cache lookup so a policy tightened during the |
| 368 | // session cannot be bypassed by a previously cached response. |
| 369 | let validated_initial_pin = match test_initial_pin { |
| 370 | Some(pin) => pin, |
| 371 | None => validate_fetch_target(&initial_url, context, tool_label).await?, |
| 372 | }; |
| 373 | |
| 374 | if let Some(cached) = (!cache_mode.is_revalidate()) |
| 375 | .then(|| { |
| 376 | cache::get( |
| 377 | &context.state_namespace, |
| 378 | &initial_url, |
| 379 | options.accept, |
| 380 | options.max_bytes, |
| 381 | ) |
| 382 | }) |
| 383 | .flatten() |
| 384 | { |
| 385 | let cached_url = reqwest::Url::parse(&cached.url).map_err(|err| { |
| 386 | ToolError::execution_failed(format!("cached response URL was invalid: {err}")) |
| 387 | })?; |
| 388 | let cached_host = cached_url.host_str().ok_or_else(|| { |
| 389 | ToolError::execution_failed("cached response URL did not include a host") |
| 390 | })?; |
| 391 | // No network request occurs on a cache hit, so DNS/SSRF validation is |
| 392 | // unnecessary. The final redirect destination still needs a policy |
| 393 | // check in case the session policy was tightened after insertion. |
| 394 | validate_network_policy(cached_host, context, tool_label)?; |
| 395 | return Ok(from_cached(cached, true, 0)); |
| 396 | } |
| 397 | |
| 398 | let deadline = Instant::now() + options.timeout; |
| 399 | let mut last_transient = None; |
| 400 | for attempt in 0..=1 { |
| 401 | let remaining = deadline.saturating_duration_since(Instant::now()); |
| 402 | if remaining.is_zero() { |
| 403 | break; |
| 404 | } |
| 405 | match fetch_attempt( |
| 406 | initial_url.clone(), |
| 407 | options, |
| 408 | context, |
| 409 | tool_label, |
| 410 | remaining, |
| 411 | validated_initial_pin.clone(), |
| 412 | cache_mode, |
| 413 | ) |
| 414 | .await |
| 415 | { |
| 416 | Ok(payload) if is_transient_status(payload.status) && attempt == 0 => { |
| 417 | last_transient = Some(format!("HTTP {}", payload.status)); |
| 418 | } |
| 419 | Ok(payload) => { |
| 420 | let fetched = from_cached(payload.clone(), false, attempt); |
| 421 | if (200..300).contains(&payload.status) { |
| 422 | cache::insert( |
| 423 | &context.state_namespace, |
| 424 | &initial_url, |
| 425 | options.accept, |
| 426 | payload, |
| 427 | ); |
| 428 | } |
| 429 | return Ok(fetched); |
| 430 | } |
| 431 | Err(AttemptError::Fatal(error)) => return Err(error), |
| 432 | Err(AttemptError::Transient(message)) if attempt == 0 => { |
| 433 | last_transient = Some(message); |
| 434 | } |
| 435 | Err(AttemptError::Transient(message)) => { |
| 436 | return Err(ToolError::execution_failed(format!( |
| 437 | "request failed after one retry: {message}" |
| 438 | ))); |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | let delay = retry_delay(); |
| 443 | if deadline.saturating_duration_since(Instant::now()) <= delay { |
| 444 | break; |
| 445 | } |
| 446 | tokio::time::sleep(delay).await; |
| 447 | } |
| 448 | |
| 449 | Err(ToolError::execution_failed(format!( |
| 450 | "request timed out before retry completed{}", |
| 451 | last_transient |
| 452 | .map(|message| format!(" (last failure: {message})")) |
| 453 | .unwrap_or_default() |
| 454 | ))) |
| 455 | } |
| 456 | |
| 457 | #[derive(Debug)] |
| 458 | enum AttemptError { |
| 459 | Fatal(ToolError), |
| 460 | Transient(String), |
| 461 | } |
| 462 | |
| 463 | async fn fetch_attempt( |
| 464 | initial_url: reqwest::Url, |
| 465 | options: &FetchOptions, |
| 466 | context: &ToolContext, |
| 467 | tool_label: &str, |
| 468 | timeout: Duration, |
| 469 | initial_pin: DnsPin, |
| 470 | cache_mode: CacheMode, |
| 471 | ) -> Result<CachedFetch, AttemptError> { |
| 472 | let mut current_url = initial_url; |
| 473 | let mut redirects = 0usize; |
| 474 | let mut initial_pin = initial_pin; |
| 475 | let deadline = Instant::now() + timeout; |
| 476 | |
| 477 | let response = loop { |
| 478 | let dns_pin = if redirects == 0 { |
| 479 | match initial_pin.take() { |
| 480 | Some(pin) => Some(pin), |
| 481 | None => validate_fetch_target(¤t_url, context, tool_label) |
| 482 | .await |
| 483 | .map_err(AttemptError::Fatal)?, |
| 484 | } |
| 485 | } else { |
| 486 | validate_fetch_target(¤t_url, context, tool_label) |
| 487 | .await |
| 488 | .map_err(AttemptError::Fatal)? |
| 489 | }; |
| 490 | |
| 491 | let remaining = deadline.saturating_duration_since(Instant::now()); |
| 492 | if remaining.is_zero() { |
| 493 | return Err(AttemptError::Transient( |
| 494 | "request timed out while following redirects".to_string(), |
| 495 | )); |
| 496 | } |
| 497 | let mut builder = guarded_reqwest_client_builder() |
| 498 | .timeout(remaining) |
| 499 | .user_agent(USER_AGENT) |
| 500 | .redirect(reqwest::redirect::Policy::none()); |
| 501 | if let Some((hostname, validated_ip)) = dns_pin { |
| 502 | builder = builder.resolve(&hostname, std::net::SocketAddr::new(validated_ip, 0)); |
| 503 | } |
| 504 | let client = builder.build().map_err(|err| { |
| 505 | AttemptError::Fatal(ToolError::execution_failed(format!( |
| 506 | "failed to build HTTP client: {err}" |
| 507 | ))) |
| 508 | })?; |
| 509 | let mut request = client |
| 510 | .get(current_url.clone()) |
| 511 | .header("Accept", options.accept) |
| 512 | .header("Accept-Language", "en-US,en;q=0.5"); |
| 513 | if cache_mode.is_revalidate() { |
| 514 | // `no-cache` (revalidate), not `no-store`: the shared caches still |
| 515 | // get to serve a validated copy, which is what recovers a page |
| 516 | // whose prerendered variant exists but was not the one served. |
| 517 | // `Pragma` is the HTTP/1.0 spelling some CDNs still honor. |
| 518 | request = request |
| 519 | .header("Cache-Control", "no-cache") |
| 520 | .header("Pragma", "no-cache"); |
| 521 | } |
| 522 | let response = request |
| 523 | .send() |
| 524 | .await |
| 525 | .map_err(|err| AttemptError::Transient(err.to_string()))?; |
| 526 | |
| 527 | if !response.status().is_redirection() { |
| 528 | break response; |
| 529 | } |
| 530 | if redirects >= MAX_REDIRECTS { |
| 531 | return Err(AttemptError::Fatal(ToolError::execution_failed( |
| 532 | "request exceeded the five-redirect limit", |
| 533 | ))); |
| 534 | } |
| 535 | let Some(location) = response |
| 536 | .headers() |
| 537 | .get(reqwest::header::LOCATION) |
| 538 | .and_then(|value| value.to_str().ok()) |
| 539 | else { |
| 540 | break response; |
| 541 | }; |
| 542 | current_url = response.url().join(location).map_err(|err| { |
| 543 | AttemptError::Fatal(ToolError::execution_failed(format!( |
| 544 | "invalid redirect location: {err}" |
| 545 | ))) |
| 546 | })?; |
| 547 | redirects += 1; |
| 548 | }; |
| 549 | |
| 550 | let final_url = response.url().to_string(); |
| 551 | let status = response.status().as_u16(); |
| 552 | let content_type = response |
| 553 | .headers() |
| 554 | .get(reqwest::header::CONTENT_TYPE) |
| 555 | .and_then(|value| value.to_str().ok()) |
| 556 | .unwrap_or("application/octet-stream") |
| 557 | .to_string(); |
| 558 | let headers = response_headers(response.headers()); |
| 559 | let mut stream = response.bytes_stream(); |
| 560 | let mut bytes = Vec::with_capacity(options.max_bytes.min(64 * 1024)); |
| 561 | let mut truncated = false; |
| 562 | while let Some(chunk) = stream.next().await { |
| 563 | let chunk = chunk.map_err(|err| AttemptError::Transient(err.to_string()))?; |
| 564 | let remaining = options.max_bytes.saturating_sub(bytes.len()); |
| 565 | if chunk.len() > remaining { |
| 566 | bytes.extend_from_slice(&chunk[..remaining]); |
| 567 | truncated = true; |
| 568 | break; |
| 569 | } |
| 570 | bytes.extend_from_slice(&chunk); |
| 571 | if bytes.len() == options.max_bytes { |
| 572 | // A response exactly at the cap may be complete. Ask for one more |
| 573 | // chunk to distinguish exact length from actual truncation. |
| 574 | if let Some(next) = stream.next().await { |
| 575 | let next = next.map_err(|err| AttemptError::Transient(err.to_string()))?; |
| 576 | truncated = !next.is_empty(); |
| 577 | } |
| 578 | break; |
| 579 | } |
| 580 | } |
| 581 | |
| 582 | Ok(CachedFetch { |
| 583 | url: final_url, |
| 584 | status, |
| 585 | headers, |
| 586 | content_type, |
| 587 | bytes: Arc::new(bytes), |
| 588 | truncated, |
| 589 | redirects, |
| 590 | }) |
| 591 | } |
| 592 | |
| 593 | fn from_cached(payload: CachedFetch, cache_hit: bool, retries: usize) -> FetchedPayload { |
| 594 | FetchedPayload { |
| 595 | url: payload.url, |
| 596 | status: payload.status, |
| 597 | headers: payload.headers, |
| 598 | content_type: payload.content_type, |
| 599 | bytes: payload.bytes, |
| 600 | truncated: payload.truncated, |
| 601 | cache_hit, |
| 602 | retries, |
| 603 | redirects: payload.redirects, |
| 604 | } |
| 605 | } |
| 606 | |
| 607 | fn response_headers(headers: &reqwest::header::HeaderMap) -> BTreeMap<String, String> { |
| 608 | headers |
| 609 | .iter() |
| 610 | .filter(|(name, _)| { |
| 611 | !matches!( |
| 612 | name.as_str(), |
| 613 | "authorization" |
| 614 | | "proxy-authorization" |
| 615 | | "cookie" |
| 616 | | "set-cookie" |
| 617 | | "set-cookie2" |
| 618 | | "x-api-key" |
| 619 | | "api-key" |
| 620 | ) |
| 621 | }) |
| 622 | .filter_map(|(name, value)| { |
| 623 | value |
| 624 | .to_str() |
| 625 | .ok() |
| 626 | .map(|value| (name.as_str().to_ascii_lowercase(), value.to_string())) |
| 627 | }) |
| 628 | .collect() |
| 629 | } |
| 630 | |
| 631 | fn is_transient_status(status: u16) -> bool { |
| 632 | (500..600).contains(&status) |
| 633 | } |
| 634 | |
| 635 | fn retry_delay() -> Duration { |
| 636 | #[cfg(test)] |
| 637 | return Duration::ZERO; |
| 638 | |
| 639 | #[cfg(not(test))] |
| 640 | { |
| 641 | let jitter_ms = SystemTime::now() |
| 642 | .duration_since(UNIX_EPOCH) |
| 643 | .map(|duration| u64::from(duration.subsec_nanos()) % 41) |
| 644 | .unwrap_or(0); |
| 645 | Duration::from_millis(30 + jitter_ms) |
| 646 | } |
| 647 | } |
| 648 | |
| 649 | #[cfg(test)] |
| 650 | mod tests { |
| 651 | use std::collections::BTreeMap; |
| 652 | use std::net::{IpAddr, Ipv4Addr}; |
| 653 | use std::sync::Arc; |
| 654 | use std::sync::atomic::{AtomicUsize, Ordering}; |
| 655 | |
| 656 | use serde_json::json; |
| 657 | use wiremock::matchers::{method, path}; |
| 658 | use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; |
| 659 | |
| 660 | use super::*; |
| 661 | |
| 662 | fn context(namespace: &str) -> ToolContext { |
| 663 | ToolContext::new(".").with_state_namespace(namespace) |
| 664 | } |
| 665 | |
| 666 | fn pin() -> DnsPin { |
| 667 | Some(( |
| 668 | "public.example".to_string(), |
| 669 | IpAddr::V4(Ipv4Addr::LOCALHOST), |
| 670 | )) |
| 671 | } |
| 672 | |
| 673 | #[derive(Clone)] |
| 674 | struct FailOnce { |
| 675 | calls: Arc<AtomicUsize>, |
| 676 | } |
| 677 | |
| 678 | impl Respond for FailOnce { |
| 679 | fn respond(&self, _request: &Request) -> ResponseTemplate { |
| 680 | if self.calls.fetch_add(1, Ordering::SeqCst) == 0 { |
| 681 | ResponseTemplate::new(503).set_body_json(json!({"error": "retry"})) |
| 682 | } else { |
| 683 | ResponseTemplate::new(200) |
| 684 | .insert_header("content-type", "text/plain") |
| 685 | .set_body_string("recovered response") |
| 686 | } |
| 687 | } |
| 688 | } |
| 689 | |
| 690 | #[tokio::test] |
| 691 | async fn transient_server_error_retries_once_then_caches() { |
| 692 | let server = MockServer::start().await; |
| 693 | let calls = Arc::new(AtomicUsize::new(0)); |
| 694 | Mock::given(method("GET")) |
| 695 | .and(path("/retry")) |
| 696 | .respond_with(FailOnce { |
| 697 | calls: Arc::clone(&calls), |
| 698 | }) |
| 699 | .mount(&server) |
| 700 | .await; |
| 701 | let url = format!("http://public.example:{}/retry", server.address().port()); |
| 702 | let options = FetchOptions::new(Duration::from_secs(5), 1_024, "text/plain"); |
| 703 | let context = context("fetch-retry-cache"); |
| 704 | |
| 705 | let first = fetch_with_initial_pin(&url, &options, &context, "test", pin()) |
| 706 | .await |
| 707 | .expect("retry succeeds"); |
| 708 | assert_eq!(first.status, 200); |
| 709 | assert_eq!(first.retries, 1); |
| 710 | assert!(!first.cache_hit); |
| 711 | assert_eq!(&*first.bytes, b"recovered response"); |
| 712 | |
| 713 | let second = fetch_with_initial_pin(&url, &options, &context, "test", pin()) |
| 714 | .await |
| 715 | .expect("cache hit"); |
| 716 | assert!(second.cache_hit); |
| 717 | assert_eq!(calls.load(Ordering::SeqCst), 2); |
| 718 | } |
| 719 | |
| 720 | #[tokio::test] |
| 721 | async fn truncated_cache_refetches_when_larger_body_is_requested() { |
| 722 | let server = MockServer::start().await; |
| 723 | Mock::given(method("GET")) |
| 724 | .and(path("/large")) |
| 725 | .respond_with( |
| 726 | ResponseTemplate::new(200) |
| 727 | .insert_header("content-type", "text/plain") |
| 728 | .set_body_string("0123456789"), |
| 729 | ) |
| 730 | .mount(&server) |
| 731 | .await; |
| 732 | let url = format!("http://public.example:{}/large", server.address().port()); |
| 733 | let context = context("fetch-truncated-refetch"); |
| 734 | |
| 735 | let small = fetch_with_initial_pin( |
| 736 | &url, |
| 737 | &FetchOptions::new(Duration::from_secs(5), 4, "text/plain"), |
| 738 | &context, |
| 739 | "test", |
| 740 | pin(), |
| 741 | ) |
| 742 | .await |
| 743 | .expect("small fetch"); |
| 744 | assert_eq!(&*small.bytes, b"0123"); |
| 745 | assert!(small.truncated); |
| 746 | |
| 747 | let large = fetch_with_initial_pin( |
| 748 | &url, |
| 749 | &FetchOptions::new(Duration::from_secs(5), 16, "text/plain"), |
| 750 | &context, |
| 751 | "test", |
| 752 | pin(), |
| 753 | ) |
| 754 | .await |
| 755 | .expect("larger refetch"); |
| 756 | assert_eq!(&*large.bytes, b"0123456789"); |
| 757 | assert!(!large.truncated); |
| 758 | assert!(!large.cache_hit); |
| 759 | } |
| 760 | |
| 761 | /// A Vercel-style edge that serves the client-side shell to an ordinary |
| 762 | /// request and the prerendered page to a revalidating one (#5904). |
| 763 | #[derive(Clone)] |
| 764 | struct ShellUntilRevalidated { |
| 765 | calls: Arc<AtomicUsize>, |
| 766 | always_shell: bool, |
| 767 | } |
| 768 | |
| 769 | const JS_SHELL_BODY: &str = "<html><head><title>Pricing</title></head><body><div id='root'></div><script>boot()</script></body></html>"; |
| 770 | const PRERENDERED_BODY: &str = "<html><head><title>Pricing</title></head><body><main><h1>Pricing</h1><p>The prerendered variant carries the full pricing table for every plan.</p></main></body></html>"; |
| 771 | |
| 772 | impl Respond for ShellUntilRevalidated { |
| 773 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 774 | self.calls.fetch_add(1, Ordering::SeqCst); |
| 775 | let revalidating = request |
| 776 | .headers |
| 777 | .get("cache-control") |
| 778 | .and_then(|value| value.to_str().ok()) |
| 779 | .is_some_and(|value| value.contains("no-cache")); |
| 780 | if revalidating && !self.always_shell { |
| 781 | ResponseTemplate::new(200) |
| 782 | .insert_header("content-type", "text/html") |
| 783 | .insert_header("x-vercel-cache", "HIT") |
| 784 | .insert_header("x-nextjs-prerender", "1") |
| 785 | .insert_header("age", "12") |
| 786 | .set_body_string(PRERENDERED_BODY) |
| 787 | } else { |
| 788 | ResponseTemplate::new(200) |
| 789 | .insert_header("content-type", "text/html") |
| 790 | .insert_header("x-vercel-cache", "MISS") |
| 791 | .set_body_string(JS_SHELL_BODY) |
| 792 | } |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | async fn js_shell_server(always_shell: bool) -> (MockServer, Arc<AtomicUsize>) { |
| 797 | let server = MockServer::start().await; |
| 798 | let calls = Arc::new(AtomicUsize::new(0)); |
| 799 | Mock::given(method("GET")) |
| 800 | .and(path("/pricing")) |
| 801 | .respond_with(ShellUntilRevalidated { |
| 802 | calls: Arc::clone(&calls), |
| 803 | always_shell, |
| 804 | }) |
| 805 | .mount(&server) |
| 806 | .await; |
| 807 | (server, calls) |
| 808 | } |
| 809 | |
| 810 | fn extract_html_document( |
| 811 | payload: FetchedPayload, |
| 812 | ) -> ExtractFuture<'static, super::super::extract::ExtractedDocument> { |
| 813 | Box::pin(async move { |
| 814 | super::super::extract::extract_document( |
| 815 | &payload.url, |
| 816 | Some(&payload.content_type), |
| 817 | &payload.bytes, |
| 818 | None, |
| 819 | ) |
| 820 | .await |
| 821 | }) |
| 822 | } |
| 823 | |
| 824 | #[tokio::test] |
| 825 | async fn js_shell_is_refetched_past_every_cache_before_it_becomes_an_error() { |
| 826 | let (server, calls) = js_shell_server(false).await; |
| 827 | let url = format!("http://public.example:{}/pricing", server.address().port()); |
| 828 | let context = context("js-shell-recovers"); |
| 829 | |
| 830 | let readable = fetch_readable_with_initial_pin( |
| 831 | &url, |
| 832 | &FetchOptions::new(Duration::from_secs(5), 65_536, "text/html"), |
| 833 | &context, |
| 834 | "fetch_url", |
| 835 | pin(), |
| 836 | extract_html_document, |
| 837 | ) |
| 838 | .await |
| 839 | .expect("the revalidated response carries the prerendered page"); |
| 840 | |
| 841 | assert!( |
| 842 | readable.document.markdown.contains("full pricing table"), |
| 843 | "the second attempt's content must be what the caller receives: {}", |
| 844 | readable.document.markdown |
| 845 | ); |
| 846 | assert_eq!(calls.load(Ordering::SeqCst), 2, "exactly one extra request"); |
| 847 | assert_eq!(readable.attempts.len(), 2); |
| 848 | assert!(!readable.attempts[0].cache_busted); |
| 849 | assert!(!readable.attempts[0].produced_content); |
| 850 | assert_eq!( |
| 851 | readable.attempts[0].cache_headers.get("x-vercel-cache"), |
| 852 | Some(&"MISS".to_string()), |
| 853 | "the failing attempt keeps the header that explains its cache state" |
| 854 | ); |
| 855 | assert!(readable.attempts[1].cache_busted); |
| 856 | assert!(readable.attempts[1].produced_content); |
| 857 | assert_eq!( |
| 858 | readable.attempts[1].cache_headers.get("x-vercel-cache"), |
| 859 | Some(&"HIT".to_string()) |
| 860 | ); |
| 861 | assert_eq!( |
| 862 | readable.attempts[1].cache_headers.get("x-nextjs-prerender"), |
| 863 | Some(&"1".to_string()) |
| 864 | ); |
| 865 | assert_eq!( |
| 866 | readable.attempts[1].cache_headers.get("age"), |
| 867 | Some(&"12".to_string()) |
| 868 | ); |
| 869 | } |
| 870 | |
| 871 | #[tokio::test] |
| 872 | async fn two_shells_fail_with_the_escalation_the_calling_role_owns() { |
| 873 | let (server, calls) = js_shell_server(true).await; |
| 874 | let url = format!("http://public.example:{}/pricing", server.address().port()); |
| 875 | let options = FetchOptions::new(Duration::from_secs(5), 65_536, "text/html"); |
| 876 | |
| 877 | let error = fetch_readable_with_initial_pin( |
| 878 | &url, |
| 879 | &options, |
| 880 | &context("js-shell-browser-role"), |
| 881 | "fetch_url", |
| 882 | pin(), |
| 883 | extract_html_document, |
| 884 | ) |
| 885 | .await |
| 886 | .expect_err("two shells must fail"); |
| 887 | let message = error.to_string(); |
| 888 | assert_eq!(calls.load(Ordering::SeqCst), 2, "no third request"); |
| 889 | assert!( |
| 890 | message.contains("web.run"), |
| 891 | "a role that has the browse surface must be told to use it: {message}" |
| 892 | ); |
| 893 | assert!( |
| 894 | message.contains("attempt 1") |
| 895 | && message.contains("attempt 2 (Cache-Control: no-cache)"), |
| 896 | "the failure receipt names both attempts: {message}" |
| 897 | ); |
| 898 | assert!( |
| 899 | message.contains("x-vercel-cache=MISS"), |
| 900 | "the failure receipt carries the cache-state headers: {message}" |
| 901 | ); |
| 902 | |
| 903 | // A read-only worker whose envelope denies network keeps `Web{fetch}` |
| 904 | // but loses `web.run` and any shell fallback, so the error must say so |
| 905 | // instead of naming a surface the role cannot call. |
| 906 | let mut denied = context("js-shell-read-only-role"); |
| 907 | denied.shell_policy = ShellPolicy::ReadOnly; |
| 908 | denied.execution.tool_authority = |
| 909 | Some(Arc::new(crate::tools::spec::ToolAuthorityEnvelope { |
| 910 | schema_version: 1, |
| 911 | owner: "scout".to_string(), |
| 912 | authority: crate::tools::spec::ToolMutationAuthority::ReadOnly, |
| 913 | network_access: Some(false), |
| 914 | shell: crate::tools::spec::ToolShellAuthority::ReadOnly, |
| 915 | verification: crate::tools::spec::ToolVerificationAuthority::None, |
| 916 | writable_roots: Vec::new(), |
| 917 | writable_files: Vec::new(), |
| 918 | coordination_contracts: Vec::new(), |
| 919 | })); |
| 920 | let error = fetch_readable_with_initial_pin( |
| 921 | &url, |
| 922 | &options, |
| 923 | &denied, |
| 924 | "fetch_url", |
| 925 | pin(), |
| 926 | extract_html_document, |
| 927 | ) |
| 928 | .await |
| 929 | .expect_err("two shells must fail"); |
| 930 | let message = error.to_string(); |
| 931 | assert!( |
| 932 | message.contains("not available to this role"), |
| 933 | "a role without the browse surface must be told plainly: {message}" |
| 934 | ); |
| 935 | assert!( |
| 936 | message.contains("cannot fall back to a shell fetch"), |
| 937 | "read-only roles must not be sent to curl: {message}" |
| 938 | ); |
| 939 | } |
| 940 | |
| 941 | #[tokio::test] |
| 942 | async fn transport_failures_do_not_earn_a_cache_busting_refetch() { |
| 943 | let server = MockServer::start().await; |
| 944 | let calls = Arc::new(AtomicUsize::new(0)); |
| 945 | Mock::given(method("GET")) |
| 946 | .and(path("/flaky")) |
| 947 | .respond_with(FailOnce { |
| 948 | calls: Arc::clone(&calls), |
| 949 | }) |
| 950 | .mount(&server) |
| 951 | .await; |
| 952 | let url = format!("http://public.example:{}/flaky", server.address().port()); |
| 953 | let context = context("js-shell-transport-retry"); |
| 954 | |
| 955 | let readable = fetch_readable_with_initial_pin( |
| 956 | &url, |
| 957 | &FetchOptions::new(Duration::from_secs(5), 1_024, "text/plain"), |
| 958 | &context, |
| 959 | "fetch_url", |
| 960 | pin(), |
| 961 | |payload: FetchedPayload| { |
| 962 | Box::pin(async move { Ok(String::from_utf8_lossy(&payload.bytes).into_owned()) }) |
| 963 | }, |
| 964 | ) |
| 965 | .await |
| 966 | .expect("the existing transport retry still recovers"); |
| 967 | |
| 968 | assert_eq!(readable.document, "recovered response"); |
| 969 | assert_eq!( |
| 970 | calls.load(Ordering::SeqCst), |
| 971 | 2, |
| 972 | "the 503 costs the existing single transport retry and nothing more" |
| 973 | ); |
| 974 | assert_eq!( |
| 975 | readable.attempts.len(), |
| 976 | 1, |
| 977 | "a transport retry is not a readable-fetch attempt" |
| 978 | ); |
| 979 | assert_eq!(readable.payload.retries, 1); |
| 980 | assert!(!readable.attempts[0].cache_busted); |
| 981 | assert!(readable.attempts[0].produced_content); |
| 982 | } |
| 983 | |
| 984 | #[test] |
| 985 | fn fetch_user_agent_tracks_the_crate_version() { |
| 986 | assert!( |
| 987 | USER_AGENT.contains(concat!("codewhale/", env!("CARGO_PKG_VERSION"))), |
| 988 | "guarded-fetch UA must never pin a stale release: {USER_AGENT}" |
| 989 | ); |
| 990 | } |
| 991 | |
| 992 | #[test] |
| 993 | fn response_headers_drop_set_cookie_values() { |
| 994 | let mut headers = reqwest::header::HeaderMap::new(); |
| 995 | headers.insert("content-type", "text/plain".parse().unwrap()); |
| 996 | headers.insert("set-cookie", "session=secret".parse().unwrap()); |
| 997 | headers.insert("x-api-key", "secret".parse().unwrap()); |
| 998 | |
| 999 | let filtered = response_headers(&headers); |
| 1000 | |
| 1001 | assert_eq!( |
| 1002 | filtered.get("content-type").map(String::as_str), |
| 1003 | Some("text/plain") |
| 1004 | ); |
| 1005 | assert!(!filtered.contains_key("set-cookie")); |
| 1006 | assert!(!filtered.contains_key("x-api-key")); |
| 1007 | } |
| 1008 | |
| 1009 | #[tokio::test] |
| 1010 | async fn tightened_network_policy_blocks_an_existing_cache_entry() { |
| 1011 | use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider}; |
| 1012 | |
| 1013 | let url = reqwest::Url::parse("https://example.com/cached").unwrap(); |
| 1014 | cache::insert( |
| 1015 | "policy-cache", |
| 1016 | &url, |
| 1017 | "text/plain", |
| 1018 | CachedFetch { |
| 1019 | url: url.to_string(), |
| 1020 | status: 200, |
| 1021 | headers: BTreeMap::new(), |
| 1022 | content_type: "text/plain".to_string(), |
| 1023 | bytes: Arc::new(b"cached".to_vec()), |
| 1024 | truncated: false, |
| 1025 | redirects: 0, |
| 1026 | }, |
| 1027 | ); |
| 1028 | let policy = NetworkPolicy { |
| 1029 | default: Decision::Deny.into(), |
| 1030 | allow: Vec::new(), |
| 1031 | deny: Vec::new(), |
| 1032 | proxy: Vec::new(), |
| 1033 | proxy_fake_ip_cidrs: Vec::new(), |
| 1034 | audit: false, |
| 1035 | }; |
| 1036 | let context = |
| 1037 | context("policy-cache").with_network_policy(NetworkPolicyDecider::new(policy, None)); |
| 1038 | |
| 1039 | let error = fetch_inner( |
| 1040 | url.as_str(), |
| 1041 | &FetchOptions::new(Duration::from_secs(1), 100, "text/plain"), |
| 1042 | &context, |
| 1043 | "fetch_url", |
| 1044 | None, |
| 1045 | CacheMode::Default, |
| 1046 | ) |
| 1047 | .await |
| 1048 | .expect_err("policy must win over cache"); |
| 1049 | assert!(error.to_string().contains("blocked by network policy")); |
| 1050 | } |
| 1051 | |
| 1052 | #[tokio::test] |
| 1053 | async fn tightened_network_policy_checks_cached_redirect_destination() { |
| 1054 | use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider}; |
| 1055 | |
| 1056 | let initial_url = reqwest::Url::parse("https://8.8.8.8/cached").unwrap(); |
| 1057 | cache::insert( |
| 1058 | "redirect-policy-cache", |
| 1059 | &initial_url, |
| 1060 | "text/plain", |
| 1061 | CachedFetch { |
| 1062 | url: "https://1.1.1.1/redirected".to_string(), |
| 1063 | status: 200, |
| 1064 | headers: BTreeMap::new(), |
| 1065 | content_type: "text/plain".to_string(), |
| 1066 | bytes: Arc::new(b"cached".to_vec()), |
| 1067 | truncated: false, |
| 1068 | redirects: 1, |
| 1069 | }, |
| 1070 | ); |
| 1071 | let policy = NetworkPolicy { |
| 1072 | default: Decision::Allow.into(), |
| 1073 | allow: Vec::new(), |
| 1074 | deny: vec!["1.1.1.1".to_string()], |
| 1075 | proxy: Vec::new(), |
| 1076 | proxy_fake_ip_cidrs: Vec::new(), |
| 1077 | audit: false, |
| 1078 | }; |
| 1079 | let context = context("redirect-policy-cache") |
| 1080 | .with_network_policy(NetworkPolicyDecider::new(policy, None)); |
| 1081 | |
| 1082 | let error = fetch_inner( |
| 1083 | initial_url.as_str(), |
| 1084 | &FetchOptions::new(Duration::from_secs(1), 100, "text/plain"), |
| 1085 | &context, |
| 1086 | "fetch_url", |
| 1087 | None, |
| 1088 | CacheMode::Default, |
| 1089 | ) |
| 1090 | .await |
| 1091 | .expect_err("final redirect policy must win over cache"); |
| 1092 | assert!(error.to_string().contains("1.1.1.1")); |
| 1093 | assert!(error.to_string().contains("blocked by network policy")); |
| 1094 | } |
| 1095 | } |
| 1096 |