| 1 | //! Provider-neutral web-search request, result, and receipt types. |
| 2 | //! |
| 3 | //! Search backends vary widely in what they accept and return. These types are |
| 4 | //! the model-visible honesty boundary shared by `web_search` and `web.run`: |
| 5 | //! callers can distinguish requested knobs, actually honored behavior, and |
| 6 | //! degraded/post-filtered execution without depending on provider payloads. |
| 7 | |
| 8 | use serde::{Deserialize, Serialize}; |
| 9 | |
| 10 | pub(crate) const MAX_SEARCH_RESULTS: u8 = 10; |
| 11 | |
| 12 | /// Shared retrieval defaults. `web_search` and `web.run` both derive their |
| 13 | /// search knobs from these so the surfaces cannot drift apart. |
| 14 | pub(crate) const DEFAULT_SEARCH_RESULTS: usize = 5; |
| 15 | pub(crate) const DEFAULT_SEARCH_TIMEOUT_MS: u64 = 15_000; |
| 16 | pub(crate) const MAX_SEARCH_TIMEOUT_MS: u64 = 60_000; |
| 17 | |
| 18 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] |
| 19 | #[serde(rename_all = "snake_case")] |
| 20 | pub(crate) enum BackendId { |
| 21 | ProviderNative, |
| 22 | Bing, |
| 23 | #[serde(rename = "duckduckgo")] |
| 24 | DuckDuckGo, |
| 25 | Firecrawl, |
| 26 | Tavily, |
| 27 | Bocha, |
| 28 | Metaso, |
| 29 | Searxng, |
| 30 | Baidu, |
| 31 | Volcengine, |
| 32 | Sofya, |
| 33 | Serply, |
| 34 | } |
| 35 | |
| 36 | impl BackendId { |
| 37 | #[must_use] |
| 38 | pub(crate) const fn as_str(self) -> &'static str { |
| 39 | match self { |
| 40 | Self::ProviderNative => "provider_native", |
| 41 | Self::Bing => "bing", |
| 42 | Self::DuckDuckGo => "duckduckgo", |
| 43 | Self::Firecrawl => "firecrawl", |
| 44 | Self::Tavily => "tavily", |
| 45 | Self::Bocha => "bocha", |
| 46 | Self::Metaso => "metaso", |
| 47 | Self::Searxng => "searxng", |
| 48 | Self::Baidu => "baidu", |
| 49 | Self::Volcengine => "volcengine", |
| 50 | Self::Sofya => "sofya", |
| 51 | Self::Serply => "serply", |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] |
| 57 | #[serde(rename_all = "snake_case")] |
| 58 | pub(crate) enum Recency { |
| 59 | Day, |
| 60 | Week, |
| 61 | Month, |
| 62 | Year, |
| 63 | Days(u16), |
| 64 | } |
| 65 | |
| 66 | impl Recency { |
| 67 | #[must_use] |
| 68 | #[cfg(test)] |
| 69 | pub(crate) const fn days(self) -> u16 { |
| 70 | match self { |
| 71 | Self::Day => 1, |
| 72 | Self::Week => 7, |
| 73 | Self::Month => 30, |
| 74 | Self::Year => 365, |
| 75 | Self::Days(days) => days, |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] |
| 81 | pub(crate) struct SearchQuery { |
| 82 | pub(crate) query: String, |
| 83 | pub(crate) max_results: u8, |
| 84 | #[serde(skip_serializing_if = "Option::is_none")] |
| 85 | pub(crate) recency: Option<Recency>, |
| 86 | #[serde(default, skip_serializing_if = "Vec::is_empty")] |
| 87 | pub(crate) domains: Vec<String>, |
| 88 | #[serde(skip_serializing_if = "Option::is_none")] |
| 89 | pub(crate) locale: Option<String>, |
| 90 | } |
| 91 | |
| 92 | impl SearchQuery { |
| 93 | #[must_use] |
| 94 | pub(crate) fn new( |
| 95 | query: String, |
| 96 | max_results: usize, |
| 97 | recency: Option<Recency>, |
| 98 | domains: Vec<String>, |
| 99 | locale: Option<String>, |
| 100 | ) -> Self { |
| 101 | let mut domains = domains |
| 102 | .into_iter() |
| 103 | .map(|domain| { |
| 104 | let domain = domain.trim().trim_end_matches('.').to_ascii_lowercase(); |
| 105 | domain.trim_start_matches("www.").to_string() |
| 106 | }) |
| 107 | .filter(|domain| !domain.is_empty()) |
| 108 | .collect::<Vec<_>>(); |
| 109 | domains.sort_unstable(); |
| 110 | domains.dedup(); |
| 111 | Self { |
| 112 | query, |
| 113 | max_results: u8::try_from(max_results.clamp(1, usize::from(MAX_SEARCH_RESULTS))) |
| 114 | .unwrap_or(MAX_SEARCH_RESULTS), |
| 115 | recency, |
| 116 | domains, |
| 117 | locale: locale |
| 118 | .map(|value| value.trim().to_string()) |
| 119 | .filter(|value| !value.is_empty()), |
| 120 | } |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 125 | #[serde(rename_all = "snake_case")] |
| 126 | pub(crate) enum CapabilityState { |
| 127 | Supported, |
| 128 | Unsupported, |
| 129 | Unknown, |
| 130 | } |
| 131 | |
| 132 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 133 | pub(crate) struct QueryCapabilities { |
| 134 | pub(crate) max_results: CapabilityState, |
| 135 | pub(crate) recency: CapabilityState, |
| 136 | pub(crate) domains: CapabilityState, |
| 137 | pub(crate) locale: CapabilityState, |
| 138 | pub(crate) published_date: CapabilityState, |
| 139 | } |
| 140 | |
| 141 | impl QueryCapabilities { |
| 142 | #[must_use] |
| 143 | pub(crate) const fn count_only() -> Self { |
| 144 | Self { |
| 145 | max_results: CapabilityState::Supported, |
| 146 | recency: CapabilityState::Unsupported, |
| 147 | domains: CapabilityState::Unsupported, |
| 148 | locale: CapabilityState::Unsupported, |
| 149 | published_date: CapabilityState::Unknown, |
| 150 | } |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] |
| 155 | pub(crate) struct HonoredQueryCapabilities { |
| 156 | pub(crate) max_results: bool, |
| 157 | pub(crate) recency: bool, |
| 158 | pub(crate) domains: bool, |
| 159 | pub(crate) locale: bool, |
| 160 | } |
| 161 | |
| 162 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 163 | #[serde(rename_all = "snake_case")] |
| 164 | pub(crate) enum QueryKnob { |
| 165 | Recency, |
| 166 | Domains, |
| 167 | Locale, |
| 168 | } |
| 169 | |
| 170 | impl QueryKnob { |
| 171 | const fn as_str(self) -> &'static str { |
| 172 | match self { |
| 173 | Self::Recency => "recency", |
| 174 | Self::Domains => "domains", |
| 175 | Self::Locale => "locale", |
| 176 | } |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 181 | #[serde(tag = "kind", rename_all = "snake_case")] |
| 182 | pub(crate) enum DegradedReason { |
| 183 | BackendUnavailable { backend: BackendId }, |
| 184 | NoUsableResults { backend: BackendId }, |
| 185 | BackendFallback { from: BackendId, to: BackendId }, |
| 186 | ChallengeDetected { backend: BackendId }, |
| 187 | ScrapeFallback { from: BackendId, to: BackendId }, |
| 188 | KnobIgnored { knob: QueryKnob }, |
| 189 | PostFiltered { knob: QueryKnob }, |
| 190 | SynthesizedResults, |
| 191 | } |
| 192 | |
| 193 | impl DegradedReason { |
| 194 | #[must_use] |
| 195 | pub(crate) fn message(&self) -> String { |
| 196 | match self { |
| 197 | Self::BackendUnavailable { backend } => { |
| 198 | format!("{} was unavailable", backend.as_str()) |
| 199 | } |
| 200 | Self::NoUsableResults { backend } => { |
| 201 | format!("{} returned no usable results", backend.as_str()) |
| 202 | } |
| 203 | Self::BackendFallback { from, to } => format!( |
| 204 | "{} did not answer; tried {} next", |
| 205 | from.as_str(), |
| 206 | to.as_str() |
| 207 | ), |
| 208 | Self::ChallengeDetected { backend } => { |
| 209 | format!("{} returned a bot challenge", backend.as_str()) |
| 210 | } |
| 211 | Self::ScrapeFallback { from, to } => format!( |
| 212 | "{} returned no usable results; used {} fallback", |
| 213 | from.as_str(), |
| 214 | to.as_str() |
| 215 | ), |
| 216 | Self::KnobIgnored { knob } => { |
| 217 | format!("{} filter was not enforced by this backend", knob.as_str()) |
| 218 | } |
| 219 | Self::PostFiltered { knob } => { |
| 220 | format!("results were post-filtered by {}", knob.as_str()) |
| 221 | } |
| 222 | Self::SynthesizedResults => { |
| 223 | "results were synthesized by a model-backed search response".to_string() |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 230 | pub(crate) struct SearchResult { |
| 231 | pub(crate) rank: u8, |
| 232 | /// Session-scoped citation handle. Backends leave this empty; the shared |
| 233 | /// execution surface mints it before any result crosses the tool boundary. |
| 234 | pub(crate) ref_id: String, |
| 235 | pub(crate) title: String, |
| 236 | pub(crate) url: String, |
| 237 | #[serde(skip_serializing_if = "Option::is_none")] |
| 238 | pub(crate) snippet: Option<String>, |
| 239 | #[serde(skip_serializing_if = "Option::is_none")] |
| 240 | pub(crate) published: Option<String>, |
| 241 | pub(crate) domain: String, |
| 242 | } |
| 243 | |
| 244 | impl SearchResult { |
| 245 | #[must_use] |
| 246 | pub(crate) fn new( |
| 247 | rank: usize, |
| 248 | title: String, |
| 249 | url: String, |
| 250 | snippet: Option<String>, |
| 251 | published: Option<String>, |
| 252 | ) -> Self { |
| 253 | let domain = reqwest::Url::parse(&url) |
| 254 | .ok() |
| 255 | .and_then(|parsed| parsed.host_str().map(str::to_ascii_lowercase)) |
| 256 | .unwrap_or_default(); |
| 257 | Self { |
| 258 | rank: u8::try_from(rank.clamp(1, 255)).unwrap_or(u8::MAX), |
| 259 | ref_id: String::new(), |
| 260 | title, |
| 261 | url, |
| 262 | snippet, |
| 263 | published, |
| 264 | domain, |
| 265 | } |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | #[derive(Debug, Clone)] |
| 270 | pub(crate) struct BackendSearch { |
| 271 | pub(crate) backend: BackendId, |
| 272 | pub(crate) source: String, |
| 273 | pub(crate) backend_detail: Option<String>, |
| 274 | pub(crate) results: Vec<SearchResult>, |
| 275 | pub(crate) degraded: Vec<DegradedReason>, |
| 276 | pub(crate) note: Option<String>, |
| 277 | } |
| 278 | |
| 279 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 280 | pub(crate) struct SearchReceipt { |
| 281 | pub(crate) backend: BackendId, |
| 282 | #[serde(skip_serializing_if = "Option::is_none")] |
| 283 | pub(crate) backend_detail: Option<String>, |
| 284 | pub(crate) requested: SearchQuery, |
| 285 | pub(crate) capabilities: QueryCapabilities, |
| 286 | pub(crate) honored: HonoredQueryCapabilities, |
| 287 | #[serde(default, skip_serializing_if = "Vec::is_empty")] |
| 288 | pub(crate) degraded: Vec<DegradedReason>, |
| 289 | pub(crate) latency_ms: u32, |
| 290 | pub(crate) cache_hit: bool, |
| 291 | } |
| 292 | |
| 293 | impl SearchReceipt { |
| 294 | #[must_use] |
| 295 | pub(crate) fn warning(&self) -> Option<String> { |
| 296 | if self.degraded.is_empty() { |
| 297 | None |
| 298 | } else { |
| 299 | Some( |
| 300 | self.degraded |
| 301 | .iter() |
| 302 | .map(DegradedReason::message) |
| 303 | .collect::<Vec<_>>() |
| 304 | .join("; "), |
| 305 | ) |
| 306 | } |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 311 | pub(crate) struct SearchResponse { |
| 312 | pub(crate) query: String, |
| 313 | pub(crate) source: String, |
| 314 | pub(crate) count: usize, |
| 315 | pub(crate) message: String, |
| 316 | pub(crate) results: Vec<SearchResult>, |
| 317 | pub(crate) receipt: SearchReceipt, |
| 318 | } |
| 319 | |
| 320 | #[cfg(test)] |
| 321 | mod tests { |
| 322 | use super::*; |
| 323 | |
| 324 | #[test] |
| 325 | fn retrieval_defaults_are_coherent_across_search_and_fetch() { |
| 326 | use super::super::fetch::{DEFAULT_TIMEOUT, HARD_MAX_TIMEOUT}; |
| 327 | |
| 328 | // Search and fetch share one default and one hard-cap timeout so the |
| 329 | // two halves of the retrieval path behave identically by default. |
| 330 | assert_eq!( |
| 331 | u128::from(DEFAULT_SEARCH_TIMEOUT_MS), |
| 332 | DEFAULT_TIMEOUT.as_millis() |
| 333 | ); |
| 334 | assert_eq!( |
| 335 | u128::from(MAX_SEARCH_TIMEOUT_MS), |
| 336 | HARD_MAX_TIMEOUT.as_millis() |
| 337 | ); |
| 338 | assert!(DEFAULT_SEARCH_RESULTS <= usize::from(MAX_SEARCH_RESULTS)); |
| 339 | } |
| 340 | |
| 341 | #[test] |
| 342 | fn search_query_normalizes_domains_and_bounds_count() { |
| 343 | let query = SearchQuery::new( |
| 344 | "rust async".to_string(), |
| 345 | 99, |
| 346 | Some(Recency::Week), |
| 347 | vec![" WWW.Example.COM. ".to_string(), "example.com".to_string()], |
| 348 | Some(" en-US ".to_string()), |
| 349 | ); |
| 350 | |
| 351 | assert_eq!(query.max_results, 10); |
| 352 | assert_eq!(query.domains, vec!["example.com"]); |
| 353 | assert_eq!(query.locale.as_deref(), Some("en-US")); |
| 354 | assert_eq!(query.recency.map(Recency::days), Some(7)); |
| 355 | } |
| 356 | |
| 357 | #[test] |
| 358 | fn normalized_result_derives_domain_and_rank() { |
| 359 | let result = SearchResult::new( |
| 360 | 2, |
| 361 | "Example".to_string(), |
| 362 | "https://Docs.Example.com/path".to_string(), |
| 363 | Some("summary".to_string()), |
| 364 | None, |
| 365 | ); |
| 366 | |
| 367 | assert_eq!(result.rank, 2); |
| 368 | assert_eq!(result.domain, "docs.example.com"); |
| 369 | } |
| 370 | |
| 371 | #[test] |
| 372 | fn degraded_receipt_is_machine_readable_and_human_visible() { |
| 373 | let receipt = SearchReceipt { |
| 374 | backend: BackendId::DuckDuckGo, |
| 375 | backend_detail: None, |
| 376 | requested: SearchQuery::new( |
| 377 | "fresh result".to_string(), |
| 378 | 5, |
| 379 | Some(Recency::Day), |
| 380 | Vec::new(), |
| 381 | None, |
| 382 | ), |
| 383 | capabilities: QueryCapabilities::count_only(), |
| 384 | honored: HonoredQueryCapabilities { |
| 385 | max_results: true, |
| 386 | ..HonoredQueryCapabilities::default() |
| 387 | }, |
| 388 | degraded: vec![DegradedReason::KnobIgnored { |
| 389 | knob: QueryKnob::Recency, |
| 390 | }], |
| 391 | latency_ms: 4, |
| 392 | cache_hit: false, |
| 393 | }; |
| 394 | |
| 395 | let value = serde_json::to_value(&receipt).expect("receipt serializes"); |
| 396 | assert_eq!(value["backend"], "duckduckgo"); |
| 397 | assert_eq!(value["degraded"][0]["kind"], "knob_ignored"); |
| 398 | assert!(receipt.warning().expect("warning").contains("recency")); |
| 399 | } |
| 400 | |
| 401 | #[test] |
| 402 | fn scrape_fallback_receipt_preserves_backend_transitions() { |
| 403 | let receipt = SearchReceipt { |
| 404 | backend: BackendId::Bing, |
| 405 | backend_detail: None, |
| 406 | requested: SearchQuery::new("fallback query".to_string(), 5, None, Vec::new(), None), |
| 407 | capabilities: QueryCapabilities::count_only(), |
| 408 | honored: HonoredQueryCapabilities { |
| 409 | max_results: true, |
| 410 | ..HonoredQueryCapabilities::default() |
| 411 | }, |
| 412 | degraded: vec![ |
| 413 | DegradedReason::ChallengeDetected { |
| 414 | backend: BackendId::DuckDuckGo, |
| 415 | }, |
| 416 | DegradedReason::ScrapeFallback { |
| 417 | from: BackendId::DuckDuckGo, |
| 418 | to: BackendId::Bing, |
| 419 | }, |
| 420 | ], |
| 421 | latency_ms: 12, |
| 422 | cache_hit: false, |
| 423 | }; |
| 424 | |
| 425 | let value = serde_json::to_value(&receipt).expect("receipt serializes"); |
| 426 | assert_eq!(value["backend"], "bing"); |
| 427 | assert_eq!(value["degraded"][0]["kind"], "challenge_detected"); |
| 428 | assert_eq!(value["degraded"][0]["backend"], "duckduckgo"); |
| 429 | assert_eq!(value["degraded"][1]["kind"], "scrape_fallback"); |
| 430 | assert_eq!(value["degraded"][1]["from"], "duckduckgo"); |
| 431 | assert_eq!(value["degraded"][1]["to"], "bing"); |
| 432 | let warning = receipt.warning().expect("warning"); |
| 433 | assert!(warning.contains("bot challenge")); |
| 434 | assert!(warning.contains("used bing fallback")); |
| 435 | } |
| 436 | } |
| 437 |