| 1 | //! Small session-scoped TTL caches for web searches and fetched bodies. |
| 2 | |
| 3 | use std::num::NonZeroUsize; |
| 4 | use std::sync::{Arc, OnceLock}; |
| 5 | use std::time::{Duration, Instant}; |
| 6 | |
| 7 | use lru::LruCache; |
| 8 | use parking_lot::Mutex; |
| 9 | |
| 10 | use super::contract::{BackendId, SearchQuery, SearchResponse}; |
| 11 | |
| 12 | const FETCH_CACHE_ENTRIES: usize = 256; |
| 13 | const SEARCH_CACHE_ENTRIES: usize = 128; |
| 14 | const FETCH_CACHE_TTL: Duration = Duration::from_secs(15 * 60); |
| 15 | const SEARCH_CACHE_TTL: Duration = Duration::from_secs(15 * 60); |
| 16 | |
| 17 | static FETCH_CACHE: OnceLock<Mutex<LruCache<FetchCacheKey, FetchCacheEntry>>> = OnceLock::new(); |
| 18 | static SEARCH_CACHE: OnceLock<Mutex<LruCache<SearchCacheKey, SearchCacheEntry>>> = OnceLock::new(); |
| 19 | |
| 20 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] |
| 21 | struct FetchCacheKey { |
| 22 | namespace: String, |
| 23 | url: String, |
| 24 | accept: String, |
| 25 | } |
| 26 | |
| 27 | #[derive(Debug, Clone)] |
| 28 | pub(crate) struct CachedFetch { |
| 29 | pub(crate) url: String, |
| 30 | pub(crate) status: u16, |
| 31 | pub(crate) headers: std::collections::BTreeMap<String, String>, |
| 32 | pub(crate) content_type: String, |
| 33 | pub(crate) bytes: Arc<Vec<u8>>, |
| 34 | pub(crate) truncated: bool, |
| 35 | pub(crate) redirects: usize, |
| 36 | } |
| 37 | |
| 38 | #[derive(Debug, Clone)] |
| 39 | struct FetchCacheEntry { |
| 40 | fetched_at: Instant, |
| 41 | payload: CachedFetch, |
| 42 | } |
| 43 | |
| 44 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] |
| 45 | struct SearchCacheKey { |
| 46 | namespace: String, |
| 47 | initial_backend: BackendId, |
| 48 | base_url: Option<String>, |
| 49 | query: SearchQuery, |
| 50 | } |
| 51 | |
| 52 | #[derive(Debug, Clone)] |
| 53 | struct SearchCacheEntry { |
| 54 | searched_at: Instant, |
| 55 | response: SearchResponse, |
| 56 | } |
| 57 | |
| 58 | fn cache() -> &'static Mutex<LruCache<FetchCacheKey, FetchCacheEntry>> { |
| 59 | FETCH_CACHE.get_or_init(|| { |
| 60 | Mutex::new(LruCache::new( |
| 61 | NonZeroUsize::new(FETCH_CACHE_ENTRIES).expect("non-zero cache capacity"), |
| 62 | )) |
| 63 | }) |
| 64 | } |
| 65 | |
| 66 | fn search_cache() -> &'static Mutex<LruCache<SearchCacheKey, SearchCacheEntry>> { |
| 67 | SEARCH_CACHE.get_or_init(|| { |
| 68 | Mutex::new(LruCache::new( |
| 69 | NonZeroUsize::new(SEARCH_CACHE_ENTRIES).expect("non-zero search cache capacity"), |
| 70 | )) |
| 71 | }) |
| 72 | } |
| 73 | |
| 74 | fn search_key( |
| 75 | namespace: &str, |
| 76 | initial_backend: BackendId, |
| 77 | base_url: Option<&str>, |
| 78 | query: &SearchQuery, |
| 79 | ) -> SearchCacheKey { |
| 80 | SearchCacheKey { |
| 81 | namespace: namespace.to_string(), |
| 82 | initial_backend, |
| 83 | base_url: base_url.map(str::to_string), |
| 84 | query: query.clone(), |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | fn key(namespace: &str, url: &reqwest::Url, accept: &str) -> FetchCacheKey { |
| 89 | let mut canonical = url.clone(); |
| 90 | canonical.set_fragment(None); |
| 91 | FetchCacheKey { |
| 92 | namespace: namespace.to_string(), |
| 93 | url: canonical.to_string(), |
| 94 | accept: accept.to_string(), |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | pub(crate) fn get( |
| 99 | namespace: &str, |
| 100 | url: &reqwest::Url, |
| 101 | accept: &str, |
| 102 | max_bytes: usize, |
| 103 | ) -> Option<CachedFetch> { |
| 104 | let key = key(namespace, url, accept); |
| 105 | let mut cache = cache().lock(); |
| 106 | let entry = cache.get(&key)?.clone(); |
| 107 | if entry.fetched_at.elapsed() > FETCH_CACHE_TTL { |
| 108 | cache.pop(&key); |
| 109 | return None; |
| 110 | } |
| 111 | |
| 112 | // A truncated entry can answer an equal or smaller request. Asking for |
| 113 | // more is an explicit refetch so the cached cap never becomes permanent. |
| 114 | if entry.payload.truncated && max_bytes > entry.payload.bytes.len() { |
| 115 | cache.pop(&key); |
| 116 | return None; |
| 117 | } |
| 118 | |
| 119 | let mut payload = entry.payload; |
| 120 | if payload.bytes.len() > max_bytes { |
| 121 | payload.bytes = Arc::new(payload.bytes[..max_bytes].to_vec()); |
| 122 | payload.truncated = true; |
| 123 | } |
| 124 | Some(payload) |
| 125 | } |
| 126 | |
| 127 | pub(crate) fn insert(namespace: &str, url: &reqwest::Url, accept: &str, payload: CachedFetch) { |
| 128 | cache().lock().put( |
| 129 | key(namespace, url, accept), |
| 130 | FetchCacheEntry { |
| 131 | fetched_at: Instant::now(), |
| 132 | payload, |
| 133 | }, |
| 134 | ); |
| 135 | } |
| 136 | |
| 137 | pub(crate) fn get_search( |
| 138 | namespace: &str, |
| 139 | initial_backend: BackendId, |
| 140 | base_url: Option<&str>, |
| 141 | query: &SearchQuery, |
| 142 | ) -> Option<SearchResponse> { |
| 143 | let key = search_key(namespace, initial_backend, base_url, query); |
| 144 | let mut cache = search_cache().lock(); |
| 145 | let entry = cache.get(&key)?.clone(); |
| 146 | if entry.searched_at.elapsed() > SEARCH_CACHE_TTL { |
| 147 | cache.pop(&key); |
| 148 | return None; |
| 149 | } |
| 150 | |
| 151 | Some(entry.response) |
| 152 | } |
| 153 | |
| 154 | pub(crate) fn insert_search( |
| 155 | namespace: &str, |
| 156 | initial_backend: BackendId, |
| 157 | base_url: Option<&str>, |
| 158 | query: &SearchQuery, |
| 159 | response: SearchResponse, |
| 160 | ) { |
| 161 | search_cache().lock().put( |
| 162 | search_key(namespace, initial_backend, base_url, query), |
| 163 | SearchCacheEntry { |
| 164 | searched_at: Instant::now(), |
| 165 | response, |
| 166 | }, |
| 167 | ); |
| 168 | } |
| 169 | |
| 170 | #[cfg(test)] |
| 171 | pub(crate) fn reset() { |
| 172 | cache().lock().clear(); |
| 173 | } |
| 174 | |
| 175 | #[cfg(test)] |
| 176 | pub(crate) fn reset_search() { |
| 177 | search_cache().lock().clear(); |
| 178 | } |
| 179 | |
| 180 | #[cfg(test)] |
| 181 | mod tests { |
| 182 | use super::*; |
| 183 | |
| 184 | /// `FETCH_CACHE` is process-global and `reset()` empties it for every |
| 185 | /// thread, so the tests that reset it cannot run concurrently: one test's |
| 186 | /// `reset` landing between another's `insert` and its assertion takes the |
| 187 | /// entry out from under it, and the failure reads as a cache-scoping bug |
| 188 | /// rather than the test collision it is. Serializing the resetters is |
| 189 | /// cheaper than teaching the cache about tests. |
| 190 | fn fetch_cache_test_guard() -> std::sync::MutexGuard<'static, ()> { |
| 191 | static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); |
| 192 | GUARD.lock().unwrap_or_else(|e| e.into_inner()) |
| 193 | } |
| 194 | |
| 195 | fn payload(bytes: &[u8], truncated: bool) -> CachedFetch { |
| 196 | CachedFetch { |
| 197 | url: "https://example.com/doc".to_string(), |
| 198 | status: 200, |
| 199 | headers: Default::default(), |
| 200 | content_type: "text/plain".to_string(), |
| 201 | bytes: Arc::new(bytes.to_vec()), |
| 202 | truncated, |
| 203 | redirects: 0, |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | fn search_response(query: SearchQuery) -> SearchResponse { |
| 208 | use super::super::contract::{ |
| 209 | HonoredQueryCapabilities, QueryCapabilities, SearchReceipt, SearchResult, |
| 210 | }; |
| 211 | |
| 212 | SearchResponse { |
| 213 | query: query.query.clone(), |
| 214 | source: "duckduckgo".to_string(), |
| 215 | count: 1, |
| 216 | message: "Found 1 result(s)".to_string(), |
| 217 | results: vec![SearchResult::new( |
| 218 | 1, |
| 219 | "Cached result".to_string(), |
| 220 | "https://example.com/result".to_string(), |
| 221 | None, |
| 222 | None, |
| 223 | )], |
| 224 | receipt: SearchReceipt { |
| 225 | backend: BackendId::DuckDuckGo, |
| 226 | backend_detail: None, |
| 227 | requested: query, |
| 228 | capabilities: QueryCapabilities::count_only(), |
| 229 | honored: HonoredQueryCapabilities { |
| 230 | max_results: true, |
| 231 | ..HonoredQueryCapabilities::default() |
| 232 | }, |
| 233 | degraded: Vec::new(), |
| 234 | latency_ms: 4, |
| 235 | cache_hit: false, |
| 236 | }, |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | #[test] |
| 241 | fn truncated_entry_refetches_only_when_request_asks_for_more() { |
| 242 | let _guard = fetch_cache_test_guard(); |
| 243 | reset(); |
| 244 | let url = reqwest::Url::parse("https://example.com/doc#fragment").unwrap(); |
| 245 | insert("cache-unit", &url, "text/plain", payload(b"12345", true)); |
| 246 | |
| 247 | let same = get("cache-unit", &url, "text/plain", 5).expect("same cap hit"); |
| 248 | assert!(same.truncated); |
| 249 | let smaller = get("cache-unit", &url, "text/plain", 3).expect("smaller cap hit"); |
| 250 | assert_eq!(&*smaller.bytes, b"123"); |
| 251 | assert!(smaller.truncated); |
| 252 | assert!(get("cache-unit", &url, "text/plain", 6).is_none()); |
| 253 | } |
| 254 | |
| 255 | #[test] |
| 256 | fn cache_is_scoped_by_session_and_accept_header() { |
| 257 | let _guard = fetch_cache_test_guard(); |
| 258 | reset(); |
| 259 | let url = reqwest::Url::parse("https://example.com/doc").unwrap(); |
| 260 | insert("session-a", &url, "text/html", payload(b"body", false)); |
| 261 | |
| 262 | assert!(get("session-a", &url, "text/html", 10).is_some()); |
| 263 | assert!(get("session-b", &url, "text/html", 10).is_none()); |
| 264 | assert!(get("session-a", &url, "application/json", 10).is_none()); |
| 265 | } |
| 266 | |
| 267 | #[test] |
| 268 | fn search_cache_is_scoped_by_session_backend_endpoint_and_query() { |
| 269 | reset_search(); |
| 270 | let query = SearchQuery::new("cached query".to_string(), 5, None, Vec::new(), None); |
| 271 | insert_search( |
| 272 | "session-a", |
| 273 | BackendId::Tavily, |
| 274 | None, |
| 275 | &query, |
| 276 | search_response(query.clone()), |
| 277 | ); |
| 278 | |
| 279 | assert!(get_search("session-a", BackendId::Tavily, None, &query).is_some()); |
| 280 | assert!(get_search("session-b", BackendId::Tavily, None, &query).is_none()); |
| 281 | assert!(get_search("session-a", BackendId::DuckDuckGo, None, &query).is_none()); |
| 282 | assert!( |
| 283 | get_search( |
| 284 | "session-a", |
| 285 | BackendId::Tavily, |
| 286 | Some("https://search.example/"), |
| 287 | &query, |
| 288 | ) |
| 289 | .is_none() |
| 290 | ); |
| 291 | let other_query = |
| 292 | SearchQuery::new("different query".to_string(), 5, None, Vec::new(), None); |
| 293 | assert!(get_search("session-a", BackendId::Tavily, None, &other_query).is_none()); |
| 294 | } |
| 295 | } |
| 296 |