web_run.rs - CodeWhale

web_run.rs

1	//! Web browsing tool with multi-command support (search/open/click/find/screenshot).
2	//!
3	//! This mirrors the Codex harness `web.run` interface so models can use a single
4	//! tool call to perform multiple web actions and cite sources with ref_ids.
5
6	use super::spec::{
7	ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
8	optional_u64, required_str,
9	};
10	use super::web::extract::{DocumentKind, extract_document};
11	#[cfg(test)]
12	use super::web::fetch::fetch_with_initial_pin;
13	use super::web::fetch::{FetchOptions, HARD_MAX_BYTES, fetch};
14	#[cfg(test)]
15	use super::web::guard::DnsPin;
16	use super::web::overflow::bound_text as bound_web_text;
17	#[cfg(test)]
18	use super::web::overflow::inline_char_budget;
19	use async_trait::async_trait;
20	use regex::Regex;
21	use serde::{Deserialize, Serialize};
22	use serde_json::{Value, json};
23	use std::collections::{HashMap, VecDeque};
24	use std::hash::{Hash, Hasher};
25	use std::sync::{Arc, OnceLock};
26	use std::time::{Duration, Instant};
27
28	use parking_lot::{RwLock, RwLockWriteGuard};
29
30	use super::web::contract::{
31	DEFAULT_SEARCH_RESULTS, DEFAULT_SEARCH_TIMEOUT_MS, MAX_SEARCH_RESULTS, MAX_SEARCH_TIMEOUT_MS,
32	Recency, SearchQuery, SearchReceipt, SearchResult as NormalizedSearchResult,
33	};
34	use super::web::scrape::BROWSER_USER_AGENT as USER_AGENT;
35	use super::web_search::{domain_matches, execute_search};
36
37	// Search and open share the retrieval-path defaults from `web::contract` and
38	// `web::fetch` so `web.run` cannot drift from `web_search` / `fetch_url`.
39	const DEFAULT_OPEN_TIMEOUT_MS: u64 = super::web::fetch::DEFAULT_TIMEOUT.as_millis() as u64;
40	const MAX_WEB_RUN_SESSIONS: usize = 64;
41	const MAX_PAGES_PER_SESSION: usize = 256;
42	const WEB_RUN_SESSION_TTL: Duration = Duration::from_secs(30 * 60);
43
44	static WEB_RUN_STATE: OnceLock<WebRunCache> = OnceLock::new();
45
46	#[derive(Default)]
47	struct WebRunCache {
48	sessions: RwLock<HashMap<String, WebRunSessionState>>,
49	pages: RwLock<HashMap<String, StoredWebPage>>,
50	}
51
52	#[derive(Default)]
53	struct WebRunState {
54	sessions: HashMap<String, WebRunSessionState>,
55	pages: HashMap<String, StoredWebPage>,
56	}
57
58	struct WebRunSessionState {
59	next_turn: u64,
60	refs: VecDeque<String>,
61	last_access: Instant,
62	}
63
64	impl Default for WebRunSessionState {
65	fn default() -> Self {
66	Self {
67	next_turn: 0,
68	refs: VecDeque::new(),
69	last_access: Instant::now(),
70	}
71	}
72	}
73
74	#[derive(Debug, Clone)]
75	struct StoredWebPage {
76	namespace: String,
77	page: Arc<WebPage>,
78	}
79
80	impl WebRunState {
81	fn cleanup(&mut self) {
82	let now = Instant::now();
83	let expired = self
84	.sessions
85	.iter()
86	.filter_map(\|(namespace, session)\| {
87	if now.duration_since(session.last_access) > WEB_RUN_SESSION_TTL {
88	Some(namespace.clone())
89	} else {
90	None
91	}
92	})
93	.collect::<Vec<_>>();
94	for namespace in expired {
95	self.remove_session(&namespace);
96	}
97
98	while self.sessions.len() > MAX_WEB_RUN_SESSIONS {
99	let Some(oldest_namespace) = self
100	.sessions
101	.iter()
102	.min_by_key(\|(_, session)\| session.last_access)
103	.map(\|(namespace, _)\| namespace.clone())
104	else {
105	break;
106	};
107	self.remove_session(&oldest_namespace);
108	}
109	}
110
111	fn remove_session(&mut self, namespace: &str) {
112	if let Some(session) = self.sessions.remove(namespace) {
113	for ref_id in session.refs {
114	self.pages.remove(&ref_id);
115	}
116	}
117	}
118
119	fn touch_session(&mut self, namespace: &str) {
120	self.cleanup();
121	if !self.sessions.contains_key(namespace)
122	&& self.sessions.len() >= MAX_WEB_RUN_SESSIONS
123	&& let Some(oldest_namespace) = self
124	.sessions
125	.iter()
126	.min_by_key(\|(_, session)\| session.last_access)
127	.map(\|(existing_namespace, _)\| existing_namespace.clone())
128	{
129	self.remove_session(&oldest_namespace);
130	}
131
132	let session = self.sessions.entry(namespace.to_string()).or_default();
133	session.last_access = Instant::now();
134	}
135
136	fn next_turn(&mut self, namespace: &str) -> u64 {
137	self.touch_session(namespace);
138	let session = self
139	.sessions
140	.get_mut(namespace)
141	.expect("session should exist after touch");
142	let current = session.next_turn;
143	session.next_turn = session.next_turn.saturating_add(1);
144	current
145	}
146
147	fn store_page(&mut self, namespace: &str, ref_id: &str, page: WebPage) {
148	self.touch_session(namespace);
149	let mut evicted_refs = Vec::new();
150	{
151	let session = self
152	.sessions
153	.get_mut(namespace)
154	.expect("session should exist after touch");
155	if let Some(existing_idx) = session.refs.iter().position(\|existing\| existing == ref_id)
156	{
157	session.refs.remove(existing_idx);
158	}
159	session.refs.push_back(ref_id.to_string());
160
161	while session.refs.len() > MAX_PAGES_PER_SESSION {
162	if let Some(evicted_ref) = session.refs.pop_front() {
163	evicted_refs.push(evicted_ref);
164	}
165	}
166	}
167
168	self.pages.insert(
169	ref_id.to_string(),
170	StoredWebPage {
171	namespace: namespace.to_string(),
172	page: Arc::new(page),
173	},
174	);
175	for evicted_ref in evicted_refs {
176	self.pages.remove(&evicted_ref);
177	}
178	}
179	}
180
181	#[derive(Debug, Clone, Serialize)]
182	struct WebLink {
183	id: usize,
184	url: String,
185	text: String,
186	}
187
188	#[derive(Debug, Clone)]
189	struct WebPage {
190	url: String,
191	title: Option<String>,
192	content_type: Option<String>,
193	lines: Vec<String>,
194	links: Vec<WebLink>,
195	pdf_pages: Option<Vec<Vec<String>>>,
196	truncated: bool,
197	}
198
199	#[derive(Debug, Clone, Copy)]
200	enum ResponseLength {
201	Short,
202	Medium,
203	Long,
204	}
205
206	impl ResponseLength {
207	fn from_input(input: Option<&Value>) -> Self {
208	let raw = input.and_then(\|v\| v.as_str()).unwrap_or("medium");
209	match raw.to_lowercase().as_str() {
210	"short" => Self::Short,
211	"long" => Self::Long,
212	_ => Self::Medium,
213	}
214	}
215
216	fn view_lines(self) -> usize {
217	match self {
218	Self::Short => 40,
219	Self::Medium => 80,
220	Self::Long => 160,
221	}
222	}
223
224	fn wrap_width(self) -> usize {
225	match self {
226	Self::Short => 88,
227	Self::Medium => 110,
228	Self::Long => 140,
229	}
230	}
231
232	fn max_results(self) -> usize {
233	match self {
234	Self::Short => DEFAULT_SEARCH_RESULTS,
235	Self::Medium => 8,
236	Self::Long => usize::from(MAX_SEARCH_RESULTS),
237	}
238	}
239
240	fn max_find_matches(self) -> usize {
241	match self {
242	Self::Short => 8,
243	Self::Medium => 15,
244	Self::Long => 30,
245	}
246	}
247	}
248
249	#[derive(Debug, Clone, Serialize)]
250	struct WebRunSearchResult {
251	ref_id: String,
252	query: String,
253	source: String,
254	count: usize,
255	results: Vec<NormalizedSearchResult>,
256	#[serde(skip_serializing_if = "Option::is_none")]
257	warning: Option<String>,
258	receipt: SearchReceipt,
259	}
260
261	#[derive(Debug, Clone, Serialize)]
262	struct PageViewResult {
263	ref_id: String,
264	url: String,
265	#[serde(skip_serializing_if = "Option::is_none")]
266	title: Option<String>,
267	#[serde(skip_serializing_if = "Option::is_none")]
268	content_type: Option<String>,
269	line_start: usize,
270	line_end: usize,
271	total_lines: usize,
272	#[serde(default, skip_serializing_if = "is_false")]
273	truncated: bool,
274	content: String,
275	links: Vec<WebLink>,
276	}
277
278	#[derive(Debug, Clone, Serialize)]
279	struct FindMatch {
280	line: usize,
281	text: String,
282	}
283
284	fn is_false(value: &bool) -> bool {
285	!*value
286	}
287
288	#[derive(Debug, Clone, Serialize)]
289	struct FindResult {
290	ref_id: String,
291	pattern: String,
292	count: usize,
293	matches: Vec<FindMatch>,
294	}
295
296	#[derive(Debug, Clone, Serialize)]
297	struct ScreenshotResult {
298	ref_id: String,
299	pageno: usize,
300	total_pages: usize,
301	content: String,
302	}
303
304	#[derive(Debug, Clone, Serialize)]
305	struct ImageResultEntry {
306	ref_id: String,
307	image: String,
308	#[serde(skip_serializing_if = "Option::is_none")]
309	thumbnail: Option<String>,
310	#[serde(skip_serializing_if = "Option::is_none")]
311	title: Option<String>,
312	#[serde(skip_serializing_if = "Option::is_none")]
313	url: Option<String>,
314	#[serde(skip_serializing_if = "Option::is_none")]
315	source: Option<String>,
316	#[serde(skip_serializing_if = "Option::is_none")]
317	width: Option<u32>,
318	#[serde(skip_serializing_if = "Option::is_none")]
319	height: Option<u32>,
320	}
321
322	#[derive(Debug, Clone, Serialize)]
323	struct ImageQueryResult {
324	query: String,
325	source: String,
326	count: usize,
327	results: Vec<ImageResultEntry>,
328	#[serde(skip_serializing_if = "Option::is_none")]
329	warning: Option<String>,
330	}
331
332	#[derive(Debug, Clone, Serialize, Default)]
333	struct WebRunOutput {
334	#[serde(skip_serializing_if = "Option::is_none")]
335	search_query: Option<Vec<WebRunSearchResult>>,
336	#[serde(skip_serializing_if = "Option::is_none")]
337	image_query: Option<Vec<ImageQueryResult>>,
338	#[serde(skip_serializing_if = "Option::is_none")]
339	open: Option<Vec<PageViewResult>>,
340	#[serde(skip_serializing_if = "Option::is_none")]
341	click: Option<Vec<PageViewResult>>,
342	#[serde(skip_serializing_if = "Option::is_none")]
343	find: Option<Vec<FindResult>>,
344	#[serde(skip_serializing_if = "Option::is_none")]
345	screenshot: Option<Vec<ScreenshotResult>>,
346	#[serde(skip_serializing_if = "Vec::is_empty", default)]
347	warnings: Vec<String>,
348	}
349
350	pub struct WebRunTool;
351
352	#[async_trait]
353	impl ToolSpec for WebRunTool {
354	fn name(&self) -> &'static str {
355	"web.run"
356	}
357
358	fn description(&self) -> &'static str {
359	"Browse the web (search/open/click/find/screenshot/image_query) and return structured results with ref_ids for citations."
360	}
361
362	fn input_schema(&self) -> Value {
363	json!({
364	"type": "object",
365	"properties": {
366	"search_query": {
367	"type": "array",
368	"items": {
369	"type": "object",
370	"properties": {
371	"q": { "type": "string" },
372	"recency": { "type": "integer", "minimum": 1, "maximum": 3650 },
373	"max_results": { "type": "integer" },
374	"timeout_ms": { "type": "integer" },
375	"domains": { "type": "array", "items": { "type": "string" } }
376	},
377	"required": ["q"]
378	}
379	},
380	"image_query": {
381	"type": "array",
382	"items": {
383	"type": "object",
384	"properties": {
385	"q": { "type": "string" },
386	"recency": { "type": "integer" },
387	"max_results": { "type": "integer" },
388	"timeout_ms": { "type": "integer" },
389	"domains": { "type": "array", "items": { "type": "string" } }
390	},
391	"required": ["q"]
392	}
393	},
394	"open": {
395	"type": "array",
396	"items": {
397	"type": "object",
398	"properties": {
399	"ref_id": { "type": "string" },
400	"lineno": { "type": "integer" }
401	},
402	"required": ["ref_id"]
403	}
404	},
405	"click": {
406	"type": "array",
407	"items": {
408	"type": "object",
409	"properties": {
410	"ref_id": { "type": "string" },
411	"id": { "type": "integer" }
412	},
413	"required": ["ref_id", "id"]
414	}
415	},
416	"find": {
417	"type": "array",
418	"items": {
419	"type": "object",
420	"properties": {
421	"ref_id": { "type": "string" },
422	"pattern": { "type": "string" }
423	},
424	"required": ["ref_id", "pattern"]
425	}
426	},
427	"screenshot": {
428	"type": "array",
429	"items": {
430	"type": "object",
431	"properties": {
432	"ref_id": { "type": "string" },
433	"pageno": { "type": "integer" }
434	},
435	"required": ["ref_id", "pageno"]
436	}
437	},
438	"response_length": {
439	"type": "string",
440	"enum": ["short", "medium", "long"],
441	"description": "Controls result verbosity"
442	}
443	}
444	})
445	}
446
447	fn capabilities(&self) -> Vec<ToolCapability> {
448	vec![ToolCapability::ReadOnly, ToolCapability::Network]
449	}
450
451	fn approval_requirement(&self) -> ApprovalRequirement {
452	ApprovalRequirement::Auto
453	}
454
455	async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
456	let response_length = ResponseLength::from_input(input.get("response_length"));
457	let mut output = WebRunOutput::default();
458	let scope = scoped_ref_prefix(&context.state_namespace);
459	let turn = with_state(\|state\| state.next_turn(&context.state_namespace));
460
461	let mut search_counter = 0usize;
462	let mut view_counter = 0usize;
463	let mut click_counter = 0usize;
464
465	if let Some(searches) = input.get("search_query").and_then(\|v\| v.as_array()) {
466	let mut results = Vec::new();
467	for search in searches {
468	let query = required_str(search, "q")?.trim().to_string();
469	if query.is_empty() {
470	continue;
471	}
472	let recency = optional_u64(search, "recency", 0)?;
473	let max_results = usize::try_from(optional_u64(
474	search,
475	"max_results",
476	response_length.max_results() as u64,
477	)?)
478	.unwrap_or(response_length.max_results())
479	.clamp(1, usize::from(MAX_SEARCH_RESULTS));
480	let timeout_ms = optional_u64(search, "timeout_ms", DEFAULT_SEARCH_TIMEOUT_MS)?
481	.min(MAX_SEARCH_TIMEOUT_MS);
482
483	let domains = search
484	.get("domains")
485	.and_then(\|v\| v.as_array())
486	.map(\|arr\| {
487	arr.iter()
488	.filter_map(\|v\| v.as_str().map(\|s\| s.to_string()))
489	.collect::<Vec<_>>()
490	})
491	.unwrap_or_default();
492
493	let requested_recency = if recency == 0 {
494	None
495	} else {
496	let days = u16::try_from(recency)
497	.ok()
498	.filter(\|days\| *days <= 3650)
499	.ok_or_else(\|\| {
500	ToolError::invalid_input(
501	"Field 'search_query[].recency' must be between 1 and 3650 days",
502	)
503	})?;
504	Some(Recency::Days(days))
505	};
506	let response = execute_search(
507	SearchQuery::new(query, max_results, requested_recency, domains, None),
508	timeout_ms,
509	context,
510	)
511	.await?;
512	let warning = response.receipt.warning();
513	search_counter += 1;
514	let ref_id = format!("{scope}turn{turn}search{search_counter}");
515
516	let page = page_from_search(&response.query, &response.results);
517	store_page(&context.state_namespace, &ref_id, page);
518
519	results.push(WebRunSearchResult {
520	ref_id,
521	query: response.query,
522	source: response.source,
523	count: response.count,
524	results: response.results,
525	warning,
526	receipt: response.receipt,
527	});
528	}
529	if !results.is_empty() {
530	output.search_query = Some(results);
531	}
532	}
533
534	if let Some(images) = input.get("image_query").and_then(\|v\| v.as_array()) {
535	let mut results = Vec::new();
536	for image in images {
537	let query = required_str(image, "q")?.trim().to_string();
538	if query.is_empty() {
539	continue;
540	}
541	let recency = optional_u64(image, "recency", 0)?;
542	let max_results = usize::try_from(optional_u64(
543	image,
544	"max_results",
545	response_length.max_results() as u64,
546	)?)
547	.unwrap_or(response_length.max_results())
548	.clamp(1, usize::from(MAX_SEARCH_RESULTS));
549	let timeout_ms = optional_u64(image, "timeout_ms", DEFAULT_SEARCH_TIMEOUT_MS)?
550	.min(MAX_SEARCH_TIMEOUT_MS);
551
552	let domains = image
553	.get("domains")
554	.and_then(\|v\| v.as_array())
555	.map(\|arr\| {
556	arr.iter()
557	.filter_map(\|v\| v.as_str().map(\|s\| s.to_string()))
558	.collect::<Vec<_>>()
559	})
560	.unwrap_or_default();
561
562	let (mut entries, warning) =
563	run_image_search(&query, max_results, timeout_ms, &domains).await?;
564	entries.retain_mut(\|entry\| {
565	let canonical_url = entry.url.as_deref().unwrap_or(&entry.image);
566	let Some(citation) = super::web::citations::register(
567	&context.state_namespace,
568	canonical_url,
569	entry.title.as_deref(),
570	) else {
571	return false;
572	};
573	entry.ref_id = citation.ref_id;
574	true
575	});
576
577	let mut warnings = Vec::new();
578	if recency > 0 {
579	warnings.push(format!(
580	"Recency filter not enforced (requested last {recency} days)"
581	));
582	}
583	if let Some(w) = warning {
584	warnings.push(w);
585	}
586
587	results.push(ImageQueryResult {
588	query,
589	source: "duckduckgo_images".to_string(),
590	count: entries.len(),
591	results: entries,
592	warning: if warnings.is_empty() {
593	None
594	} else {
595	Some(warnings.join("; "))
596	},
597	});
598	}
599	if !results.is_empty() {
600	output.image_query = Some(results);
601	}
602	}
603
604	if let Some(opens) = input.get("open").and_then(\|v\| v.as_array()) {
605	let mut views = Vec::new();
606	for open in opens {
607	let ref_id = required_str(open, "ref_id")?.to_string();
608	let lineno = optional_u64(open, "lineno", 1)?.max(1) as usize;
609
610	let page = resolve_or_fetch_page(&ref_id, DEFAULT_OPEN_TIMEOUT_MS, context).await?;
611	view_counter += 1;
612	let view_ref = format!("{scope}turn{turn}view{view_counter}");
613	store_page(&context.state_namespace, &view_ref, (*page).clone());
614
615	let view = render_view(&view_ref, &page, lineno, response_length);
616	views.push(view);
617	}
618	if !views.is_empty() {
619	output.open = Some(views);
620	}
621	}
622
623	if let Some(clicks) = input.get("click").and_then(\|v\| v.as_array()) {
624	let mut views = Vec::new();
625	for click in clicks {
626	let ref_id = required_str(click, "ref_id")?.to_string();
627	let link_id = optional_u64(click, "id", 0)? as usize;
628	if link_id == 0 {
629	return Err(ToolError::invalid_input("click.id must be >= 1"));
630	}
631	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
632	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
633	})?;
634	let link = page.links.iter().find(\|l\| l.id == link_id).ok_or_else(\|\| {
635	ToolError::invalid_input(format!(
636	"Link id {link_id} not found for ref_id '{ref_id}'"
637	))
638	})?;
639	let target = link.url.clone();
640	let fetched =
641	resolve_or_fetch_page(&target, DEFAULT_OPEN_TIMEOUT_MS, context).await?;
642	click_counter += 1;
643	let click_ref = format!("{scope}turn{turn}click{click_counter}");
644	store_page(&context.state_namespace, &click_ref, (*fetched).clone());
645	let view = render_view(&click_ref, &fetched, 1, response_length);
646	views.push(view);
647	}
648	if !views.is_empty() {
649	output.click = Some(views);
650	}
651	}
652
653	if let Some(find_requests) = input.get("find").and_then(\|v\| v.as_array()) {
654	let mut finds = Vec::new();
655	for find_req in find_requests {
656	let ref_id = required_str(find_req, "ref_id")?.to_string();
657	let pattern = required_str(find_req, "pattern")?.to_string();
658	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
659	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
660	})?;
661	let find_result = find_in_page(&ref_id, &pattern, &page, response_length);
662	finds.push(find_result);
663	}
664	if !finds.is_empty() {
665	output.find = Some(finds);
666	}
667	}
668
669	if let Some(shots) = input.get("screenshot").and_then(\|v\| v.as_array()) {
670	let mut screenshots = Vec::new();
671	for shot in shots {
672	let ref_id = required_str(shot, "ref_id")?.to_string();
673	let pageno = optional_u64(shot, "pageno", 0)? as usize;
674	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
675	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
676	})?;
677	let screenshot = screenshot_page(&ref_id, pageno, &page)?;
678	screenshots.push(screenshot);
679	}
680	if !screenshots.is_empty() {
681	output.screenshot = Some(screenshots);
682	}
683	}
684
685	if output.performed_no_op() {
686	// #5123-class: an empty success here reads as "nothing found"
687	// rather than "you called the tool wrong" (e.g. the natural
688	// {"query": …} shape, which matches no op key).
689	let received: Vec<&str> = input
690	.as_object()
691	.map(\|object\| object.keys().map(String::as_str).collect())
692	.unwrap_or_default();
693	return Err(ToolError::invalid_input(format!(
694	"web.run performed no operation. Pass at least one non-empty op array \
695	({}). Received keys: [{}].",
696	WEB_RUN_OP_KEYS.join(", "),
697	received.join(", ")
698	)));
699	}
700
701	bounded_web_run_result(&output, context)
702	}
703	}
704
705	const WEB_RUN_OP_KEYS: [&str; 6] = [
706	"search_query",
707	"image_query",
708	"open",
709	"click",
710	"find",
711	"screenshot",
712	];
713
714	impl WebRunOutput {
715	fn performed_no_op(&self) -> bool {
716	self.search_query.is_none()
717	&& self.image_query.is_none()
718	&& self.open.is_none()
719	&& self.click.is_none()
720	&& self.find.is_none()
721	&& self.screenshot.is_none()
722	}
723	}
724
725	fn with_state<T>(f: impl FnOnce(&mut WebRunState) -> T) -> T {
726	let cache = WEB_RUN_STATE.get_or_init(WebRunCache::default);
727	let sessions = cache.sessions.write();
728	let pages = cache.pages.write();
729	let mut guard = WebRunStateWriteBack::new(sessions, pages);
730	guard.state_mut().cleanup();
731	let result = f(guard.state_mut());
732	guard.write_back();
733	result
734	}
735
736	struct WebRunStateWriteBack<'a> {
737	sessions: RwLockWriteGuard<'a, HashMap<String, WebRunSessionState>>,
738	pages: RwLockWriteGuard<'a, HashMap<String, StoredWebPage>>,
739	state: Option<WebRunState>,
740	}
741
742	impl<'a> WebRunStateWriteBack<'a> {
743	fn new(
744	mut sessions: RwLockWriteGuard<'a, HashMap<String, WebRunSessionState>>,
745	mut pages: RwLockWriteGuard<'a, HashMap<String, StoredWebPage>>,
746	) -> Self {
747	let state = WebRunState {
748	sessions: std::mem::take(&mut *sessions),
749	pages: std::mem::take(&mut *pages),
750	};
751	Self {
752	sessions,
753	pages,
754	state: Some(state),
755	}
756	}
757
758	fn state_mut(&mut self) -> &mut WebRunState {
759	self.state
760	.as_mut()
761	.expect("web run state should be present until write-back")
762	}
763
764	fn write_back(mut self) {
765	self.restore();
766	}
767
768	fn restore(&mut self) {
769	if let Some(state) = self.state.take() {
770	*self.sessions = state.sessions;
771	*self.pages = state.pages;
772	}
773	}
774	}
775
776	impl Drop for WebRunStateWriteBack<'_> {
777	fn drop(&mut self) {
778	self.restore();
779	}
780	}
781
782	fn scoped_ref_prefix(namespace: &str) -> String {
783	let mut hasher = std::collections::hash_map::DefaultHasher::new();
784	namespace.hash(&mut hasher);
785	format!("s{:016x}_", hasher.finish())
786	}
787
788	fn store_page(namespace: &str, ref_id: &str, page: WebPage) {
789	let _ = super::web::citations::register_with_ref(
790	namespace,
791	ref_id,
792	&page.url,
793	page.title.as_deref(),
794	);
795	with_state(\|state\| {
796	state.store_page(namespace, ref_id, page);
797	});
798	}
799
800	fn get_page(namespace: &str, ref_id: &str) -> Option<Arc<WebPage>> {
801	let cache = WEB_RUN_STATE.get_or_init(WebRunCache::default);
802	let stored = {
803	let pages = cache.pages.read();
804	pages.get(ref_id).cloned()
805	}?;
806	if stored.namespace != namespace {
807	return None;
808	}
809	{
810	let mut sessions = cache.sessions.write();
811	if let Some(session) = sessions.get_mut(namespace) {
812	session.last_access = Instant::now();
813	}
814	}
815	Some(stored.page)
816	}
817
818	#[cfg(test)]
819	fn reset_web_run_state() {
820	with_state(\|state\| {
821	*state = WebRunState::default();
822	});
823	}
824
825	#[cfg(test)]
826	fn next_turn_for_namespace(namespace: &str) -> u64 {
827	with_state(\|state\| state.next_turn(namespace))
828	}
829
830	async fn resolve_or_fetch_page(
831	ref_id: &str,
832	timeout_ms: u64,
833	context: &ToolContext,
834	) -> Result<Arc<WebPage>, ToolError> {
835	if let Some(page) = get_page(&context.state_namespace, ref_id) {
836	return Ok(page);
837	}
838	if let Some(citation) = super::web::citations::resolve(&context.state_namespace, ref_id) {
839	return fetch_page(&citation.url, timeout_ms, context)
840	.await
841	.map(Arc::new);
842	}
843	if looks_like_url(ref_id) {
844	return fetch_page(ref_id, timeout_ms, context).await.map(Arc::new);
845	}
846	Err(ToolError::invalid_input(format!(
847	"Unknown ref_id '{ref_id}'"
848	)))
849	}
850
851	fn looks_like_url(value: &str) -> bool {
852	value.starts_with("http://") \|\| value.starts_with("https://")
853	}
854
855	#[derive(Debug, Clone, Deserialize)]
856	struct DuckDuckGoImageResponse {
857	#[serde(default)]
858	results: Vec<DuckDuckGoImageResult>,
859	}
860
861	#[derive(Debug, Clone, Deserialize)]
862	struct DuckDuckGoImageResult {
863	image: String,
864	#[serde(default)]
865	thumbnail: Option<String>,
866	#[serde(default)]
867	title: Option<String>,
868	#[serde(default)]
869	url: Option<String>,
870	#[serde(default)]
871	source: Option<String>,
872	#[serde(default)]
873	width: Option<u32>,
874	#[serde(default)]
875	height: Option<u32>,
876	}
877
878	fn extract_duckduckgo_vqd(html: &str) -> Option<String> {
879	let html = html.trim();
880	if html.is_empty() {
881	return None;
882	}
883
884	for (prefix, suffix) in [("vqd='", "'"), ("vqd=\"", "\"")] {
885	if let Some(start) = html.find(prefix) {
886	let rest = &html[start + prefix.len()..];
887	if let Some(end) = rest.find(suffix) {
888	let token = rest[..end].trim();
889	if !token.is_empty() {
890	return Some(token.to_string());
891	}
892	}
893	}
894	}
895
896	// Fallback: look for `vqd=` and accept a conservative token charset.
897	if let Some(start) = html.find("vqd=") {
898	let rest = &html[start + 4..];
899	let mut token = String::new();
900	for ch in rest.chars() {
901	if ch.is_ascii_alphanumeric() \|\| ch == '-' \|\| ch == '_' {
902	token.push(ch);
903	} else {
904	break;
905	}
906	}
907	if !token.is_empty() {
908	return Some(token);
909	}
910	}
911
912	None
913	}
914
915	async fn run_image_search(
916	query: &str,
917	max_results: usize,
918	timeout_ms: u64,
919	domains: &[String],
920	) -> Result<(Vec<ImageResultEntry>, Option<String>), ToolError> {
921	let client = crate::tls::reqwest_client_builder()
922	.timeout(Duration::from_millis(timeout_ms))
923	.user_agent(USER_AGENT)
924	.build()
925	.map_err(\|e\| ToolError::execution_failed(format!("Failed to build HTTP client: {e}")))?;
926
927	// Step 1: fetch the HTML page to obtain the `vqd` token used by the images API.
928	let encoded = url_encode(query);
929	let seed_url = format!("https://duckduckgo.com/?q={encoded}&iax=images&ia=images");
930	let seed_resp = client
931	.get(&seed_url)
932	.header(
933	"Accept",
934	"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8",
935	)
936	.header("Accept-Language", "en-US,en;q=0.5")
937	.send()
938	.await
939	.map_err(\|e\| {
940	ToolError::execution_failed(format!("Image search seed request failed: {e}"))
941	})?;
942
943	let seed_status = seed_resp.status();
944	let seed_body = seed_resp.text().await.map_err(\|e\| {
945	ToolError::execution_failed(format!("Failed to read image seed response: {e}"))
946	})?;
947
948	if !seed_status.is_success() {
949	return Err(ToolError::execution_failed(format!(
950	"Image search seed request failed: HTTP {}",
951	seed_status.as_u16()
952	)));
953	}
954
955	let vqd = extract_duckduckgo_vqd(&seed_body).ok_or_else(\|\| {
956	ToolError::execution_failed("Failed to extract DuckDuckGo image token (vqd)")
957	})?;
958
959	// Step 2: query the DuckDuckGo images JSON endpoint.
960	let api_url = format!("https://duckduckgo.com/i.js?l=us-en&o=json&q={encoded}&vqd={vqd}&p=1");
961	let api_resp = client
962	.get(&api_url)
963	.header("Accept", "application/json")
964	.header("Referer", "https://duckduckgo.com/")
965	.send()
966	.await
967	.map_err(\|e\| ToolError::execution_failed(format!("Image search request failed: {e}")))?;
968
969	let api_status = api_resp.status();
970	let api_body = api_resp
971	.text()
972	.await
973	.map_err(\|e\| ToolError::execution_failed(format!("Failed to read image response: {e}")))?;
974
975	if !api_status.is_success() {
976	return Err(ToolError::execution_failed(format!(
977	"Image search failed: HTTP {}",
978	api_status.as_u16()
979	)));
980	}
981
982	let parsed: DuckDuckGoImageResponse = serde_json::from_str(&api_body).map_err(\|e\| {
983	ToolError::execution_failed(format!("Failed to parse image search JSON: {e}"))
984	})?;
985
986	let mut results = parsed
987	.results
988	.into_iter()
989	.filter(\|item\| !item.image.trim().is_empty())
990	.map(\|item\| ImageResultEntry {
991	ref_id: String::new(),
992	image: item.image,
993	thumbnail: item.thumbnail,
994	title: item.title,
995	url: item.url,
996	source: item.source,
997	width: item.width,
998	height: item.height,
999	})
1000	.collect::<Vec<_>>();
1001
1002	// Domain filter is applied to the source page URL when available.
1003	let warning = if !domains.is_empty() {
1004	let before = results.len();
1005	results.retain(\|entry\| match entry.url.as_deref() {
1006	Some(url) => domain_matches(url, domains),
1007	None => true,
1008	});
1009	if before != results.len() {
1010	Some("Filtered image results by domain list".to_string())
1011	} else {
1012	None
1013	}
1014	} else {
1015	None
1016	};
1017
1018	results.truncate(max_results);
1019	Ok((results, warning))
1020	}
1021
1022	fn page_from_search(query: &str, results: &[NormalizedSearchResult]) -> WebPage {
1023	let mut lines = Vec::new();
1024	let mut links = Vec::new();
1025
1026	lines.push(format!("Search results for: {query}"));
1027	for (idx, entry) in results.iter().enumerate() {
1028	let id = idx + 1;
1029	links.push(WebLink {
1030	id,
1031	url: entry.url.clone(),
1032	text: entry.title.clone(),
1033	});
1034	lines.push(format!("{}. [{}] {}", id, id, entry.title));
1035	if let Some(snippet) = entry.snippet.as_ref()
1036	&& !snippet.trim().is_empty()
1037	{
1038	lines.push(format!(" {snippet}"));
1039	}
1040	lines.push(format!(" {url}", url = entry.url));
1041	}
1042
1043	WebPage {
1044	url: "https://html.duckduckgo.com/html/".to_string(),
1045	title: Some("Search Results".to_string()),
1046	content_type: Some("text/html".to_string()),
1047	lines,
1048	links,
1049	pdf_pages: None,
1050	truncated: false,
1051	}
1052	}
1053
1054	async fn fetch_page(
1055	url: &str,
1056	timeout_ms: u64,
1057	context: &ToolContext,
1058	) -> Result<WebPage, ToolError> {
1059	let payload = fetch(
1060	url,
1061	&FetchOptions::new(
1062	Duration::from_millis(timeout_ms),
1063	HARD_MAX_BYTES,
1064	"text/html,text/markdown,text/plain,application/xhtml+xml,application/pdf,image/,audio/,video/,/*;q=0.5",
1065	),
1066	context,
1067	"web_run",
1068	)
1069	.await?;
1070	page_from_fetched(payload, context).await
1071	}
1072
1073	#[cfg(test)]
1074	async fn fetch_page_with_initial_pin(
1075	url: &str,
1076	timeout_ms: u64,
1077	context: &ToolContext,
1078	initial_pin: Option<DnsPin>,
1079	) -> Result<WebPage, ToolError> {
1080	let payload = fetch_with_initial_pin(
1081	url,
1082	&FetchOptions::new(
1083	Duration::from_millis(timeout_ms),
1084	HARD_MAX_BYTES,
1085	"text/html,text/markdown,text/plain,application/xhtml+xml,application/pdf,image/,audio/,video/,/*;q=0.5",
1086	),
1087	context,
1088	"web_run",
1089	initial_pin.flatten(),
1090	)
1091	.await?;
1092	page_from_fetched(payload, context).await
1093	}
1094
1095	async fn page_from_fetched(
1096	payload: super::web::fetch::FetchedPayload,
1097	context: &ToolContext,
1098	) -> Result<WebPage, ToolError> {
1099	if !(200..300).contains(&payload.status) {
1100	return Err(ToolError::execution_failed(format!(
1101	"Web request to {} failed: HTTP {}",
1102	payload.url, payload.status
1103	)));
1104	}
1105	let document = extract_document(
1106	&payload.url,
1107	Some(&payload.content_type),
1108	&payload.bytes,
1109	context.cancel_token.as_ref(),
1110	)
1111	.await?;
1112	let content_type = Some(payload.content_type);
1113	match document.kind {
1114	DocumentKind::Html => {
1115	let html = document.cleaned_html.ok_or_else(\|\| {
1116	ToolError::execution_failed("Readable HTML extraction returned no document")
1117	})?;
1118	let (lines, links, parsed_title) = parse_html(&html, &payload.url);
1119	Ok(WebPage {
1120	url: payload.url,
1121	title: document.title.or(parsed_title),
1122	content_type,
1123	lines,
1124	links,
1125	pdf_pages: None,
1126	truncated: payload.truncated,
1127	})
1128	}
1129	DocumentKind::Markdown => {
1130	let (lines, links) = parse_markdown(&document.markdown, &payload.url);
1131	Ok(WebPage {
1132	url: payload.url,
1133	title: document.title,
1134	content_type,
1135	lines,
1136	links,
1137	pdf_pages: None,
1138	truncated: payload.truncated,
1139	})
1140	}
1141	DocumentKind::Text => Ok(WebPage {
1142	url: payload.url,
1143	title: document.title,
1144	content_type,
1145	lines: readable_lines(&document.text),
1146	links: Vec::new(),
1147	pdf_pages: None,
1148	truncated: payload.truncated,
1149	}),
1150	DocumentKind::Pdf => {
1151	let pages = document.pdf_pages.unwrap_or_default();
1152	Ok(WebPage {
1153	url: payload.url,
1154	title: document.title,
1155	content_type,
1156	lines: pages.first().cloned().unwrap_or_default(),
1157	links: Vec::new(),
1158	pdf_pages: Some(pages),
1159	truncated: payload.truncated,
1160	})
1161	}
1162	DocumentKind::Media => {
1163	let extension = document.media_extension.unwrap_or("bin");
1164	let digest = crate::hashing::sha256_hex(&*payload.bytes);
1165	let artifact_id = format!("web_media_{}", &digest[..16]);
1166	let (_absolute, relative) = crate::artifacts::write_session_artifact_bytes(
1167	&context.state_namespace,
1168	&artifact_id,
1169	extension,
1170	&payload.bytes,
1171	)
1172	.map_err(\|error\| {
1173	ToolError::execution_failed(format!(
1174	"failed to preserve fetched media artifact: {error}"
1175	))
1176	})?;
1177	let path = crate::artifacts::format_artifact_relative_path(&relative);
1178	Ok(WebPage {
1179	url: payload.url,
1180	title: Some("Media artifact".to_string()),
1181	content_type,
1182	lines: vec![format!("Fetched media saved to {path}")],
1183	links: Vec::new(),
1184	pdf_pages: None,
1185	truncated: payload.truncated,
1186	})
1187	}
1188	}
1189	}
1190
1191	fn bounded_web_run_result(
1192	output: &WebRunOutput,
1193	context: &ToolContext,
1194	) -> Result<ToolResult, ToolError> {
1195	let full = serde_json::to_string_pretty(output)
1196	.map_err(\|error\| ToolError::execution_failed(error.to_string()))?;
1197	let bounded = bound_web_text(
1198	full,
1199	context,
1200	\|body\| {
1201	let digest = crate::hashing::sha256_hex(body.as_bytes());
1202	format!("web_run_{}", &digest[..16])
1203	},
1204	"web.run result",
1205	)?;
1206	let metadata = bounded.artifact.map(\|artifact\| {
1207	json!({
1208	"spillover_path": artifact.absolute_path.display().to_string(),
1209	"artifact_session_id": artifact.session_id,
1210	"artifact_relative_path": crate::artifacts::format_artifact_relative_path(&artifact.relative_path),
1211	"artifact_byte_size": artifact.byte_size,
1212	"artifact_preview": artifact.preview,
1213	})
1214	});
1215
1216	Ok(ToolResult {
1217	content: bounded.content,
1218	success: true,
1219	metadata,
1220	})
1221	}
1222
1223	fn render_view(
1224	ref_id: &str,
1225	page: &WebPage,
1226	lineno: usize,
1227	response: ResponseLength,
1228	) -> PageViewResult {
1229	let total = page.lines.len();
1230	let view_lines = response.view_lines();
1231	let start = if total == 0 {
1232	1
1233	} else if lineno > total {
1234	total.saturating_sub(view_lines.saturating_sub(1)).max(1)
1235	} else {
1236	lineno
1237	};
1238	let end = if total == 0 {
1239	0
1240	} else {
1241	(start + view_lines - 1).min(total)
1242	};
1243
1244	let content = if total == 0 {
1245	"(no content)".to_string()
1246	} else {
1247	render_lines(&page.lines, start, end)
1248	};
1249
1250	PageViewResult {
1251	ref_id: ref_id.to_string(),
1252	url: page.url.clone(),
1253	title: page.title.clone(),
1254	content_type: page.content_type.clone(),
1255	line_start: start,
1256	line_end: end,
1257	total_lines: total,
1258	truncated: page.truncated,
1259	content,
1260	links: page.links.clone(),
1261	}
1262	}
1263
1264	fn render_lines(lines: &[String], start: usize, end: usize) -> String {
1265	lines
1266	.iter()
1267	.enumerate()
1268	.filter_map(\|(idx, line)\| {
1269	let line_no = idx + 1;
1270	if line_no < start \|\| line_no > end {
1271	return None;
1272	}
1273	Some(format!("{line_no:>4} {line}"))
1274	})
1275	.collect::<Vec<_>>()
1276	.join("\n")
1277	}
1278
1279	fn find_in_page(
1280	ref_id: &str,
1281	pattern: &str,
1282	page: &WebPage,
1283	response: ResponseLength,
1284	) -> FindResult {
1285	let needle = pattern.to_lowercase();
1286	let mut matches = Vec::new();
1287	for (idx, line) in page.lines.iter().enumerate() {
1288	if line.to_lowercase().contains(&needle) {
1289	matches.push(FindMatch {
1290	line: idx + 1,
1291	text: line.clone(),
1292	});
1293	}
1294	if matches.len() >= response.max_find_matches() {
1295	break;
1296	}
1297	}
1298
1299	FindResult {
1300	ref_id: ref_id.to_string(),
1301	pattern: pattern.to_string(),
1302	count: matches.len(),
1303	matches,
1304	}
1305	}
1306
1307	fn screenshot_page(
1308	ref_id: &str,
1309	pageno: usize,
1310	page: &WebPage,
1311	) -> Result<ScreenshotResult, ToolError> {
1312	let pages = page
1313	.pdf_pages
1314	.as_ref()
1315	.ok_or_else(\|\| ToolError::invalid_input("screenshot is only supported for PDF pages"))?;
1316	if pages.is_empty() {
1317	return Err(ToolError::execution_failed("PDF has no pages"));
1318	}
1319	if pageno >= pages.len() {
1320	return Err(ToolError::invalid_input(format!(
1321	"pageno {pageno} out of range (0..{max})",
1322	max = pages.len().saturating_sub(1)
1323	)));
1324	}
1325	let content = pages[pageno].join("\n");
1326	Ok(ScreenshotResult {
1327	ref_id: ref_id.to_string(),
1328	pageno,
1329	total_pages: pages.len(),
1330	content,
1331	})
1332	}
1333
1334	// === HTML Parsing ===
1335
1336	static ANCHOR_RE: OnceLock<Regex> = OnceLock::new();
1337	static TAG_RE: OnceLock<Regex> = OnceLock::new();
1338	static BLOCK_RE: OnceLock<Regex> = OnceLock::new();
1339	static SCRIPT_RE: OnceLock<Regex> = OnceLock::new();
1340	static STYLE_RE: OnceLock<Regex> = OnceLock::new();
1341	static TITLE_RE: OnceLock<Regex> = OnceLock::new();
1342	static MARKDOWN_LINK_RE: OnceLock<Regex> = OnceLock::new();
1343
1344	fn get_anchor_re() -> &'static Regex {
1345	ANCHOR_RE.get_or_init(\|\| {
1346	Regex::new(r#"(?is)<a\s+[^>]href\s=\s['\"]([^'\"]+)['\"][^>]>(.*?)</a>"#)
1347	.expect("anchor regex")
1348	})
1349	}
1350
1351	fn get_tag_re() -> &'static Regex {
1352	TAG_RE.get_or_init(\|\| Regex::new(r"<[^>]+>").expect("tag regex"))
1353	}
1354
1355	fn get_block_re() -> &'static Regex {
1356	BLOCK_RE.get_or_init(\|\| {
1357	Regex::new(r"(?is)</?(p\|div\|li\|ul\|ol\|br\|h[1-6]\|tr\|td\|th\|table\|section\|article)[^>]*>")
1358	.expect("block regex")
1359	})
1360	}
1361
1362	fn get_script_re() -> &'static Regex {
1363	SCRIPT_RE.get_or_init(\|\| Regex::new(r"(?is)<script[^>]>.?</script>").unwrap())
1364	}
1365
1366	fn get_style_re() -> &'static Regex {
1367	STYLE_RE.get_or_init(\|\| Regex::new(r"(?is)<style[^>]>.?</style>").unwrap())
1368	}
1369
1370	fn get_title_re() -> &'static Regex {
1371	TITLE_RE.get_or_init(\|\| Regex::new(r"(?is)<title[^>]>(.?)</title>").unwrap())
1372	}
1373
1374	fn parse_html(html: &str, base_url: &str) -> (Vec<String>, Vec<WebLink>, Option<String>) {
1375	let title = extract_title(html);
1376	let without_scripts = get_script_re().replace_all(html, "").to_string();
1377	let without_styles = get_style_re().replace_all(&without_scripts, "").to_string();
1378
1379	let (with_links, links) = replace_links(&without_styles, base_url);
1380	let with_breaks = get_block_re().replace_all(&with_links, "\n").to_string();
1381	let without_tags = get_tag_re().replace_all(&with_breaks, "").to_string();
1382	let decoded = decode_html_entities(&without_tags);
1383
1384	let mut lines = Vec::new();
1385	for line in decoded.lines() {
1386	let trimmed = normalize_whitespace(line);
1387	if trimmed.is_empty() {
1388	continue;
1389	}
1390	for wrapped in wrap_line(&trimmed, ResponseLength::Medium.wrap_width()) {
1391	lines.push(wrapped);
1392	}
1393	}
1394
1395	(lines, links, title)
1396	}
1397
1398	fn parse_markdown(markdown: &str, base_url: &str) -> (Vec<String>, Vec<WebLink>) {
1399	let re = MARKDOWN_LINK_RE.get_or_init(\|\| {
1400	Regex::new(r#"\[([^\]]+)\]\(([^\s)]+)(?:\s+"[^"]*")?\)"#).expect("markdown link regex")
1401	});
1402	let mut links = Vec::new();
1403	let mut replaced = String::with_capacity(markdown.len());
1404	let mut last = 0;
1405	for capture in re.captures_iter(markdown) {
1406	let Some(full) = capture.get(0) else { continue };
1407	let Some(text) = capture.get(1) else { continue };
1408	let Some(target) = capture.get(2) else {
1409	continue;
1410	};
1411	replaced.push_str(&markdown[last..full.start()]);
1412	let id = links.len() + 1;
1413	let text = normalize_whitespace(text.as_str());
1414	let url = resolve_url(base_url, target.as_str());
1415	links.push(WebLink {
1416	id,
1417	url,
1418	text: text.clone(),
1419	});
1420	replaced.push_str(&format!("[{id}] {text}"));
1421	last = full.end();
1422	}
1423	replaced.push_str(&markdown[last..]);
1424	(readable_lines(&replaced), links)
1425	}
1426
1427	fn readable_lines(text: &str) -> Vec<String> {
1428	text.lines()
1429	.flat_map(\|line\| {
1430	let line = normalize_whitespace(line);
1431	wrap_line(&line, ResponseLength::Medium.wrap_width())
1432	})
1433	.filter(\|line\| !line.is_empty())
1434	.collect()
1435	}
1436
1437	fn extract_title(html: &str) -> Option<String> {
1438	let re = get_title_re();
1439	let cap = re.captures(html)?;
1440	let raw = cap.get(1)?.as_str();
1441	let cleaned = normalize_whitespace(&decode_html_entities(raw));
1442	if cleaned.is_empty() {
1443	None
1444	} else {
1445	Some(cleaned)
1446	}
1447	}
1448
1449	fn replace_links(html: &str, base_url: &str) -> (String, Vec<WebLink>) {
1450	let re = get_anchor_re();
1451	let mut links = Vec::new();
1452	let mut output = String::with_capacity(html.len());
1453	let mut last = 0;
1454
1455	for cap in re.captures_iter(html) {
1456	let Some(full) = cap.get(0) else { continue };
1457	let Some(href) = cap.get(1) else { continue };
1458	let Some(text_match) = cap.get(2) else {
1459	continue;
1460	};
1461
1462	output.push_str(&html[last..full.start()]);
1463	let text = normalize_whitespace(&strip_tags(text_match.as_str()));
1464	let resolved = resolve_url(base_url, href.as_str());
1465	if !text.is_empty() {
1466	let id = links.len() + 1;
1467	links.push(WebLink {
1468	id,
1469	url: resolved.clone(),
1470	text: text.clone(),
1471	});
1472	output.push_str(&format!("[{id}] {text}"));
1473	} else {
1474	output.push_str(&resolved);
1475	}
1476	last = full.end();
1477	}
1478
1479	output.push_str(&html[last..]);
1480	(output, links)
1481	}
1482
1483	fn resolve_url(base: &str, href: &str) -> String {
1484	if href.starts_with("http://") \|\| href.starts_with("https://") {
1485	return href.to_string();
1486	}
1487	if href.starts_with("//") {
1488	return format!("https:{href}");
1489	}
1490	if let Ok(base_url) = reqwest::Url::parse(base)
1491	&& let Ok(joined) = base_url.join(href)
1492	{
1493	return joined.to_string();
1494	}
1495	href.to_string()
1496	}
1497
1498	fn strip_tags(text: &str) -> String {
1499	get_tag_re().replace_all(text, "").to_string()
1500	}
1501
1502	fn normalize_whitespace(text: &str) -> String {
1503	text.split_whitespace().collect::<Vec<_>>().join(" ")
1504	}
1505
1506	fn wrap_line(text: &str, width: usize) -> Vec<String> {
1507	if text.len() <= width {
1508	return vec![text.to_string()];
1509	}
1510	let mut lines = Vec::new();
1511	let mut current = String::new();
1512	for word in text.split_whitespace() {
1513	if current.is_empty() {
1514	current.push_str(word);
1515	} else if current.len() + word.len() < width {
1516	current.push(' ');
1517	current.push_str(word);
1518	} else {
1519	lines.push(current);
1520	current = word.to_string();
1521	}
1522	}
1523	if !current.is_empty() {
1524	lines.push(current);
1525	}
1526	lines
1527	}
1528
1529	fn decode_html_entities(text: &str) -> String {
1530	text.replace("&", "&")
1531	.replace(""", "\"")
1532	.replace("'", "'")
1533	.replace("'", "'")
1534	.replace("<", "<")
1535	.replace(">", ">")
1536	.replace(" ", " ")
1537	}
1538
1539	fn url_encode(input: &str) -> String {
1540	crate::utils::url_encode(input)
1541	}
1542
1543	// === Tests ===
1544
1545	#[cfg(test)]
1546	mod tests {
1547	use super::*;
1548	use crate::tools::web::scrape::{parse_bing_results, parse_duckduckgo_results};
1549	use std::path::PathBuf;
1550	use tokio::sync::{Mutex, MutexGuard};
1551
1552	static WEB_RUN_TEST_LOCK: Mutex<()> = Mutex::const_new(());
1553
1554	struct ArtifactRootRestore(Option<PathBuf>);
1555
1556	impl Drop for ArtifactRootRestore {
1557	fn drop(&mut self) {
1558	crate::artifacts::set_test_artifact_sessions_root(self.0.take());
1559	}
1560	}
1561
1562	fn lock_web_run_test_state() -> MutexGuard<'static, ()> {
1563	WEB_RUN_TEST_LOCK.blocking_lock()
1564	}
1565
1566	fn sample_page(url: &str) -> WebPage {
1567	WebPage {
1568	url: url.to_string(),
1569	title: Some("Example".to_string()),
1570	content_type: Some("text/html".to_string()),
1571	lines: vec!["example line".to_string()],
1572	links: Vec::new(),
1573	pdf_pages: None,
1574	truncated: false,
1575	}
1576	}
1577
1578	fn sample_page_with_link(url: &str, target: &str) -> WebPage {
1579	let mut page = sample_page(url);
1580	page.links.push(WebLink {
1581	id: 1,
1582	url: target.to_string(),
1583	text: "target".to_string(),
1584	});
1585	page
1586	}
1587
1588	#[test]
1589	fn html_link_parsing_extracts_links() {
1590	let html = r#"
1591	<html><body>
1592	<p>Hello <a href="https://example.com">Example</a> world.</p>
1593	</body></html>
1594	"#;
1595	let (lines, links, title) = parse_html(html, "https://example.com");
1596	assert!(title.is_none());
1597	assert_eq!(links.len(), 1);
1598	assert_eq!(links[0].url, "https://example.com");
1599	assert!(lines.iter().any(\|line\| line.contains("Example")));
1600	}
1601
1602	#[test]
1603	fn markdown_link_parsing_preserves_click_targets() {
1604	let (lines, links) = parse_markdown(
1605	"## Guide\n\nRead [the proof](/proof) before shipping.",
1606	"https://example.com/docs/start",
1607	);
1608
1609	assert_eq!(links.len(), 1);
1610	assert_eq!(links[0].url, "https://example.com/proof");
1611	assert!(lines.iter().any(\|line\| line.contains("[1] the proof")));
1612	}
1613
1614	#[test]
1615	fn oversized_web_run_output_round_trips_through_session_artifact() {
1616	let _lock = crate::artifacts::TEST_ARTIFACT_SESSIONS_GUARD
1617	.lock()
1618	.unwrap_or_else(\|error\| error.into_inner());
1619	let tmp = tempfile::tempdir().unwrap();
1620	let prior =
1621	crate::artifacts::set_test_artifact_sessions_root(Some(tmp.path().join("sessions")));
1622	let _restore = ArtifactRootRestore(prior);
1623	let context = ToolContext::new(".")
1624	.with_state_namespace("web-run-overflow")
1625	.with_route_context_window(10_000);
1626	let output = WebRunOutput {
1627	warnings: vec!["large receipt ".repeat(200)],
1628	..WebRunOutput::default()
1629	};
1630
1631	let result = bounded_web_run_result(&output, &context).unwrap();
1632	let metadata = result.metadata.expect("artifact metadata");
1633	let path = metadata["spillover_path"].as_str().unwrap();
1634	let full = std::fs::read_to_string(path).unwrap();
1635
1636	assert!(result.content.contains("retrieve_tool_result"));
1637	assert!(result.content.chars().count() <= inline_char_budget(&context));
1638	assert_eq!(
1639	serde_json::from_str::<Value>(&full).unwrap()["warnings"][0],
1640	output.warnings[0]
1641	);
1642	}
1643
1644	#[test]
1645	fn wrap_line_splits_long_lines() {
1646	let line = "This is a long line that should wrap cleanly at word boundaries";
1647	let wrapped = wrap_line(line, 20);
1648	assert!(wrapped.len() > 1);
1649	assert!(wrapped.iter().all(\|l\| l.len() <= 20));
1650	}
1651
1652	#[test]
1653	fn extracts_duckduckgo_vqd_token() {
1654	let html_single = "<script>var x = {vqd='3-1234567890'};</script>";
1655	assert_eq!(
1656	extract_duckduckgo_vqd(html_single),
1657	Some("3-1234567890".to_string())
1658	);
1659
1660	let html_double = "<script>var x = {vqd=\"3-abcdef\"};</script>";
1661	assert_eq!(
1662	extract_duckduckgo_vqd(html_double),
1663	Some("3-abcdef".to_string())
1664	);
1665
1666	let html_plain = "https://duckduckgo.com/?q=test&vqd=3-xyz_123&ia=images";
1667	assert_eq!(
1668	extract_duckduckgo_vqd(html_plain),
1669	Some("3-xyz_123".to_string())
1670	);
1671	}
1672
1673	#[tokio::test]
1674	async fn text_search_uses_configured_shared_backend_and_exposes_receipt() {
1675	use crate::config::SearchProvider;
1676	use crate::tools::spec::ToolSpec;
1677	use wiremock::matchers::{method, path, query_param};
1678	use wiremock::{Mock, MockServer, ResponseTemplate};
1679
1680	let server = MockServer::start().await;
1681	Mock::given(method("GET"))
1682	.and(path("/search"))
1683	.and(query_param("q", "shared seam"))
1684	.and(query_param("format", "json"))
1685	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
1686	"results": [{
1687	"title": "Shared result",
1688	"url": "https://docs.example.com/shared",
1689	"content": "one adapter path"
1690	}]
1691	})))
1692	.mount(&server)
1693	.await;
1694
1695	let tmp = tempfile::tempdir().expect("tempdir");
1696	let mut context = ToolContext::new(tmp.path().to_path_buf());
1697	context.search_provider = SearchProvider::Searxng;
1698	context.search_base_url = Some(server.uri());
1699	context.state_namespace = "shared-backend-test".to_string();
1700
1701	let result = WebRunTool
1702	.execute(
1703	json!({
1704	"search_query": [{
1705	"q": "shared seam",
1706	"recency": 7,
1707	"domains": ["example.com"]
1708	}]
1709	}),
1710	&context,
1711	)
1712	.await
1713	.expect("web.run should use configured SearXNG backend");
1714	let value: Value = serde_json::from_str(&result.content).expect("web.run json");
1715	let search = &value["search_query"][0];
1716
1717	assert_eq!(search["source"], "searxng");
1718	assert_eq!(search["count"], 1);
1719	assert_eq!(search["results"][0]["rank"], 1);
1720	assert_eq!(search["results"][0]["domain"], "docs.example.com");
1721	assert_eq!(search["receipt"]["backend"], "searxng");
1722	assert_eq!(search["receipt"]["honored"]["domains"], true);
1723	assert!(
1724	search["warning"]
1725	.as_str()
1726	.expect("visible degraded warning")
1727	.contains("recency")
1728	);
1729	}
1730
1731	#[tokio::test]
1732	async fn search_ref_ids_resolve_to_their_source_urls_for_open() {
1733	use crate::config::SearchProvider;
1734	use crate::tools::spec::ToolSpec;
1735	use wiremock::matchers::{method, path, query_param};
1736	use wiremock::{Mock, MockServer, ResponseTemplate};
1737
1738	let server = MockServer::start().await;
1739	Mock::given(method("GET"))
1740	.and(path("/search"))
1741	.and(query_param("q", "citation handoff"))
1742	.and(query_param("format", "json"))
1743	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
1744	"results": [{
1745	"title": "Handoff target",
1746	"url": "https://docs.example.com/handoff",
1747	"content": "the page a later open command fetches"
1748	}]
1749	})))
1750	.mount(&server)
1751	.await;
1752
1753	let tmp = tempfile::tempdir().expect("tempdir");
1754	let mut context = ToolContext::new(tmp.path().to_path_buf());
1755	context.search_provider = SearchProvider::Searxng;
1756	context.search_base_url = Some(server.uri());
1757	context.state_namespace = "handoff-citation-test".to_string();
1758
1759	let result = WebRunTool
1760	.execute(
1761	json!({"search_query": [{"q": "citation handoff"}]}),
1762	&context,
1763	)
1764	.await
1765	.expect("web.run search should succeed");
1766	let value: Value = serde_json::from_str(&result.content).expect("web.run json");
1767	let ref_id = value["search_query"][0]["results"][0]["ref_id"]
1768	.as_str()
1769	.expect("every search result carries a minted ref_id")
1770	.to_string();
1771
1772	// `open` resolves search-result refs through the shared citation
1773	// registry; the handoff must preserve the exact source URL.
1774	let citation = crate::tools::web::citations::resolve(&context.state_namespace, &ref_id)
1775	.expect("search result ref must resolve for a later open");
1776	assert_eq!(citation.url, "https://docs.example.com/handoff");
1777	assert_eq!(citation.title.as_deref(), Some("Handoff target"));
1778	assert!(
1779	crate::tools::web::citations::resolve("foreign-session", &ref_id).is_none(),
1780	"citation handles must stay session-scoped"
1781	);
1782	}
1783
1784	#[tokio::test]
1785	async fn open_failure_reports_url_and_status() {
1786	let payload = crate::tools::web::fetch::FetchedPayload {
1787	url: "https://example.com/missing".to_string(),
1788	status: 404,
1789	headers: std::collections::BTreeMap::new(),
1790	content_type: "text/html".to_string(),
1791	bytes: Arc::new(Vec::new()),
1792	truncated: false,
1793	cache_hit: false,
1794	retries: 0,
1795	redirects: 0,
1796	};
1797	let context = ToolContext::new(PathBuf::from("."));
1798
1799	let error = page_from_fetched(payload, &context)
1800	.await
1801	.expect_err("non-2xx pages must not be rendered");
1802	let message = error.to_string();
1803	assert!(
1804	message.contains("https://example.com/missing"),
1805	"transport failures must name the URL: {message}"
1806	);
1807	assert!(message.contains("404"), "got `{message}`");
1808	}
1809
1810	#[test]
1811	fn response_length_result_counts_are_anchored_to_the_shared_contract() {
1812	assert_eq!(ResponseLength::Short.max_results(), DEFAULT_SEARCH_RESULTS);
1813	assert_eq!(
1814	ResponseLength::Long.max_results(),
1815	usize::from(MAX_SEARCH_RESULTS)
1816	);
1817	assert!(ResponseLength::Medium.max_results() <= usize::from(MAX_SEARCH_RESULTS));
1818	}
1819
1820	#[test]
1821	fn parses_bing_results_and_decodes_redirect_url() {
1822	let html = r#"
1823	<ol>
1824	<li class="b_algo">
1825	<h2><a href="https://www.bing.com/ck/a?u=a1aHR0cHM6Ly9leGFtcGxlLmNvbS9wYXRoP3E9MQ">Example & Result</a></h2>
1826	<div class="b_caption"><p>A <strong>useful</strong> snippet.</p></div>
1827	</li>
1828	</ol>
1829	"#;
1830
1831	let results = parse_bing_results(html, 5);
1832
1833	assert_eq!(results.len(), 1);
1834	assert_eq!(results[0].title, "Example & Result");
1835	assert_eq!(results[0].url, "https://example.com/path?q=1");
1836	assert_eq!(results[0].snippet.as_deref(), Some("A useful snippet."));
1837	}
1838
1839	#[test]
1840	fn web_run_search_path_filters_known_spam_domain() {
1841	// The shared scraper used by web_run filters the known #964 spam family.
1842	let html = r#"
1843	<a class="result__a" href="https://astralia.forumgratuit.org/a">A</a>
1844	<a class="result__snippet">spam</a>
1845	<a class="result__a" href="https://russia.forumgratuit.org/b">B</a>
1846	<a class="result__snippet">spam</a>
1847	<a class="result__a" href="https://other.forumgratuit.org/c">C</a>
1848	<a class="result__snippet">spam</a>
1849	<a class="result__a" href="https://hello.forumgratuit.org/d">D</a>
1850	<a class="result__snippet">spam</a>
1851	<a class="result__a" href="https://world.forumgratuit.org/e">E</a>
1852	<a class="result__snippet">spam</a>
1853	"#;
1854	let results = parse_duckduckgo_results(html, 10);
1855	assert!(
1856	results.is_empty(),
1857	"web_run path must drop the known spam family via the shared scraper"
1858	);
1859	}
1860
1861	#[test]
1862	fn domain_scoped_fixture_preserves_legitimate_same_site_results() {
1863	let html = r#"
1864	<a class="result__a" href="https://docs.example.co.uk/a">A</a>
1865	<a class="result__snippet">s</a>
1866	<a class="result__a" href="https://docs.example.co.uk/b">B</a>
1867	<a class="result__snippet">s</a>
1868	<a class="result__a" href="https://docs.example.co.uk/c">C</a>
1869	<a class="result__snippet">s</a>
1870	<a class="result__a" href="https://other.example/d">D</a>
1871	<a class="result__snippet">s</a>
1872	"#;
1873	let domains = vec!["docs.example.co.uk".to_string()];
1874	let mut results = parse_duckduckgo_results(html, 10);
1875	results.retain(\|entry\| domain_matches(&entry.url, &domains));
1876
1877	assert_eq!(results.len(), 3);
1878	assert!(
1879	results
1880	.iter()
1881	.all(\|entry\| entry.url.contains("docs.example.co.uk"))
1882	);
1883	}
1884
1885	#[test]
1886	fn scoped_ref_prefix_is_session_specific() {
1887	let _lock = lock_web_run_test_state();
1888	reset_web_run_state();
1889	let alpha = scoped_ref_prefix("session-alpha");
1890	let beta = scoped_ref_prefix("session-beta");
1891
1892	assert_ne!(alpha, beta);
1893	assert!(alpha.starts_with('s'));
1894	assert!(alpha.ends_with('_'));
1895	assert_eq!(alpha.len(), 18);
1896	}
1897
1898	#[test]
1899	fn stored_pages_do_not_cross_scoped_sessions() {
1900	let _lock = lock_web_run_test_state();
1901	reset_web_run_state();
1902	let shared_suffix = "turn1search1";
1903	let ref_alpha = format!("{}{}", scoped_ref_prefix("session-alpha"), shared_suffix);
1904	let ref_beta = format!("{}{}", scoped_ref_prefix("session-beta"), shared_suffix);
1905
1906	store_page(
1907	"session-alpha",
1908	&ref_alpha,
1909	sample_page("https://example.com/alpha"),
1910	);
1911
1912	assert!(get_page("session-alpha", &ref_alpha).is_some());
1913	assert!(get_page("session-beta", &ref_alpha).is_none());
1914	assert!(get_page("session-beta", &ref_beta).is_none());
1915	}
1916
1917	#[tokio::test(flavor = "current_thread")]
1918	async fn execute_open_rejects_exact_foreign_session_ref() {
1919	let _lock = WEB_RUN_TEST_LOCK.lock().await;
1920	reset_web_run_state();
1921	let ref_id = format!("{}turn0search1", scoped_ref_prefix("foreign-open-owner"));
1922	store_page(
1923	"foreign-open-owner",
1924	&ref_id,
1925	sample_page("https://example.com/private-session-page"),
1926	);
1927	let context =
1928	ToolContext::new(PathBuf::from(".")).with_state_namespace("foreign-open-caller");
1929
1930	let err = WebRunTool
1931	.execute(json!({"open": [{"ref_id": ref_id}]}), &context)
1932	.await
1933	.expect_err("foreign exact ref must not open");
1934
1935	assert!(format!("{err}").contains("Unknown ref_id"));
1936	}
1937
1938	#[tokio::test(flavor = "current_thread")]
1939	async fn execute_click_rejects_exact_foreign_session_ref() {
1940	let _lock = WEB_RUN_TEST_LOCK.lock().await;
1941	reset_web_run_state();
1942	let ref_id = format!("{}turn0search1", scoped_ref_prefix("foreign-click-owner"));
1943	store_page(
1944	"foreign-click-owner",
1945	&ref_id,
1946	sample_page_with_link(
1947	"https://example.com/private-session-page",
1948	"https://example.com/target",
1949	),
1950	);
1951	let context =
1952	ToolContext::new(PathBuf::from(".")).with_state_namespace("foreign-click-caller");
1953
1954	let err = WebRunTool
1955	.execute(json!({"click": [{"ref_id": ref_id, "id": 1}]}), &context)
1956	.await
1957	.expect_err("foreign exact ref must not be clickable");
1958
1959	assert!(format!("{err}").contains("Unknown ref_id"));
1960	}
1961
1962	#[tokio::test(flavor = "current_thread")]
1963	async fn execute_click_routes_target_through_shared_ssrf_guard() {
1964	let _lock = WEB_RUN_TEST_LOCK.lock().await;
1965	reset_web_run_state();
1966	let namespace = "guarded-click-session";
1967	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
1968	store_page(
1969	namespace,
1970	&ref_id,
1971	sample_page_with_link("https://example.com/source", "http://127.0.0.1/admin"),
1972	);
1973	let context = ToolContext::new(PathBuf::from(".")).with_state_namespace(namespace);
1974
1975	let err = WebRunTool
1976	.execute(json!({"click": [{"ref_id": ref_id, "id": 1}]}), &context)
1977	.await
1978	.expect_err("click target must be SSRF-guarded");
1979
1980	assert!(format!("{err}").contains("restricted address"));
1981	}
1982
1983	#[test]
1984	fn cached_page_reads_share_page_arc() {
1985	let _lock = lock_web_run_test_state();
1986	reset_web_run_state();
1987	let namespace = "session-alpha";
1988	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
1989	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
1990
1991	let first = get_page(namespace, &ref_id).expect("first page read");
1992	let second = get_page(namespace, &ref_id).expect("second page read");
1993
1994	assert!(Arc::ptr_eq(&first, &second));
1995	}
1996
1997	#[test]
1998	fn turn_counters_are_scoped_per_session() {
1999	let _lock = lock_web_run_test_state();
2000	reset_web_run_state();
2001
2002	assert_eq!(next_turn_for_namespace("session-alpha"), 0);
2003	assert_eq!(next_turn_for_namespace("session-alpha"), 1);
2004	assert_eq!(next_turn_for_namespace("session-beta"), 0);
2005	}
2006
2007	#[test]
2008	fn with_state_restores_cache_after_panic() {
2009	let _lock = lock_web_run_test_state();
2010	reset_web_run_state();
2011	let namespace = "session-alpha";
2012	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2013	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
2014
2015	let panic_result = std::panic::catch_unwind(\|\| {
2016	with_state(\|state\| {
2017	let session = state
2018	.sessions
2019	.get_mut(namespace)
2020	.expect("session should exist");
2021	session.next_turn = 42;
2022	panic!("exercise web_run write-back guard");
2023	});
2024	});
2025
2026	assert!(panic_result.is_err());
2027	assert!(get_page(namespace, &ref_id).is_some());
2028	assert_eq!(next_turn_for_namespace(namespace), 42);
2029	}
2030
2031	#[test]
2032	fn stale_session_pages_are_evicted() {
2033	let _lock = lock_web_run_test_state();
2034	reset_web_run_state();
2035	let namespace = "session-alpha";
2036	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2037	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
2038
2039	// On Windows, Instant's epoch is system boot. If the CI runner has
2040	// been up for less than WEB_RUN_SESSION_TTL the subtraction would
2041	// underflow, so we skip the test in that case.
2042	let stale = WEB_RUN_SESSION_TTL + Duration::from_secs(1);
2043	let can_test = with_state(\|state\| {
2044	let session = state
2045	.sessions
2046	.get_mut(namespace)
2047	.expect("session should exist");
2048	match Instant::now().checked_sub(stale) {
2049	Some(past) => {
2050	session.last_access = past;
2051	true
2052	}
2053	None => false,
2054	}
2055	});
2056	if !can_test {
2057	// System uptime shorter than session TTL; can't test eviction.
2058	return;
2059	}
2060
2061	let _ = next_turn_for_namespace("session-beta");
2062
2063	assert!(get_page(namespace, &ref_id).is_none());
2064	}
2065
2066	#[test]
2067	fn direct_urls_remain_compatible_open_refs() {
2068	assert!(looks_like_url("https://example.com"));
2069	assert!(looks_like_url("http://example.com"));
2070	assert!(!looks_like_url("turn0search0"));
2071	}
2072
2073	#[tokio::test]
2074	async fn network_policy_denies_direct_open_url() {
2075	use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider};
2076
2077	let policy = NetworkPolicy {
2078	default: Decision::Deny.into(),
2079	allow: vec!["api.deepseek.com".to_string()],
2080	deny: vec![],
2081	proxy: Vec::new(),
2082	proxy_fake_ip_cidrs: Vec::new(),
2083	audit: false,
2084	};
2085	let decider = NetworkPolicyDecider::new(policy, None);
2086	let ctx = ToolContext::new(PathBuf::from(".")).with_network_policy(decider);
2087
2088	let err = fetch_page("https://example.com/private", 5_000, &ctx)
2089	.await
2090	.expect_err("blocked host should fail");
2091	assert!(format!("{err}").contains("blocked by network policy"));
2092	}
2093
2094	fn ssrf_ctx() -> ToolContext {
2095	ToolContext::new(PathBuf::from("."))
2096	}
2097
2098	#[tokio::test]
2099	async fn open_refuses_loopback_ip_url() {
2100	let err = resolve_or_fetch_page("http://127.0.0.1/", 5_000, &ssrf_ctx())
2101	.await
2102	.expect_err("loopback open must be refused");
2103	assert!(
2104	format!("{err}").contains("restricted address"),
2105	"expected restricted-address error; got {err}"
2106	);
2107	}
2108
2109	#[tokio::test]
2110	async fn open_resolves_shared_citation_refs_only_in_their_session() {
2111	let namespace = "shared-citation-open-session";
2112	let citation = crate::tools::web::citations::register(
2113	namespace,
2114	"http://127.0.0.1/private",
2115	Some("Private target"),
2116	)
2117	.expect("valid HTTP citation metadata");
2118	let owner_context = ToolContext::new(PathBuf::from(".")).with_state_namespace(namespace);
2119	let owner_error = resolve_or_fetch_page(&citation.ref_id, 5_000, &owner_context)
2120	.await
2121	.expect_err("resolved citation must still use the SSRF guard");
2122	assert!(format!("{owner_error}").contains("restricted address"));
2123
2124	let foreign_context = ToolContext::new(PathBuf::from("."))
2125	.with_state_namespace("shared-citation-foreign-session");
2126	let foreign_error = resolve_or_fetch_page(&citation.ref_id, 5_000, &foreign_context)
2127	.await
2128	.expect_err("foreign session must not resolve citation ref");
2129	assert!(format!("{foreign_error}").contains("Unknown ref_id"));
2130	}
2131
2132	#[tokio::test]
2133	async fn open_refuses_private_range_ip_url() {
2134	let err = resolve_or_fetch_page("http://192.168.1.50/admin", 5_000, &ssrf_ctx())
2135	.await
2136	.expect_err("private-range open must be refused");
2137	assert!(
2138	format!("{err}").contains("restricted address"),
2139	"expected restricted-address error; got {err}"
2140	);
2141	}
2142
2143	#[tokio::test]
2144	async fn open_refuses_metadata_endpoint_ip_url() {
2145	let err = resolve_or_fetch_page(
2146	"http://169.254.169.254/latest/meta-data",
2147	5_000,
2148	&ssrf_ctx(),
2149	)
2150	.await
2151	.expect_err("cloud metadata open must be refused");
2152	assert!(
2153	format!("{err}").contains("restricted address"),
2154	"expected restricted-address error; got {err}"
2155	);
2156	}
2157
2158	#[tokio::test]
2159	async fn open_refuses_redirect_from_public_host_to_private_ip() {
2160	use wiremock::matchers::method;
2161	use wiremock::{Mock, MockServer, ResponseTemplate};
2162
2163	// Use a public-looking hostname pinned to the local fixture for the
2164	// already-validated first hop. The redirect itself still goes through
2165	// the real shared guard inside fetch_page's redirect loop.
2166	let server = MockServer::start().await;
2167	let private_location = "http://10.0.0.5/internal";
2168	Mock::given(method("GET"))
2169	.respond_with(ResponseTemplate::new(302).insert_header("Location", private_location))
2170	.mount(&server)
2171	.await;
2172	let host = "public-redirect.example.test";
2173	let initial_url = format!("http://{host}:{}/", server.address().port());
2174	let pin = Some((host.to_string(), "127.0.0.1".parse().unwrap()));
2175
2176	let err = fetch_page_with_initial_pin(&initial_url, 5_000, &ssrf_ctx(), Some(pin))
2177	.await
2178	.expect_err("guarded redirect to private IP must be refused");
2179	assert!(
2180	format!("{err}").contains("restricted address"),
2181	"redirect loop must surface the shared guard rejection; got {err}"
2182	);
2183	}
2184
2185	#[tokio::test(flavor = "current_thread")]
2186	async fn execute_fails_fast_when_no_op_key_matches() {
2187	// #5123-class: the natural {"query": …} shape matched no op key and
2188	// previously returned an empty SUCCESS ({"warnings":[]}).
2189	let _lock = WEB_RUN_TEST_LOCK.lock().await;
2190	reset_web_run_state();
2191	let tmp = tempfile::tempdir().expect("tempdir");
2192	let context = ToolContext::new(tmp.path());
2193
2194	let err = WebRunTool
2195	.execute(json!({"query": "rust async runtime"}), &context)
2196	.await
2197	.expect_err("unmatched op keys must fail, not return empty success");
2198	let message = format!("{err}");
2199	assert!(message.contains("performed no operation"), "{message}");
2200	assert!(message.contains("search_query"), "{message}");
2201	assert!(message.contains("query"), "{message}");
2202
2203	let err = WebRunTool
2204	.execute(json!({}), &context)
2205	.await
2206	.expect_err("empty input must fail fast");
2207	assert!(format!("{err}").contains("performed no operation"), "{err}");
2208	}
2209	}
2210

2210 lines RUST