返回 DeepSeek-TUI-2026
fetch_url.rs
根目录 / crates / tui / src / tools / fetch_url.rs
1 //! Direct-fetch HTTP tool. Complements `web_search` for cases where the user
2 //! already knows the URL — a known repo, a blog post, a spec page — and
3 //! search is overkill or actively unhelpful.
4 //!
5 //! Returns a structured `{url, status, content_type, content, truncated}`
6 //! payload. HTML responses are stripped to readable text by default
7 //! (`format = "markdown"`); pass `format = "raw"` to keep the bytes intact
8 //! when the model wants to do its own parsing.
9
10 use super::spec::{
11 ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64,
12 };
13 use crate::network_policy::{Decision, host_from_url};
14 use async_trait::async_trait;
15 use regex::Regex;
16 use serde::Serialize;
17 use serde_json::{Value, json};
18 use std::sync::OnceLock;
19 use std::time::Duration;
20
21 const DEFAULT_MAX_BYTES: u64 = 1_000_000;
22 const HARD_MAX_BYTES: u64 = 10 * 1024 * 1024;
23 const DEFAULT_TIMEOUT_MS: u64 = 15_000;
24 const HARD_MAX_TIMEOUT_MS: u64 = 60_000;
25 const MAX_REDIRECTS: usize = 5;
26 const USER_AGENT: &str =
27 "Mozilla/5.0 (compatible; deepseek-tui/0.5; +https://github.com/Hmbown/DeepSeek-TUI)";
28
29 static SCRIPT_RE: OnceLock<Regex> = OnceLock::new();
30 static STYLE_RE: OnceLock<Regex> = OnceLock::new();
31 static TAG_RE: OnceLock<Regex> = OnceLock::new();
32 static WHITESPACE_RE: OnceLock<Regex> = OnceLock::new();
33
34 fn script_re() -> &'static Regex {
35 SCRIPT_RE.get_or_init(|| Regex::new(r"(?is)<script[^>]*>.*?</script>").expect("script re"))
36 }
37 fn style_re() -> &'static Regex {
38 STYLE_RE.get_or_init(|| Regex::new(r"(?is)<style[^>]*>.*?</style>").expect("style re"))
39 }
40 fn tag_re() -> &'static Regex {
41 TAG_RE.get_or_init(|| Regex::new(r"<[^>]+>").expect("tag re"))
42 }
43 fn whitespace_re() -> &'static Regex {
44 WHITESPACE_RE.get_or_init(|| Regex::new(r"\s+").expect("ws re"))
45 }
46
47 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
48 enum Format {
49 Text,
50 Markdown,
51 Raw,
52 }
53
54 impl Format {
55 fn parse(value: Option<&str>) -> Result<Self, ToolError> {
56 match value
57 .unwrap_or("markdown")
58 .trim()
59 .to_ascii_lowercase()
60 .as_str()
61 {
62 "text" | "txt" | "plain" => Ok(Self::Text),
63 "markdown" | "md" => Ok(Self::Markdown),
64 "raw" | "html" | "bytes" => Ok(Self::Raw),
65 other => Err(ToolError::invalid_input(format!(
66 "unknown format `{other}` (allowed: text, markdown, raw)"
67 ))),
68 }
69 }
70 }
71
72 #[derive(Debug, Serialize)]
73 struct FetchResponse {
74 url: String,
75 status: u16,
76 content_type: String,
77 content: String,
78 truncated: bool,
79 }
80
81 pub struct FetchUrlTool;
82
83 #[async_trait]
84 impl ToolSpec for FetchUrlTool {
85 fn name(&self) -> &'static str {
86 "fetch_url"
87 }
88
89 fn description(&self) -> &'static str {
90 "Fetch a known URL directly (HTTP GET) and return its content. Use this when the user gives a URL or you already know the canonical link — it's faster and more reliable than web_search for known pages."
91 }
92
93 fn input_schema(&self) -> Value {
94 json!({
95 "type": "object",
96 "properties": {
97 "url": {
98 "type": "string",
99 "description": "Absolute HTTP/HTTPS URL to fetch."
100 },
101 "format": {
102 "type": "string",
103 "enum": ["text", "markdown", "raw"],
104 "description": "Post-processing for the response body. `markdown` (default) and `text` strip HTML tags to readable text; `raw` returns the body bytes as-is."
105 },
106 "max_bytes": {
107 "type": "integer",
108 "description": "Truncate response body after this many bytes (default 1,000,000; hard max 10,485,760)."
109 },
110 "timeout_ms": {
111 "type": "integer",
112 "description": "Request timeout in milliseconds (default 15,000; max 60,000)."
113 }
114 },
115 "required": ["url"]
116 })
117 }
118
119 fn capabilities(&self) -> Vec<ToolCapability> {
120 vec![ToolCapability::ReadOnly, ToolCapability::Network]
121 }
122
123 fn approval_requirement(&self) -> ApprovalRequirement {
124 ApprovalRequirement::Auto
125 }
126
127 async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
128 let url = input
129 .get("url")
130 .and_then(Value::as_str)
131 .ok_or_else(|| ToolError::invalid_input("`url` is required"))?
132 .trim()
133 .to_string();
134
135 if url.is_empty() {
136 return Err(ToolError::invalid_input("`url` cannot be empty"));
137 }
138 let scheme_ok = url.starts_with("http://") || url.starts_with("https://");
139 if !scheme_ok {
140 return Err(ToolError::invalid_input(
141 "only http:// and https:// URLs are supported",
142 ));
143 }
144
145 // Extract host once for reuse across network policy + SSRF checks.
146 let url_host = host_from_url(&url);
147
148 // Per-domain network policy gate (#135). If no policy is attached
149 // (e.g. ad-hoc tests), behavior is permissive — match pre-v0.7.0.
150 if let Some(decider) = context.network_policy.as_ref()
151 && let Some(ref host) = url_host
152 {
153 match decider.evaluate(host, "fetch_url") {
154 Decision::Allow => {}
155 Decision::Deny => {
156 return Err(ToolError::permission_denied(format!(
157 "network call to '{host}' blocked by network policy"
158 )));
159 }
160 Decision::Prompt => {
161 return Err(ToolError::permission_denied(format!(
162 "network call to '{host}' requires approval; \
163 re-run after `/network allow {host}` or set network.default = \"allow\" in config"
164 )));
165 }
166 }
167 }
168
169 // SSRF protection: resolve hostname and reject private/link-local/loopback IPs.
170 // Prevents LLM-prompted requests to cloud metadata (169.254.169.254),
171 // localhost services, and internal networks.
172 // Pin the validated IP via ClientBuilder::resolve() to close the DNS rebinding
173 // TOCTOU window — reqwest will use the pinned IP instead of re-resolving.
174 let mut dns_pinning = None; // (hostname, validated_ip)
175 if let Some(host) = &url_host {
176 if host == "localhost" || host == "localhost.localdomain" {
177 return Err(ToolError::permission_denied(
178 "requests to localhost are not allowed",
179 ));
180 }
181 if let Ok(ip) = host.parse::<std::net::IpAddr>() {
182 if is_restricted_ip(&ip) {
183 return Err(ToolError::permission_denied(format!(
184 "IP {ip} is a restricted address (private/loopback/link-local)"
185 )));
186 }
187 } else if let Ok(addrs) = tokio::net::lookup_host((&**host, 0u16)).await {
188 let mut first_valid: Option<std::net::IpAddr> = None;
189 for addr in addrs {
190 if is_restricted_ip(&addr.ip()) {
191 return Err(ToolError::permission_denied(format!(
192 "resolved IP {} is a restricted address (private/loopback/link-local)",
193 addr.ip()
194 )));
195 }
196 if first_valid.is_none() {
197 first_valid = Some(addr.ip());
198 }
199 }
200 if let Some(validated_ip) = first_valid {
201 dns_pinning = Some((host.clone(), validated_ip));
202 }
203 }
204 // If DNS resolution fails, let the HTTP request proceed and fail naturally.
205 }
206
207 let format = Format::parse(input.get("format").and_then(Value::as_str))?;
208 let max_bytes = optional_u64(&input, "max_bytes", DEFAULT_MAX_BYTES).min(HARD_MAX_BYTES);
209 let timeout_ms =
210 optional_u64(&input, "timeout_ms", DEFAULT_TIMEOUT_MS).min(HARD_MAX_TIMEOUT_MS);
211
212 let mut client_builder = reqwest::Client::builder()
213 .timeout(Duration::from_millis(timeout_ms))
214 .user_agent(USER_AGENT)
215 .redirect(reqwest::redirect::Policy::limited(MAX_REDIRECTS));
216
217 // Pin validated IP to prevent DNS rebinding (TOCTOU) — reqwest will
218 // connect to the validated IP directly instead of re-resolving.
219 if let Some((hostname, validated_ip)) = dns_pinning {
220 client_builder =
221 client_builder.resolve(&hostname, std::net::SocketAddr::new(validated_ip, 0));
222 }
223
224 let client = client_builder.build().map_err(|e| {
225 ToolError::execution_failed(format!("failed to build HTTP client: {e}"))
226 })?;
227
228 let resp = client
229 .get(&url)
230 .header("Accept", "text/html,text/plain,application/json,*/*;q=0.5")
231 .header("Accept-Language", "en-US,en;q=0.5")
232 .send()
233 .await
234 .map_err(|e| ToolError::execution_failed(format!("request failed: {e}")))?;
235
236 let final_url = resp.url().to_string();
237 let status = resp.status();
238 let content_type = resp
239 .headers()
240 .get(reqwest::header::CONTENT_TYPE)
241 .and_then(|v| v.to_str().ok())
242 .unwrap_or("application/octet-stream")
243 .to_string();
244
245 let bytes = resp
246 .bytes()
247 .await
248 .map_err(|e| ToolError::execution_failed(format!("failed to read body: {e}")))?;
249 let total_bytes = bytes.len() as u64;
250 let truncated = total_bytes > max_bytes;
251 let usable = if truncated {
252 &bytes[..max_bytes as usize]
253 } else {
254 &bytes[..]
255 };
256
257 let body_text = String::from_utf8_lossy(usable).to_string();
258 let processed = match format {
259 Format::Raw => body_text,
260 Format::Text | Format::Markdown => {
261 if content_type.contains("text/html") || body_text.contains("<html") {
262 html_to_text(&body_text)
263 } else {
264 body_text
265 }
266 }
267 };
268
269 let response = FetchResponse {
270 url: final_url,
271 status: status.as_u16(),
272 content_type,
273 content: processed,
274 truncated,
275 };
276
277 if !status.is_success() {
278 // Don't `Err` on 4xx/5xx — the caller often wants to see the body
279 // (e.g. a JSON error envelope). Mark the result as a failure so the
280 // engine renders it as such.
281 return Ok(ToolResult {
282 content: serde_json::to_string_pretty(&response).map_err(|e| {
283 ToolError::execution_failed(format!("failed to serialize response: {e}"))
284 })?,
285 success: false,
286 metadata: None,
287 });
288 }
289
290 ToolResult::json(&response)
291 .map_err(|e| ToolError::execution_failed(format!("failed to serialize response: {e}")))
292 }
293 }
294
295 /// Check if an IP address is loopback, private, link-local, cloud-metadata,
296 /// multicast, or reserved — all addresses that should not be reachable via
297 /// an LLM-initiated fetch_url request (SSRF prevention).
298 fn is_restricted_ip(ip: &std::net::IpAddr) -> bool {
299 match ip {
300 std::net::IpAddr::V4(v4) => {
301 v4.is_loopback()
302 || v4.is_private()
303 || v4.is_link_local()
304 || v4.is_multicast()
305 || v4.is_broadcast()
306 || v4.is_unspecified()
307 // 100.64.0.0/10 — Carrier-grade NAT (CGNAT / shared address space)
308 || matches!(v4.octets(), [100, 64..=127, ..])
309 // 169.254.169.254 — cloud metadata (AWS/GCP/Azure)
310 || *ip == std::net::IpAddr::V4(std::net::Ipv4Addr::new(169, 254, 169, 254))
311 // 198.18.0.0/15 — IETF benchmark testing
312 || matches!(v4.octets(), [198, 18..=19, ..])
313 // 240.0.0.0/4 — reserved (former Class E)
314 || v4.octets()[0] >= 240
315 }
316 std::net::IpAddr::V6(v6) => {
317 // IPv4-mapped IPv6 addresses (::ffff:a.b.c.d) — unwrap and check as IPv4
318 // to prevent bypass via ::ffff:127.0.0.1 etc.
319 if v6.is_unspecified()
320 || matches!(v6.octets(), [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, ..])
321 {
322 return true;
323 }
324 if let Some(v4) = v6.to_ipv4_mapped() {
325 return is_restricted_ip(&std::net::IpAddr::V4(v4));
326 }
327 v6.is_loopback()
328 || v6.is_multicast()
329 || matches!(v6.segments(), [0xfc00..=0xfdff, ..]) // ULA fc00::/7
330 || matches!(v6.segments(), [0xfe80..=0xfebf, ..]) // Link-local fe80::/10
331 }
332 }
333 }
334
335 /// Strip `<script>` / `<style>` blocks, drop remaining tags, and collapse
336 /// whitespace. Good enough for "let the model read this page" — not a full
337 /// HTML-to-Markdown converter.
338 fn html_to_text(html: &str) -> String {
339 let no_script = script_re().replace_all(html, "");
340 let no_style = style_re().replace_all(&no_script, "");
341 let no_tags = tag_re().replace_all(&no_style, " ");
342 let decoded = decode_entities(&no_tags);
343 whitespace_re()
344 .replace_all(&decoded, " ")
345 .trim()
346 .to_string()
347 }
348
349 /// Decode the handful of HTML entities we expect to hit in stripped text.
350 /// Pulling in `html-escape` for the long tail isn't worth the dep weight.
351 fn decode_entities(s: &str) -> String {
352 s.replace("&amp;", "&")
353 .replace("&lt;", "<")
354 .replace("&gt;", ">")
355 .replace("&quot;", "\"")
356 .replace("&#39;", "'")
357 .replace("&apos;", "'")
358 .replace("&nbsp;", " ")
359 }
360
361 #[cfg(test)]
362 mod tests {
363 use super::*;
364 use crate::tools::spec::ToolContext;
365 use std::path::PathBuf;
366
367 fn ctx() -> ToolContext {
368 ToolContext::new(PathBuf::from("."))
369 }
370
371 #[test]
372 fn html_to_text_strips_scripts_styles_and_tags() {
373 let html = r#"
374 <html>
375 <head>
376 <style>body { color: red; }</style>
377 <script>alert("nope");</script>
378 </head>
379 <body>
380 <h1>Hello &amp; welcome</h1>
381 <p>This is <b>important</b>.</p>
382 </body>
383 </html>
384 "#;
385 let text = html_to_text(html);
386 assert!(text.contains("Hello & welcome"));
387 assert!(text.contains("This is important"));
388 assert!(!text.contains("alert"));
389 assert!(!text.contains("color: red"));
390 }
391
392 #[test]
393 fn format_parse_accepts_aliases_and_rejects_unknown() {
394 assert_eq!(Format::parse(Some("markdown")).unwrap(), Format::Markdown);
395 assert_eq!(Format::parse(Some("MD")).unwrap(), Format::Markdown);
396 assert_eq!(Format::parse(Some("text")).unwrap(), Format::Text);
397 assert_eq!(Format::parse(Some("raw")).unwrap(), Format::Raw);
398 assert_eq!(Format::parse(None).unwrap(), Format::Markdown);
399 assert!(Format::parse(Some("yaml")).is_err());
400 }
401
402 #[tokio::test]
403 async fn rejects_non_http_schemes() {
404 let tool = FetchUrlTool;
405 let res = tool
406 .execute(json!({"url": "file:///etc/passwd"}), &ctx())
407 .await;
408 let err = res.unwrap_err();
409 assert!(format!("{err:?}").contains("http"));
410 }
411
412 #[tokio::test]
413 async fn rejects_empty_url() {
414 let tool = FetchUrlTool;
415 let res = tool.execute(json!({"url": " "}), &ctx()).await;
416 assert!(res.is_err());
417 }
418
419 #[tokio::test]
420 async fn rejects_missing_url() {
421 let tool = FetchUrlTool;
422 let res = tool.execute(json!({}), &ctx()).await;
423 assert!(res.is_err());
424 }
425
426 #[test]
427 fn rejects_private_localhost_literal() {
428 assert!(is_restricted_ip(&"127.0.0.1".parse().unwrap()));
429 assert!(is_restricted_ip(&"::1".parse().unwrap()));
430 }
431
432 #[test]
433 fn rejects_private_rfc1918() {
434 assert!(is_restricted_ip(&"10.0.0.1".parse().unwrap()));
435 assert!(is_restricted_ip(&"172.16.0.1".parse().unwrap()));
436 assert!(is_restricted_ip(&"192.168.1.1".parse().unwrap()));
437 }
438
439 #[test]
440 fn rejects_cloud_metadata() {
441 assert!(is_restricted_ip(&"169.254.169.254".parse().unwrap()));
442 }
443
444 #[test]
445 fn rejects_link_local() {
446 assert!(is_restricted_ip(&"169.254.1.1".parse().unwrap()));
447 }
448
449 #[test]
450 fn rejects_cgnat() {
451 assert!(is_restricted_ip(&"100.64.0.1".parse().unwrap()));
452 assert!(!is_restricted_ip(&"100.63.0.1".parse().unwrap()));
453 assert!(!is_restricted_ip(&"100.128.0.1".parse().unwrap()));
454 }
455
456 #[test]
457 fn rejects_ipv6_ula() {
458 assert!(is_restricted_ip(&"fc00::1".parse().unwrap()));
459 assert!(is_restricted_ip(&"fd12:3456::1".parse().unwrap()));
460 }
461
462 #[test]
463 fn rejects_ipv4_mapped_ipv6() {
464 // ::ffff:127.0.0.1 — IPv4-mapped IPv6 loopback bypass
465 assert!(is_restricted_ip(&"::ffff:127.0.0.1".parse().unwrap()));
466 assert!(is_restricted_ip(&"::ffff:10.0.0.1".parse().unwrap()));
467 assert!(is_restricted_ip(&"::ffff:169.254.169.254".parse().unwrap()));
468 assert!(is_restricted_ip(&"::ffff:192.168.1.1".parse().unwrap()));
469 // :: (unspecified)
470 assert!(is_restricted_ip(&"::".parse().unwrap()));
471 }
472
473 #[test]
474 fn allows_public_ips() {
475 assert!(!is_restricted_ip(&"8.8.8.8".parse().unwrap()));
476 assert!(!is_restricted_ip(&"1.1.1.1".parse().unwrap()));
477 assert!(!is_restricted_ip(&"93.184.216.34".parse().unwrap()));
478 assert!(!is_restricted_ip(&"2606:4700::1".parse().unwrap()));
479 }
480
481 #[tokio::test]
482 async fn rejects_localhost_hostname() {
483 let tool = FetchUrlTool;
484 let res = tool
485 .execute(json!({"url": "http://localhost:8080/admin"}), &ctx())
486 .await;
487 let err = res.unwrap_err();
488 assert!(format!("{err}").contains("localhost"));
489 }
490
491 #[tokio::test]
492 async fn network_policy_denies_blocked_host() {
493 use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider};
494 let policy = NetworkPolicy {
495 default: Decision::Deny.into(),
496 allow: vec!["api.deepseek.com".to_string()],
497 deny: vec![],
498 audit: false,
499 };
500 let decider = NetworkPolicyDecider::new(policy, None);
501 let ctx = ToolContext::new(PathBuf::from(".")).with_network_policy(decider);
502 let tool = FetchUrlTool;
503 let res = tool
504 .execute(json!({"url": "https://example.com/foo"}), &ctx)
505 .await;
506 let err = res.expect_err("blocked host should fail");
507 assert!(format!("{err}").contains("blocked"));
508 }
509 }
510
510 lines RUST