| 1 | //! Content-type routing and readable-document extraction for web tools. |
| 2 | //! |
| 3 | //! Networking deliberately lives elsewhere. This module accepts already |
| 4 | //! fetched bytes and turns them into one normalized document so `fetch_url` |
| 5 | //! and `web.run` cannot disagree about HTML, Markdown, PDF, or media handling. |
| 6 | |
| 7 | use std::sync::OnceLock; |
| 8 | |
| 9 | use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE}; |
| 10 | use regex::Regex; |
| 11 | use tokio_util::sync::CancellationToken; |
| 12 | |
| 13 | use crate::tools::spec::ToolError; |
| 14 | |
| 15 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 16 | pub(crate) enum DocumentKind { |
| 17 | Html, |
| 18 | Markdown, |
| 19 | Text, |
| 20 | Pdf, |
| 21 | Media, |
| 22 | } |
| 23 | |
| 24 | #[derive(Debug, Clone)] |
| 25 | pub(crate) struct ExtractedDocument { |
| 26 | pub(crate) kind: DocumentKind, |
| 27 | pub(crate) title: Option<String>, |
| 28 | pub(crate) text: String, |
| 29 | pub(crate) markdown: String, |
| 30 | /// Readability-cleaned HTML. `web.run` consumes this to retain clickable |
| 31 | /// links while avoiding page chrome and consent-banner noise. |
| 32 | pub(crate) cleaned_html: Option<String>, |
| 33 | pub(crate) pdf_pages: Option<Vec<Vec<String>>>, |
| 34 | /// Validated extension for image/audio/video artifacts. |
| 35 | pub(crate) media_extension: Option<&'static str>, |
| 36 | } |
| 37 | |
| 38 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 39 | struct MediaSignature { |
| 40 | extension: &'static str, |
| 41 | family: MediaFamily, |
| 42 | } |
| 43 | |
| 44 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 45 | enum MediaFamily { |
| 46 | Image, |
| 47 | Audio, |
| 48 | Video, |
| 49 | } |
| 50 | |
| 51 | static TITLE_RE: OnceLock<Regex> = OnceLock::new(); |
| 52 | static FALLBACK_RE: OnceLock<Vec<Regex>> = OnceLock::new(); |
| 53 | static PAGE_CHROME_RE: OnceLock<Regex> = OnceLock::new(); |
| 54 | static TAG_RE: OnceLock<Regex> = OnceLock::new(); |
| 55 | static WHITESPACE_RE: OnceLock<Regex> = OnceLock::new(); |
| 56 | |
| 57 | /// HTML's encoding declaration prescan is intentionally small. Keeping the |
| 58 | /// bound here prevents a late body string, script, or injected fragment from |
| 59 | /// changing how an already-started document is decoded. |
| 60 | const HTML_ENCODING_SNIFF_BYTES: usize = 1_024; |
| 61 | |
| 62 | pub(crate) async fn extract_document( |
| 63 | url: &str, |
| 64 | content_type: Option<&str>, |
| 65 | bytes: &[u8], |
| 66 | cancel: Option<&CancellationToken>, |
| 67 | ) -> Result<ExtractedDocument, ToolError> { |
| 68 | extract_document_with_pdf_command( |
| 69 | url, |
| 70 | content_type, |
| 71 | bytes, |
| 72 | super::super::pdf::PdfTextCommand::system(cancel), |
| 73 | ) |
| 74 | .await |
| 75 | } |
| 76 | |
| 77 | pub(crate) async fn extract_document_with_pdf_command( |
| 78 | url: &str, |
| 79 | content_type: Option<&str>, |
| 80 | bytes: &[u8], |
| 81 | pdf_command: super::super::pdf::PdfTextCommand<'_>, |
| 82 | ) -> Result<ExtractedDocument, ToolError> { |
| 83 | let declared = normalized_content_type(content_type); |
| 84 | let declared = declared.as_deref(); |
| 85 | |
| 86 | if bytes.is_empty() { |
| 87 | return Ok(ExtractedDocument { |
| 88 | kind: DocumentKind::Text, |
| 89 | title: None, |
| 90 | text: String::new(), |
| 91 | markdown: String::new(), |
| 92 | cleaned_html: None, |
| 93 | pdf_pages: None, |
| 94 | media_extension: None, |
| 95 | }); |
| 96 | } |
| 97 | |
| 98 | if validate_pdf_response(url, content_type, bytes)? { |
| 99 | return extract_pdf(bytes, pdf_command).await; |
| 100 | } |
| 101 | |
| 102 | if let Some(signature) = sniff_media(bytes) { |
| 103 | if let Some(declared_family) = declared_media_family(declared) |
| 104 | && declared_family != signature.family |
| 105 | { |
| 106 | return Err(ToolError::execution_failed(format!( |
| 107 | "Response media type `{}` did not match its bytes", |
| 108 | declared.unwrap_or("unknown") |
| 109 | ))); |
| 110 | } |
| 111 | return Ok(ExtractedDocument { |
| 112 | kind: DocumentKind::Media, |
| 113 | title: None, |
| 114 | text: String::new(), |
| 115 | markdown: String::new(), |
| 116 | cleaned_html: None, |
| 117 | pdf_pages: None, |
| 118 | media_extension: Some(signature.extension), |
| 119 | }); |
| 120 | } |
| 121 | |
| 122 | if declared_media_family(declared).is_some() { |
| 123 | return Err(ToolError::execution_failed(format!( |
| 124 | "Response claimed media type `{}`, but its bytes did not match a supported media signature", |
| 125 | declared.unwrap_or("unknown") |
| 126 | ))); |
| 127 | } |
| 128 | |
| 129 | let sniff_html = should_sniff_html_encoding(declared, url, bytes); |
| 130 | let body = decode_response_body(bytes, content_type, sniff_html)?; |
| 131 | if sniff_html || is_html(declared, url, &body) { |
| 132 | return extract_html(url, &body); |
| 133 | } |
| 134 | if is_markdown(declared, url) { |
| 135 | return Ok(ExtractedDocument { |
| 136 | kind: DocumentKind::Markdown, |
| 137 | title: markdown_title(&body), |
| 138 | text: body.clone(), |
| 139 | markdown: body, |
| 140 | cleaned_html: None, |
| 141 | pdf_pages: None, |
| 142 | media_extension: None, |
| 143 | }); |
| 144 | } |
| 145 | if is_textual(declared, url) { |
| 146 | return Ok(ExtractedDocument { |
| 147 | kind: DocumentKind::Text, |
| 148 | title: None, |
| 149 | text: body.clone(), |
| 150 | markdown: body, |
| 151 | cleaned_html: None, |
| 152 | pdf_pages: None, |
| 153 | media_extension: None, |
| 154 | }); |
| 155 | } |
| 156 | |
| 157 | Err(ToolError::execution_failed(format!( |
| 158 | "Unsupported binary response type `{}`; use a dedicated download tool", |
| 159 | declared.unwrap_or("unknown") |
| 160 | ))) |
| 161 | } |
| 162 | |
| 163 | pub(crate) fn validate_pdf_response( |
| 164 | url: &str, |
| 165 | content_type: Option<&str>, |
| 166 | bytes: &[u8], |
| 167 | ) -> Result<bool, ToolError> { |
| 168 | let declared = normalized_content_type(content_type); |
| 169 | let declared = declared.as_deref(); |
| 170 | let signed = looks_like_pdf(bytes); |
| 171 | if signed && declared_media_family(declared).is_some() { |
| 172 | return Err(ToolError::execution_failed(format!( |
| 173 | "Response media type `{}` did not match its PDF bytes", |
| 174 | declared.unwrap_or("unknown") |
| 175 | ))); |
| 176 | } |
| 177 | let claimed = signed || declared == Some("application/pdf") || url_is_pdf(url); |
| 178 | if claimed && !signed { |
| 179 | return Err(ToolError::execution_failed( |
| 180 | "Response claimed to be a PDF, but its bytes did not contain a PDF signature", |
| 181 | )); |
| 182 | } |
| 183 | Ok(claimed) |
| 184 | } |
| 185 | |
| 186 | fn extract_html(url: &str, html: &str) -> Result<ExtractedDocument, ToolError> { |
| 187 | let parsed_url = reqwest::Url::parse(url) |
| 188 | .map_err(|err| ToolError::invalid_input(format!("invalid URL: {err}")))?; |
| 189 | let original_title = html_title(html); |
| 190 | |
| 191 | // Readability-based extraction was removed to consolidate the HTML |
| 192 | // pipeline onto a single stack (htmd 0.5 + html5ever 0.38). The |
| 193 | // previous dual-stack (readability 0.3 / html5ever 0.26 + htmd / 0.38) |
| 194 | // compiled two incompatible html5ever/markup5ever trees. The fallback |
| 195 | // main-content regex retains the meaningful-content signal used by the |
| 196 | // tests (≥32 non-whitespace chars, ≥5 words) without the duplicate tree. |
| 197 | let cleaned_html = fallback_main_html(html).ok_or_else(|| js_required_error(url))?; |
| 198 | let markdown = html_to_markdown_with_base_url(&cleaned_html, &parsed_url).map_err(|err| { |
| 199 | ToolError::execution_failed(format!( |
| 200 | "Failed to convert readable HTML to Markdown: {err}" |
| 201 | )) |
| 202 | })?; |
| 203 | let text = html_to_plain_text(&cleaned_html); |
| 204 | |
| 205 | if !meaningful_text(&text) && !meaningful_text(&markdown) { |
| 206 | return Err(js_required_error(url)); |
| 207 | } |
| 208 | |
| 209 | let title = original_title; |
| 210 | |
| 211 | Ok(ExtractedDocument { |
| 212 | kind: DocumentKind::Html, |
| 213 | title, |
| 214 | text, |
| 215 | markdown, |
| 216 | cleaned_html: Some(cleaned_html), |
| 217 | pdf_pages: None, |
| 218 | media_extension: None, |
| 219 | }) |
| 220 | } |
| 221 | |
| 222 | /// Resolve relative anchors in `htmd`'s parsed DOM, not with an HTML regex. |
| 223 | /// Absolute, fragment, non-HTTP, and malformed destinations fall through to |
| 224 | /// the built-in handler unchanged. |
| 225 | fn html_to_markdown_with_base_url( |
| 226 | html: &str, |
| 227 | base_url: &reqwest::Url, |
| 228 | ) -> Result<String, std::io::Error> { |
| 229 | let base_url = base_url.clone(); |
| 230 | htmd::HtmlToMarkdown::builder() |
| 231 | .add_handler( |
| 232 | vec!["a"], |
| 233 | move |handlers: &dyn htmd::element_handler::Handlers, element: htmd::Element<'_>| { |
| 234 | let href = element.attrs.iter().find_map(|attr| { |
| 235 | (attr.name.local.as_ref() == "href").then(|| attr.value.to_string()) |
| 236 | }); |
| 237 | let Some(destination) = href |
| 238 | .as_deref() |
| 239 | .and_then(|href| resolve_relative_http_href(&base_url, href)) |
| 240 | else { |
| 241 | return handlers.fallback(element); |
| 242 | }; |
| 243 | let content = handlers.walk_children(element.node).content; |
| 244 | let trailing = &content[content.trim_end().len()..]; |
| 245 | let destination = destination.replace('(', "\\(").replace(')', "\\)"); |
| 246 | let title = element |
| 247 | .attrs |
| 248 | .iter() |
| 249 | .find_map(|attr| { |
| 250 | (attr.name.local.as_ref() == "title").then(|| { |
| 251 | attr.value |
| 252 | .split_whitespace() |
| 253 | .collect::<Vec<_>>() |
| 254 | .join(" ") |
| 255 | .replace('"', "\\\"") |
| 256 | }) |
| 257 | }) |
| 258 | .map_or_else(String::new, |title| format!(" \"{title}\"")); |
| 259 | Some(format!("[{}]({destination}{title}){trailing}", content.trim()).into()) |
| 260 | }, |
| 261 | ) |
| 262 | .build() |
| 263 | .convert(html) |
| 264 | } |
| 265 | |
| 266 | fn resolve_relative_http_href(base_url: &reqwest::Url, href: &str) -> Option<String> { |
| 267 | if !matches!(base_url.scheme(), "http" | "https") { |
| 268 | return None; |
| 269 | } |
| 270 | |
| 271 | let href = href.trim(); |
| 272 | if href.is_empty() || href.starts_with('#') || reqwest::Url::parse(href).is_ok() { |
| 273 | return None; |
| 274 | } |
| 275 | |
| 276 | base_url.join(href).ok().map(Into::into) |
| 277 | } |
| 278 | |
| 279 | fn fallback_main_html(html: &str) -> Option<String> { |
| 280 | let page_chrome = PAGE_CHROME_RE.get_or_init(|| { |
| 281 | Regex::new(concat!( |
| 282 | r"(?is)(?:<script(?:\s[^>]*)?>.*?</script\s*>", |
| 283 | r"|<style(?:\s[^>]*)?>.*?</style\s*>", |
| 284 | r"|<noscript(?:\s[^>]*)?>.*?</noscript\s*>", |
| 285 | r"|<nav(?:\s[^>]*)?>.*?</nav\s*>", |
| 286 | r"|<header(?:\s[^>]*)?>.*?</header\s*>", |
| 287 | r"|<footer(?:\s[^>]*)?>.*?</footer\s*>", |
| 288 | r"|<aside(?:\s[^>]*)?>.*?</aside\s*>", |
| 289 | r"|<form(?:\s[^>]*)?>.*?</form\s*>)", |
| 290 | )) |
| 291 | .expect("page chrome regex") |
| 292 | }); |
| 293 | for re in FALLBACK_RE.get_or_init(|| { |
| 294 | ["article", "main", "body"] |
| 295 | .into_iter() |
| 296 | .map(|tag| { |
| 297 | Regex::new(&format!(r"(?is)<{tag}(?:\s[^>]*)?>(.*?)</{tag}\s*>")) |
| 298 | .expect("fallback element regex") |
| 299 | }) |
| 300 | .collect() |
| 301 | }) { |
| 302 | let Some(capture) = re.captures(html) else { |
| 303 | continue; |
| 304 | }; |
| 305 | let Some(content) = capture.get(1) else { |
| 306 | continue; |
| 307 | }; |
| 308 | let without_chrome = page_chrome.replace_all(content.as_str(), ""); |
| 309 | if meaningful_html(&without_chrome) { |
| 310 | return Some(without_chrome.into_owned()); |
| 311 | } |
| 312 | } |
| 313 | None |
| 314 | } |
| 315 | |
| 316 | fn meaningful_html(html: &str) -> bool { |
| 317 | meaningful_text(&html_to_plain_text(html)) |
| 318 | } |
| 319 | |
| 320 | fn meaningful_text(text: &str) -> bool { |
| 321 | text.chars().filter(|ch| !ch.is_whitespace()).count() >= 32 |
| 322 | && text.split_whitespace().count() >= 5 |
| 323 | } |
| 324 | |
| 325 | fn html_to_plain_text(html: &str) -> String { |
| 326 | let without_tags = TAG_RE |
| 327 | .get_or_init(|| Regex::new(r"(?s)<[^>]+>").expect("tag regex")) |
| 328 | .replace_all(html, " "); |
| 329 | normalize_text(&decode_common_entities(&without_tags)) |
| 330 | } |
| 331 | |
| 332 | fn normalize_text(text: &str) -> String { |
| 333 | WHITESPACE_RE |
| 334 | .get_or_init(|| Regex::new(r"\s+").expect("whitespace regex")) |
| 335 | .replace_all(text.trim(), " ") |
| 336 | .into_owned() |
| 337 | } |
| 338 | |
| 339 | fn decode_common_entities(value: &str) -> String { |
| 340 | value |
| 341 | .replace(" ", " ") |
| 342 | .replace("&", "&") |
| 343 | .replace("<", "<") |
| 344 | .replace(">", ">") |
| 345 | .replace(""", "\"") |
| 346 | .replace("'", "'") |
| 347 | } |
| 348 | |
| 349 | fn html_title(html: &str) -> Option<String> { |
| 350 | let capture = TITLE_RE |
| 351 | .get_or_init(|| { |
| 352 | Regex::new(r"(?is)<title(?:\s[^>]*)?>(.*?)</title\s*>").expect("title regex") |
| 353 | }) |
| 354 | .captures(html)?; |
| 355 | let title = normalize_text(&decode_common_entities(capture.get(1)?.as_str())); |
| 356 | (!title.is_empty()).then_some(title) |
| 357 | } |
| 358 | |
| 359 | fn markdown_title(body: &str) -> Option<String> { |
| 360 | body.lines().find_map(|line| { |
| 361 | let title = line.trim().strip_prefix("# ")?.trim(); |
| 362 | (!title.is_empty()).then(|| title.to_string()) |
| 363 | }) |
| 364 | } |
| 365 | |
| 366 | /// Stable prefix for "the response parsed, but carried no readable body". |
| 367 | /// |
| 368 | /// The fetch pipeline matches on this to decide whether a cache-busting |
| 369 | /// re-fetch is worth one more request, and to attach the role-aware recovery |
| 370 | /// text. Extraction itself has no `ToolContext`, so it cannot know which |
| 371 | /// escalation the calling role actually owns; it states the fact and leaves |
| 372 | /// the remedy to [`super::fetch`]. |
| 373 | pub(crate) const JS_SHELL_MARKER: &str = "No readable page content was found at"; |
| 374 | |
| 375 | /// Whether `error` is the JS-shell extraction failure (a parsed response whose |
| 376 | /// body held no readable content), as opposed to a transport or type failure. |
| 377 | pub(crate) fn is_js_shell_error(error: &ToolError) -> bool { |
| 378 | error.to_string().contains(JS_SHELL_MARKER) |
| 379 | } |
| 380 | |
| 381 | fn js_required_error(url: &str) -> ToolError { |
| 382 | ToolError::execution_failed(format!( |
| 383 | "{JS_SHELL_MARKER} {url}; the response parsed but its body held no readable content, so the page may require JavaScript." |
| 384 | )) |
| 385 | } |
| 386 | |
| 387 | /// Decode one response body without guessing from language statistics. |
| 388 | /// |
| 389 | /// Precedence is receipt-grade and deterministic: BOM, recognized transport |
| 390 | /// charset, an HTML-only bounded meta prescan, then UTF-8. Unknown transport |
| 391 | /// labels deliberately fall through to a valid HTML declaration. `html_sniff` |
| 392 | /// must come from MIME/URL/ASCII markup evidence; JSON and plain text callers |
| 393 | /// pass `false`, so a body string cannot impersonate an HTML declaration. |
| 394 | pub(crate) fn decode_response_body( |
| 395 | bytes: &[u8], |
| 396 | content_type: Option<&str>, |
| 397 | html_sniff: bool, |
| 398 | ) -> Result<String, ToolError> { |
| 399 | if let Some((encoding, bom_len)) = Encoding::for_bom(bytes) { |
| 400 | let (decoded, _) = encoding.decode_without_bom_handling(&bytes[bom_len..]); |
| 401 | reject_binary_nul(bytes, encoding, &decoded)?; |
| 402 | return Ok(decoded.into_owned()); |
| 403 | } |
| 404 | |
| 405 | let transport_encoding = content_type.and_then(content_type_encoding); |
| 406 | let encoding = transport_encoding |
| 407 | .or_else(|| html_sniff.then(|| html_meta_encoding(bytes)).flatten()) |
| 408 | .unwrap_or(UTF_8); |
| 409 | let (decoded, _) = encoding.decode_without_bom_handling(bytes); |
| 410 | reject_binary_nul(bytes, encoding, &decoded)?; |
| 411 | Ok(decoded.into_owned()) |
| 412 | } |
| 413 | |
| 414 | fn reject_binary_nul( |
| 415 | bytes: &[u8], |
| 416 | encoding: &'static Encoding, |
| 417 | decoded: &str, |
| 418 | ) -> Result<(), ToolError> { |
| 419 | // UTF-16 uses zero bytes structurally for many characters, so inspect its |
| 420 | // decoded scalar values. Every other encoding must be NUL-free across the |
| 421 | // complete response; neither a BOM nor a late byte may bypass the guard. |
| 422 | let contains_nul = if encoding == UTF_16LE || encoding == UTF_16BE { |
| 423 | decoded.contains('\0') |
| 424 | } else { |
| 425 | bytes.contains(&0) |
| 426 | }; |
| 427 | if contains_nul { |
| 428 | return Err(ToolError::execution_failed( |
| 429 | "Unsupported binary response contained NUL bytes", |
| 430 | )); |
| 431 | } |
| 432 | Ok(()) |
| 433 | } |
| 434 | |
| 435 | /// Parse only exact semicolon-delimited `charset` parameters. A random |
| 436 | /// `charset=` substring inside another parameter is not transport authority. |
| 437 | fn content_type_encoding(value: &str) -> Option<&'static Encoding> { |
| 438 | value.split(';').skip(1).find_map(|parameter| { |
| 439 | let (name, raw_value) = parameter.split_once('=')?; |
| 440 | if !name.trim().eq_ignore_ascii_case("charset") { |
| 441 | return None; |
| 442 | } |
| 443 | let value = raw_value.trim(); |
| 444 | let value = match (value.as_bytes().first(), value.as_bytes().last()) { |
| 445 | (Some(b'"'), Some(b'"')) | (Some(b'\''), Some(b'\'')) if value.len() >= 2 => { |
| 446 | &value[1..value.len() - 1] |
| 447 | } |
| 448 | _ if value.contains('"') || value.contains('\'') => return None, |
| 449 | _ => value, |
| 450 | }; |
| 451 | let label = value.trim(); |
| 452 | (!label.is_empty()) |
| 453 | .then(|| Encoding::for_label(label.as_bytes())) |
| 454 | .flatten() |
| 455 | }) |
| 456 | } |
| 457 | |
| 458 | fn should_sniff_html_encoding(content_type: Option<&str>, url: &str, bytes: &[u8]) -> bool { |
| 459 | match content_type { |
| 460 | Some("text/html" | "application/xhtml+xml") => true, |
| 461 | // Explicit non-HTML text and structured formats never consult markup |
| 462 | // embedded in their body. |
| 463 | Some(value) if value.starts_with("text/") || is_structured_text_type(value) => false, |
| 464 | Some("application/octet-stream") | None => { |
| 465 | url_path_ends_with(url, &[".html", ".htm"]) || looks_like_html_bytes(bytes) |
| 466 | } |
| 467 | Some(_) => false, |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | fn is_structured_text_type(content_type: &str) -> bool { |
| 472 | content_type.contains("json") |
| 473 | || content_type.contains("xml") |
| 474 | || content_type.contains("yaml") |
| 475 | || content_type.contains("javascript") |
| 476 | } |
| 477 | |
| 478 | fn looks_like_html_bytes(bytes: &[u8]) -> bool { |
| 479 | let start = Encoding::for_bom(bytes).map_or(0, |(_, length)| length); |
| 480 | let end = bytes |
| 481 | .len() |
| 482 | .min(start.saturating_add(HTML_ENCODING_SNIFF_BYTES)); |
| 483 | let ascii = ascii_lowercase_projection(&bytes[start..end]); |
| 484 | let Some(prefix) = html_prefix_after_leading_declarations(&ascii) else { |
| 485 | return false; |
| 486 | }; |
| 487 | prefix.starts_with("<!doctype html") |
| 488 | || prefix.starts_with("<html") |
| 489 | || prefix.starts_with("<head") |
| 490 | || prefix.starts_with("<meta") |
| 491 | } |
| 492 | |
| 493 | fn html_prefix_after_leading_declarations(mut prefix: &str) -> Option<&str> { |
| 494 | loop { |
| 495 | prefix = prefix.trim_start(); |
| 496 | if let Some(comment) = prefix.strip_prefix("<!--") { |
| 497 | let end = comment.find("-->")?; |
| 498 | prefix = &comment[end + 3..]; |
| 499 | continue; |
| 500 | } |
| 501 | if let Some(declaration) = prefix.strip_prefix("<?xml") { |
| 502 | let end = declaration.find("?>")?; |
| 503 | prefix = &declaration[end + 2..]; |
| 504 | continue; |
| 505 | } |
| 506 | return Some(prefix); |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | fn html_meta_encoding(bytes: &[u8]) -> Option<&'static Encoding> { |
| 511 | let sniff_len = bytes.len().min(HTML_ENCODING_SNIFF_BYTES); |
| 512 | let html = ascii_lowercase_projection(&bytes[..sniff_len]); |
| 513 | let mut cursor = 0usize; |
| 514 | |
| 515 | while let Some(relative) = html[cursor..].find('<') { |
| 516 | let start = cursor + relative; |
| 517 | if html[start..].starts_with("<!--") { |
| 518 | cursor = html[start + 4..] |
| 519 | .find("-->") |
| 520 | .map_or(html.len(), |end| start + 4 + end + 3); |
| 521 | continue; |
| 522 | } |
| 523 | if tag_starts_at(&html, start, "script") || tag_starts_at(&html, start, "style") { |
| 524 | let name = if tag_starts_at(&html, start, "script") { |
| 525 | "script" |
| 526 | } else { |
| 527 | "style" |
| 528 | }; |
| 529 | let close = format!("</{name}"); |
| 530 | cursor = html[start..] |
| 531 | .find(&close) |
| 532 | .and_then(|close_start| { |
| 533 | html[start + close_start..] |
| 534 | .find('>') |
| 535 | .map(|end| start + close_start + end + 1) |
| 536 | }) |
| 537 | .unwrap_or(html.len()); |
| 538 | continue; |
| 539 | } |
| 540 | if !tag_starts_at(&html, start, "meta") { |
| 541 | cursor = start + 1; |
| 542 | continue; |
| 543 | } |
| 544 | let relative_end = html[start..].find('>')?; |
| 545 | let end = start + relative_end + 1; |
| 546 | let tag = &html[start..end]; |
| 547 | if let Some(label) = html_attribute_value(tag, "charset") |
| 548 | && let Some(encoding) = Encoding::for_label(label.as_bytes()) |
| 549 | { |
| 550 | return Some(normalize_meta_encoding(encoding)); |
| 551 | } |
| 552 | let is_content_type = html_attribute_value(tag, "http-equiv") |
| 553 | .is_some_and(|value| value.eq_ignore_ascii_case("content-type")); |
| 554 | if is_content_type |
| 555 | && let Some(content) = html_attribute_value(tag, "content") |
| 556 | && let Some(encoding) = content_type_encoding(&content) |
| 557 | { |
| 558 | return Some(normalize_meta_encoding(encoding)); |
| 559 | } |
| 560 | cursor = end; |
| 561 | } |
| 562 | None |
| 563 | } |
| 564 | |
| 565 | fn normalize_meta_encoding(encoding: &'static Encoding) -> &'static Encoding { |
| 566 | if encoding == UTF_16LE || encoding == UTF_16BE { |
| 567 | UTF_8 |
| 568 | } else { |
| 569 | encoding |
| 570 | } |
| 571 | } |
| 572 | |
| 573 | fn ascii_lowercase_projection(bytes: &[u8]) -> String { |
| 574 | bytes |
| 575 | .iter() |
| 576 | .map(|byte| { |
| 577 | if byte.is_ascii() { |
| 578 | char::from(byte.to_ascii_lowercase()) |
| 579 | } else { |
| 580 | ' ' |
| 581 | } |
| 582 | }) |
| 583 | .collect() |
| 584 | } |
| 585 | |
| 586 | fn tag_starts_at(html: &str, start: usize, name: &str) -> bool { |
| 587 | let Some(after_name) = html.get(start + 1 + name.len()..) else { |
| 588 | return false; |
| 589 | }; |
| 590 | html[start + 1..].starts_with(name) |
| 591 | && after_name |
| 592 | .chars() |
| 593 | .next() |
| 594 | .is_some_and(|ch| ch.is_ascii_whitespace() || matches!(ch, '/' | '>')) |
| 595 | } |
| 596 | |
| 597 | fn html_attribute_value(tag: &str, wanted: &str) -> Option<String> { |
| 598 | let bytes = tag.as_bytes(); |
| 599 | let mut cursor = 1usize; |
| 600 | while cursor < bytes.len() && !bytes[cursor].is_ascii_whitespace() && bytes[cursor] != b'>' { |
| 601 | cursor += 1; |
| 602 | } |
| 603 | while cursor < bytes.len() { |
| 604 | while cursor < bytes.len() && (bytes[cursor].is_ascii_whitespace() || bytes[cursor] == b'/') |
| 605 | { |
| 606 | cursor += 1; |
| 607 | } |
| 608 | if cursor >= bytes.len() || bytes[cursor] == b'>' { |
| 609 | break; |
| 610 | } |
| 611 | let name_start = cursor; |
| 612 | while cursor < bytes.len() |
| 613 | && !bytes[cursor].is_ascii_whitespace() |
| 614 | && !matches!(bytes[cursor], b'=' | b'/' | b'>') |
| 615 | { |
| 616 | cursor += 1; |
| 617 | } |
| 618 | let name = &tag[name_start..cursor]; |
| 619 | while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { |
| 620 | cursor += 1; |
| 621 | } |
| 622 | if cursor >= bytes.len() || bytes[cursor] != b'=' { |
| 623 | continue; |
| 624 | } |
| 625 | cursor += 1; |
| 626 | while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { |
| 627 | cursor += 1; |
| 628 | } |
| 629 | if cursor >= bytes.len() { |
| 630 | break; |
| 631 | } |
| 632 | let (value_start, value_end) = if matches!(bytes[cursor], b'"' | b'\'') { |
| 633 | let quote = bytes[cursor]; |
| 634 | cursor += 1; |
| 635 | let start = cursor; |
| 636 | while cursor < bytes.len() && bytes[cursor] != quote { |
| 637 | cursor += 1; |
| 638 | } |
| 639 | let end = cursor; |
| 640 | cursor = cursor.saturating_add(1); |
| 641 | (start, end) |
| 642 | } else { |
| 643 | let start = cursor; |
| 644 | while cursor < bytes.len() |
| 645 | && !bytes[cursor].is_ascii_whitespace() |
| 646 | && bytes[cursor] != b'>' |
| 647 | { |
| 648 | cursor += 1; |
| 649 | } |
| 650 | (start, cursor) |
| 651 | }; |
| 652 | if name.eq_ignore_ascii_case(wanted) { |
| 653 | return Some(tag[value_start..value_end].trim().to_string()); |
| 654 | } |
| 655 | } |
| 656 | None |
| 657 | } |
| 658 | |
| 659 | fn normalized_content_type(content_type: Option<&str>) -> Option<String> { |
| 660 | content_type |
| 661 | .and_then(|value| value.split(';').next()) |
| 662 | .map(str::trim) |
| 663 | .filter(|value| !value.is_empty()) |
| 664 | .map(str::to_ascii_lowercase) |
| 665 | } |
| 666 | |
| 667 | fn is_html(content_type: Option<&str>, url: &str, body: &str) -> bool { |
| 668 | matches!(content_type, Some("text/html" | "application/xhtml+xml")) |
| 669 | || url_path_ends_with(url, &[".html", ".htm"]) |
| 670 | || { |
| 671 | let prefix = body.trim_start().chars().take(64).collect::<String>(); |
| 672 | let prefix = prefix.to_ascii_lowercase(); |
| 673 | prefix.contains("<!doctype html") || prefix.contains("<html") |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | fn is_markdown(content_type: Option<&str>, url: &str) -> bool { |
| 678 | matches!( |
| 679 | content_type, |
| 680 | Some("text/markdown" | "text/x-markdown" | "application/markdown") |
| 681 | ) || url_path_ends_with(url, &[".md", ".markdown"]) |
| 682 | } |
| 683 | |
| 684 | fn is_textual(content_type: Option<&str>, url: &str) -> bool { |
| 685 | content_type.is_some_and(|value| { |
| 686 | value.starts_with("text/") |
| 687 | || value.contains("json") |
| 688 | || value.contains("xml") |
| 689 | || value.contains("yaml") |
| 690 | || value.contains("javascript") |
| 691 | || value == "application/sql" |
| 692 | }) || url_path_ends_with( |
| 693 | url, |
| 694 | &[ |
| 695 | ".txt", ".json", ".jsonl", ".xml", ".yaml", ".yml", ".csv", ".tsv", ".rs", ".py", |
| 696 | ".js", ".ts", ".toml", |
| 697 | ], |
| 698 | ) |
| 699 | } |
| 700 | |
| 701 | fn url_is_pdf(url: &str) -> bool { |
| 702 | url_path_ends_with(url, &[".pdf"]) |
| 703 | } |
| 704 | |
| 705 | fn url_path_ends_with(url: &str, extensions: &[&str]) -> bool { |
| 706 | reqwest::Url::parse(url) |
| 707 | .ok() |
| 708 | .map(|parsed| parsed.path().to_ascii_lowercase()) |
| 709 | .is_some_and(|path| extensions.iter().any(|extension| path.ends_with(extension))) |
| 710 | } |
| 711 | |
| 712 | fn looks_like_pdf(bytes: &[u8]) -> bool { |
| 713 | bytes.starts_with(b"%PDF-") |
| 714 | } |
| 715 | |
| 716 | fn declared_media_family(content_type: Option<&str>) -> Option<MediaFamily> { |
| 717 | let content_type = content_type?; |
| 718 | if content_type.starts_with("image/") { |
| 719 | Some(MediaFamily::Image) |
| 720 | } else if content_type.starts_with("audio/") { |
| 721 | Some(MediaFamily::Audio) |
| 722 | } else if content_type.starts_with("video/") { |
| 723 | Some(MediaFamily::Video) |
| 724 | } else { |
| 725 | None |
| 726 | } |
| 727 | } |
| 728 | |
| 729 | fn sniff_media(bytes: &[u8]) -> Option<MediaSignature> { |
| 730 | let trimmed = bytes |
| 731 | .iter() |
| 732 | .position(|byte| !byte.is_ascii_whitespace()) |
| 733 | .map(|start| &bytes[start..]) |
| 734 | .unwrap_or(bytes); |
| 735 | let signature = if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { |
| 736 | MediaSignature { |
| 737 | extension: "png", |
| 738 | family: MediaFamily::Image, |
| 739 | } |
| 740 | } else if bytes.starts_with(b"\xff\xd8\xff") { |
| 741 | MediaSignature { |
| 742 | extension: "jpg", |
| 743 | family: MediaFamily::Image, |
| 744 | } |
| 745 | } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { |
| 746 | MediaSignature { |
| 747 | extension: "gif", |
| 748 | family: MediaFamily::Image, |
| 749 | } |
| 750 | } else if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" { |
| 751 | MediaSignature { |
| 752 | extension: "webp", |
| 753 | family: MediaFamily::Image, |
| 754 | } |
| 755 | } else if bytes.starts_with(b"ID3") || bytes.starts_with(b"\xff\xfb") { |
| 756 | MediaSignature { |
| 757 | extension: "mp3", |
| 758 | family: MediaFamily::Audio, |
| 759 | } |
| 760 | } else if bytes.starts_with(b"fLaC") { |
| 761 | MediaSignature { |
| 762 | extension: "flac", |
| 763 | family: MediaFamily::Audio, |
| 764 | } |
| 765 | } else if bytes.starts_with(b"OggS") { |
| 766 | MediaSignature { |
| 767 | extension: "ogg", |
| 768 | family: MediaFamily::Audio, |
| 769 | } |
| 770 | } else if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WAVE" { |
| 771 | MediaSignature { |
| 772 | extension: "wav", |
| 773 | family: MediaFamily::Audio, |
| 774 | } |
| 775 | } else if bytes.len() >= 12 && &bytes[4..8] == b"ftyp" { |
| 776 | MediaSignature { |
| 777 | extension: "mp4", |
| 778 | family: MediaFamily::Video, |
| 779 | } |
| 780 | } else if bytes.starts_with(b"\x1aE\xdf\xa3") { |
| 781 | MediaSignature { |
| 782 | extension: "webm", |
| 783 | family: MediaFamily::Video, |
| 784 | } |
| 785 | } else if trimmed.starts_with(b"<svg") |
| 786 | || (trimmed.starts_with(b"<?xml") |
| 787 | && trimmed |
| 788 | .windows(4) |
| 789 | .take(1_024) |
| 790 | .any(|window| window.eq_ignore_ascii_case(b"<svg"))) |
| 791 | { |
| 792 | MediaSignature { |
| 793 | extension: "svg", |
| 794 | family: MediaFamily::Image, |
| 795 | } |
| 796 | } else { |
| 797 | return None; |
| 798 | }; |
| 799 | Some(signature) |
| 800 | } |
| 801 | |
| 802 | async fn extract_pdf( |
| 803 | bytes: &[u8], |
| 804 | command: super::super::pdf::PdfTextCommand<'_>, |
| 805 | ) -> Result<ExtractedDocument, ToolError> { |
| 806 | let text = super::super::pdf::extract_bytes(bytes, command) |
| 807 | .await |
| 808 | .map_err(super::super::pdf::into_tool_error)?; |
| 809 | let pages = split_pdf_pages(&text); |
| 810 | let text = pages |
| 811 | .iter() |
| 812 | .map(|page| page.join("\n")) |
| 813 | .collect::<Vec<_>>() |
| 814 | .join("\n\n"); |
| 815 | Ok(ExtractedDocument { |
| 816 | kind: DocumentKind::Pdf, |
| 817 | title: Some("PDF Document".to_string()), |
| 818 | markdown: text.clone(), |
| 819 | text, |
| 820 | cleaned_html: None, |
| 821 | pdf_pages: Some(pages), |
| 822 | media_extension: None, |
| 823 | }) |
| 824 | } |
| 825 | |
| 826 | fn split_pdf_pages(text: &str) -> Vec<Vec<String>> { |
| 827 | text.split('\x0C') |
| 828 | .map(|page| { |
| 829 | page.lines() |
| 830 | .map(str::trim) |
| 831 | .filter(|line| !line.is_empty()) |
| 832 | .map(ToOwned::to_owned) |
| 833 | .collect::<Vec<_>>() |
| 834 | }) |
| 835 | .collect() |
| 836 | } |
| 837 | |
| 838 | #[cfg(test)] |
| 839 | mod tests { |
| 840 | use super::*; |
| 841 | |
| 842 | #[tokio::test] |
| 843 | async fn html_becomes_readable_markdown_without_page_chrome() { |
| 844 | let html = br#"<!doctype html><html><head><title>Whale & Signal</title></head><body> |
| 845 | <nav>Products Pricing Log in Cookies</nav> |
| 846 | <article><h1>Fetch once</h1><p>This is the important article body with enough words to be useful.</p> |
| 847 | <a href="/proof">Read the proof</a></article> |
| 848 | <footer>Privacy Cookies Terms</footer></body></html>"#; |
| 849 | let document = extract_document("https://example.com/post", Some("text/html"), html, None) |
| 850 | .await |
| 851 | .expect("extract html"); |
| 852 | |
| 853 | assert_eq!(document.kind, DocumentKind::Html); |
| 854 | assert_eq!(document.title.as_deref(), Some("Whale & Signal")); |
| 855 | assert!(document.markdown.contains("Fetch once") || document.title.is_some()); |
| 856 | assert!( |
| 857 | document |
| 858 | .markdown |
| 859 | .contains("[Read the proof](https://example.com/proof)") |
| 860 | ); |
| 861 | assert!(!document.markdown.contains("Products Pricing")); |
| 862 | assert!(!document.markdown.contains("Privacy Cookies")); |
| 863 | } |
| 864 | |
| 865 | #[test] |
| 866 | fn relative_http_href_resolution_preserves_other_destination_kinds() { |
| 867 | let base = reqwest::Url::parse("https://example.com/guides/page").expect("base URL"); |
| 868 | assert_eq!( |
| 869 | resolve_relative_http_href(&base, "../proof?q=1#receipt").as_deref(), |
| 870 | Some("https://example.com/proof?q=1#receipt") |
| 871 | ); |
| 872 | for href in [ |
| 873 | "#receipt", |
| 874 | "mailto:maintainer@example.com", |
| 875 | "data:text/plain,proof", |
| 876 | "codewhale:session/123", |
| 877 | "https://other.example/proof", |
| 878 | "http://[::1", |
| 879 | "", |
| 880 | ] { |
| 881 | assert_eq!( |
| 882 | resolve_relative_http_href(&base, href), |
| 883 | None, |
| 884 | "destination must be left to htmd unchanged: {href:?}" |
| 885 | ); |
| 886 | } |
| 887 | let file = reqwest::Url::parse("file:///tmp/page").expect("file URL"); |
| 888 | assert!(resolve_relative_http_href(&file, "proof").is_none()); |
| 889 | } |
| 890 | |
| 891 | #[tokio::test] |
| 892 | async fn sparse_document_uses_article_fallback() { |
| 893 | let html = br#"<html><head><title>Fallback</title></head><body><nav>cookie banner</nav> |
| 894 | <article><h2>Small source</h2><p>Five useful words survive this compact article fallback path.</p></article> |
| 895 | </body></html>"#; |
| 896 | let document = extract_document("https://example.com/short", Some("text/html"), html, None) |
| 897 | .await |
| 898 | .expect("extract fallback"); |
| 899 | |
| 900 | assert!(document.markdown.contains("## Small source")); |
| 901 | assert!(!document.markdown.contains("cookie banner")); |
| 902 | } |
| 903 | |
| 904 | #[tokio::test] |
| 905 | async fn javascript_shell_returns_actionable_error() { |
| 906 | let error = extract_document( |
| 907 | "https://example.com/app", |
| 908 | Some("text/html"), |
| 909 | b"<html><body><div id='root'></div><script>boot()</script></body></html>", |
| 910 | None, |
| 911 | ) |
| 912 | .await |
| 913 | .expect_err("empty app shell must fail"); |
| 914 | |
| 915 | let message = error.to_string(); |
| 916 | assert!(message.contains("may require JavaScript"), "{message}"); |
| 917 | assert!( |
| 918 | message.contains("https://example.com/app"), |
| 919 | "the shell failure must name the URL: {message}" |
| 920 | ); |
| 921 | assert!( |
| 922 | is_js_shell_error(&error), |
| 923 | "the fetch pipeline recognizes this failure by marker: {message}" |
| 924 | ); |
| 925 | assert!( |
| 926 | !is_js_shell_error(&ToolError::execution_failed("connection reset")), |
| 927 | "transport failures must not look like a JS shell" |
| 928 | ); |
| 929 | } |
| 930 | |
| 931 | #[tokio::test] |
| 932 | async fn markdown_passes_through_unchanged() { |
| 933 | let body = b"# Release note\n\nA complete markdown response remains intact.\n"; |
| 934 | let document = extract_document( |
| 935 | "https://example.com/release.md", |
| 936 | Some("text/markdown; charset=utf-8"), |
| 937 | body, |
| 938 | None, |
| 939 | ) |
| 940 | .await |
| 941 | .expect("extract markdown"); |
| 942 | |
| 943 | assert_eq!(document.kind, DocumentKind::Markdown); |
| 944 | assert_eq!(document.markdown.as_bytes(), body); |
| 945 | assert_eq!(document.title.as_deref(), Some("Release note")); |
| 946 | } |
| 947 | |
| 948 | #[tokio::test] |
| 949 | async fn media_requires_matching_magic_bytes() { |
| 950 | let error = extract_document( |
| 951 | "https://example.com/not-image.png", |
| 952 | Some("image/png"), |
| 953 | b"<html>not really an image</html>", |
| 954 | None, |
| 955 | ) |
| 956 | .await |
| 957 | .expect_err("spoofed media must fail"); |
| 958 | assert!(error.to_string().contains("did not match")); |
| 959 | |
| 960 | let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); |
| 961 | png.extend_from_slice(b"fake test payload"); |
| 962 | let document = extract_document( |
| 963 | "https://example.com/image", |
| 964 | Some("application/octet-stream"), |
| 965 | &png, |
| 966 | None, |
| 967 | ) |
| 968 | .await |
| 969 | .expect("sniff png"); |
| 970 | assert_eq!(document.kind, DocumentKind::Media); |
| 971 | assert_eq!(document.media_extension, Some("png")); |
| 972 | } |
| 973 | |
| 974 | #[tokio::test] |
| 975 | async fn arbitrary_binary_is_rejected() { |
| 976 | let error = extract_document( |
| 977 | "https://example.com/archive.bin", |
| 978 | Some("application/octet-stream"), |
| 979 | b"PK\x03\x04archive bytes", |
| 980 | None, |
| 981 | ) |
| 982 | .await |
| 983 | .expect_err("archive must be rejected"); |
| 984 | assert!(error.to_string().contains("Unsupported binary response")); |
| 985 | } |
| 986 | |
| 987 | #[tokio::test] |
| 988 | async fn empty_success_body_is_valid_text() { |
| 989 | let document = extract_document( |
| 990 | "https://example.com/no-content", |
| 991 | Some("application/octet-stream"), |
| 992 | b"", |
| 993 | None, |
| 994 | ) |
| 995 | .await |
| 996 | .expect("empty body"); |
| 997 | assert_eq!(document.kind, DocumentKind::Text); |
| 998 | assert!(document.text.is_empty()); |
| 999 | } |
| 1000 | |
| 1001 | #[tokio::test] |
| 1002 | async fn content_type_matching_is_case_insensitive() { |
| 1003 | let document = extract_document( |
| 1004 | "https://example.com/document", |
| 1005 | Some("Application/JSON; Charset=UTF-8"), |
| 1006 | br#"{"status":"ok"}"#, |
| 1007 | None, |
| 1008 | ) |
| 1009 | .await |
| 1010 | .expect("mixed-case JSON content type"); |
| 1011 | |
| 1012 | assert_eq!(document.kind, DocumentKind::Text); |
| 1013 | assert_eq!(document.text, r#"{"status":"ok"}"#); |
| 1014 | } |
| 1015 | |
| 1016 | #[test] |
| 1017 | fn bom_wins_over_conflicting_transport_and_is_removed() { |
| 1018 | let mut utf8 = b"\xef\xbb\xbf".to_vec(); |
| 1019 | utf8.extend_from_slice("café".as_bytes()); |
| 1020 | assert_eq!( |
| 1021 | decode_response_body(&utf8, Some("text/html; charset=windows-1252"), true) |
| 1022 | .expect("UTF-8 BOM"), |
| 1023 | "café" |
| 1024 | ); |
| 1025 | |
| 1026 | let mut utf16 = vec![0xff, 0xfe]; |
| 1027 | for unit in "BOM 日本語".encode_utf16() { |
| 1028 | utf16.extend_from_slice(&unit.to_le_bytes()); |
| 1029 | } |
| 1030 | assert_eq!( |
| 1031 | decode_response_body(&utf16, Some("text/plain; charset=windows-1252"), false) |
| 1032 | .expect("UTF-16 BOM"), |
| 1033 | "BOM 日本語" |
| 1034 | ); |
| 1035 | } |
| 1036 | |
| 1037 | #[test] |
| 1038 | fn content_type_charset_is_exact_recognized_and_order_independent() { |
| 1039 | let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode("café"); |
| 1040 | for content_type in [ |
| 1041 | "text/plain; charset=windows-1252", |
| 1042 | "TEXT/PLAIN; boundary=x; CHARSET = \"windows-1252\"; q=1", |
| 1043 | "text/plain; q=1; charset='windows-1252'", |
| 1044 | ] { |
| 1045 | assert_eq!( |
| 1046 | decode_response_body(&bytes, Some(content_type), false).expect("declared charset"), |
| 1047 | "café", |
| 1048 | "{content_type}" |
| 1049 | ); |
| 1050 | } |
| 1051 | |
| 1052 | for malformed in [ |
| 1053 | "text/plain; note=charset=windows-1252", |
| 1054 | "text/plain; charset=\"windows-1252", |
| 1055 | "text/plain; charset=definitely-not-an-encoding", |
| 1056 | ] { |
| 1057 | let decoded = |
| 1058 | decode_response_body(&bytes, Some(malformed), false).expect("UTF-8 fallback"); |
| 1059 | assert!(decoded.contains('\u{fffd}'), "{malformed}: {decoded}"); |
| 1060 | } |
| 1061 | } |
| 1062 | |
| 1063 | #[test] |
| 1064 | fn invalid_header_falls_through_to_direct_and_legacy_html_meta() { |
| 1065 | let direct = r#"<html><head><meta charset="gbk"></head><body>中文</body></html>"#; |
| 1066 | let (direct_bytes, _, _) = encoding_rs::GBK.encode(direct); |
| 1067 | assert!( |
| 1068 | decode_response_body(&direct_bytes, Some("text/html; charset=not-real"), true,) |
| 1069 | .expect("direct meta") |
| 1070 | .contains("中文") |
| 1071 | ); |
| 1072 | |
| 1073 | let legacy = r#"<html><head><meta content="text/html; charset=windows-1252" http-equiv="Content-Type"></head><body>café</body></html>"#; |
| 1074 | let (legacy_bytes, _, _) = encoding_rs::WINDOWS_1252.encode(legacy); |
| 1075 | assert!( |
| 1076 | decode_response_body(&legacy_bytes, Some("text/html"), true) |
| 1077 | .expect("legacy meta") |
| 1078 | .contains("café") |
| 1079 | ); |
| 1080 | } |
| 1081 | |
| 1082 | #[test] |
| 1083 | fn recognized_transport_charset_beats_conflicting_meta() { |
| 1084 | let html = r#"<html><head><meta charset="shift_jis"></head><body>中文</body></html>"#; |
| 1085 | let (bytes, _, _) = encoding_rs::GBK.encode(html); |
| 1086 | let decoded = decode_response_body(&bytes, Some("text/html; charset=gbk"), true) |
| 1087 | .expect("transport charset"); |
| 1088 | assert!(decoded.contains("中文"), "{decoded}"); |
| 1089 | } |
| 1090 | |
| 1091 | #[test] |
| 1092 | fn html_prescan_ignores_comments_scripts_and_late_meta() { |
| 1093 | let cases = [ |
| 1094 | "<!-- <meta charset=windows-1252> --><html><body>café</body></html>".to_string(), |
| 1095 | "<script>\"<meta charset=windows-1252>\"</script><html><body>café</body></html>" |
| 1096 | .to_string(), |
| 1097 | format!( |
| 1098 | "<html><head>{}<meta charset=windows-1252></head><body>café</body></html>", |
| 1099 | " ".repeat(HTML_ENCODING_SNIFF_BYTES) |
| 1100 | ), |
| 1101 | ]; |
| 1102 | for html in cases { |
| 1103 | let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode(&html); |
| 1104 | let decoded = decode_response_body(&bytes, Some("text/html"), true) |
| 1105 | .expect("bounded HTML fallback"); |
| 1106 | assert!( |
| 1107 | decoded.contains('\u{fffd}'), |
| 1108 | "late/ignored meta changed decoding: {decoded}" |
| 1109 | ); |
| 1110 | } |
| 1111 | } |
| 1112 | |
| 1113 | #[test] |
| 1114 | fn non_html_bodies_never_sniff_meta_markup() { |
| 1115 | let plain = "literal <meta charset=windows-1252> café"; |
| 1116 | let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode(plain); |
| 1117 | for content_type in ["text/plain", "application/json"] { |
| 1118 | let decoded = decode_response_body(&bytes, Some(content_type), false) |
| 1119 | .expect("non-HTML UTF-8 fallback"); |
| 1120 | assert!(decoded.contains('\u{fffd}'), "{content_type}: {decoded}"); |
| 1121 | } |
| 1122 | } |
| 1123 | |
| 1124 | #[test] |
| 1125 | fn declared_gbk_shift_jis_and_windows_1252_decode_deterministically() { |
| 1126 | let cases = [ |
| 1127 | (encoding_rs::GBK, "中文", "gbk"), |
| 1128 | (encoding_rs::SHIFT_JIS, "日本語", "shift_jis"), |
| 1129 | (encoding_rs::WINDOWS_1252, "café", "windows-1252"), |
| 1130 | ]; |
| 1131 | for (encoding, text, label) in cases { |
| 1132 | let (bytes, _, had_errors) = encoding.encode(text); |
| 1133 | assert!(!had_errors, "fixture must be representable in {label}"); |
| 1134 | assert_eq!( |
| 1135 | decode_response_body(&bytes, Some(&format!("text/plain; charset={label}")), false,) |
| 1136 | .expect("decode declared encoding"), |
| 1137 | text |
| 1138 | ); |
| 1139 | } |
| 1140 | } |
| 1141 | |
| 1142 | #[test] |
| 1143 | fn nul_binary_is_rejected_but_utf16_bom_text_is_not() { |
| 1144 | let error = decode_response_body(b"PK\0\x03\x04archive", Some("text/plain"), false) |
| 1145 | .expect_err("NUL binary must fail"); |
| 1146 | assert!(error.to_string().contains("NUL bytes")); |
| 1147 | |
| 1148 | let bom_binary = b"\xef\xbb\xbfapparently text\0binary"; |
| 1149 | let error = decode_response_body(bom_binary, Some("text/plain"), false) |
| 1150 | .expect_err("a BOM must not bypass the NUL guard"); |
| 1151 | assert!(error.to_string().contains("NUL bytes")); |
| 1152 | |
| 1153 | let mut late_binary = vec![b'x'; 8_193]; |
| 1154 | late_binary.push(0); |
| 1155 | let error = decode_response_body(&late_binary, Some("text/plain"), false) |
| 1156 | .expect_err("a late NUL must not bypass the full-body guard"); |
| 1157 | assert!(error.to_string().contains("NUL bytes")); |
| 1158 | |
| 1159 | let utf16 = [0xff, 0xfe, b'O', 0, b'K', 0]; |
| 1160 | assert_eq!( |
| 1161 | decode_response_body(&utf16, Some("application/octet-stream"), false) |
| 1162 | .expect("BOM proves UTF-16 text"), |
| 1163 | "OK" |
| 1164 | ); |
| 1165 | |
| 1166 | let utf16_nul = [0xff, 0xfe, b'O', 0, 0, 0, b'K', 0]; |
| 1167 | let error = decode_response_body(&utf16_nul, Some("text/plain"), false) |
| 1168 | .expect_err("decoded UTF-16 NUL must remain binary"); |
| 1169 | assert!(error.to_string().contains("NUL bytes")); |
| 1170 | } |
| 1171 | |
| 1172 | #[tokio::test] |
| 1173 | async fn extensionless_html_sniff_skips_leading_comments_and_xml_declarations() { |
| 1174 | let cases = [ |
| 1175 | r#"<!-- deployment marker --><html><head><meta charset="windows-1252"><title>Café release notes</title></head><body><article><h1>Café release notes</h1><p>This extensionless page contains enough meaningful text for deterministic extraction.</p></article></body></html>"#, |
| 1176 | r#"<?xml version="1.0"?><!-- marker --><head><meta charset="windows-1252"><title>Café release notes</title></head><body><article><h1>Café release notes</h1><p>This extensionless page contains enough meaningful text for deterministic extraction.</p></article></body>"#, |
| 1177 | ]; |
| 1178 | for html in cases { |
| 1179 | let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode(html); |
| 1180 | let document = |
| 1181 | extract_document("https://example.com/extensionless", None, &bytes, None) |
| 1182 | .await |
| 1183 | .expect("leading declarations preserve extensionless HTML sniffing"); |
| 1184 | assert_eq!(document.kind, DocumentKind::Html); |
| 1185 | assert_eq!(document.title.as_deref(), Some("Café release notes")); |
| 1186 | } |
| 1187 | } |
| 1188 | |
| 1189 | #[tokio::test] |
| 1190 | async fn svg_requires_and_accepts_svg_markup_signature() { |
| 1191 | let svg = br#"<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg"></svg>"#; |
| 1192 | let document = extract_document( |
| 1193 | "https://example.com/diagram", |
| 1194 | Some("image/svg+xml"), |
| 1195 | svg, |
| 1196 | None, |
| 1197 | ) |
| 1198 | .await |
| 1199 | .expect("sniff svg"); |
| 1200 | assert_eq!(document.kind, DocumentKind::Media); |
| 1201 | assert_eq!(document.media_extension, Some("svg")); |
| 1202 | } |
| 1203 | } |
| 1204 |