scrape.rs - CodeWhale

scrape.rs

1	//! Shared DuckDuckGo / Bing HTML SERP scrapers and spam filter.
2	//!
3	//! Used by both `web_search` and `web_run` so parser behavior (including the
4	//! #964 evidence-based spam filter) cannot drift between the two paths.
5
6	use base64::{Engine as _, engine::general_purpose};
7	use regex::Regex;
8	use std::sync::OnceLock;
9
10	/// Shared browser-like user agent for public SERP scraping and image search.
11	/// Both `web_search` and `web_run` send this so scrape behavior (including
12	/// bot-challenge rates) stays identical between the two surfaces.
13	pub(crate) const BROWSER_USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15";
14
15	/// One parsed search hit: title, absolute URL, optional snippet.
16	#[derive(Debug, Clone, PartialEq, Eq)]
17	pub struct ScrapedSearchResult {
18	pub title: String,
19	pub url: String,
20	pub snippet: Option<String>,
21	}
22
23	// Cached regex patterns for HTML parsing
24	static TITLE_RE: OnceLock<Regex> = OnceLock::new();
25	static SNIPPET_RE: OnceLock<Regex> = OnceLock::new();
26	static TAG_RE: OnceLock<Regex> = OnceLock::new();
27	static BING_RESULT_RE: OnceLock<Regex> = OnceLock::new();
28	static BING_TITLE_RE: OnceLock<Regex> = OnceLock::new();
29	static BING_SNIPPET_RE: OnceLock<Regex> = OnceLock::new();
30
31	fn get_title_re() -> &'static Regex {
32	TITLE_RE.get_or_init(\|\| {
33	Regex::new(r#"<a[^>]class=\"result__a\"[^>]href=\"([^\"]+)\"[^>]>(.?)</a>"#)
34	.expect("title regex pattern is valid")
35	})
36	}
37
38	fn get_snippet_re() -> &'static Regex {
39	SNIPPET_RE.get_or_init(\|\| {
40	Regex::new(
41	r#"<a[^>]class=\"result__snippet\"[^>]>(.?)</a>\|<div[^>]class=\"result__snippet\"[^>]>(.?)</div>"#,
42	)
43	.expect("snippet regex pattern is valid")
44	})
45	}
46
47	fn get_tag_re() -> &'static Regex {
48	TAG_RE.get_or_init(\|\| Regex::new(r"<[^>]+>").expect("tag regex pattern is valid"))
49	}
50
51	fn get_bing_result_re() -> &'static Regex {
52	BING_RESULT_RE.get_or_init(\|\| {
53	Regex::new(r#"(?is)<li[^>]class=\"[^\"]\bb_algo\b[^\"]\"[^>]>(.*?)</li>"#)
54	.expect("bing result regex pattern is valid")
55	})
56	}
57
58	fn get_bing_title_re() -> &'static Regex {
59	BING_TITLE_RE.get_or_init(\|\| {
60	Regex::new(r#"(?is)<h2[^>]>.?<a[^>]href=\"([^\"]+)\"[^>]>(.*?)</a>"#)
61	.expect("bing title regex pattern is valid")
62	})
63	}
64
65	fn get_bing_snippet_re() -> &'static Regex {
66	BING_SNIPPET_RE.get_or_init(\|\| {
67	Regex::new(r#"(?is)<div[^>]class=\"[^\"]\bb_caption\b[^\"]\"[^>]>.?<p[^>]>(.*?)</p>"#)
68	.expect("bing snippet regex pattern is valid")
69	})
70	}
71
72	/// Parse DuckDuckGo HTML SERP results. Known spam-domain hits are omitted.
73	pub fn parse_duckduckgo_results(html: &str, max_results: usize) -> Vec<ScrapedSearchResult> {
74	let title_re = get_title_re();
75	let snippet_re = get_snippet_re();
76	let snippets: Vec<String> = snippet_re
77	.captures_iter(html)
78	.filter_map(\|cap\| cap.get(1).or_else(\|\| cap.get(2)))
79	.map(\|m\| normalize_text(m.as_str()))
80	.collect();
81
82	let mut results = Vec::new();
83	for (idx, cap) in title_re.captures_iter(html).enumerate() {
84	if results.len() >= max_results {
85	break;
86	}
87	let href = cap.get(1).map(\|m\| m.as_str()).unwrap_or("");
88	let title_raw = cap.get(2).map(\|m\| m.as_str()).unwrap_or("");
89	let title = normalize_text(title_raw);
90	if title.is_empty() {
91	continue;
92	}
93	let url = normalize_duckduckgo_url(href);
94	if is_known_spam_url(&url) {
95	continue;
96	}
97	let snippet = snippets
98	.get(idx)
99	.map(\|s\| s.to_string())
100	.filter(\|s\| !s.is_empty());
101
102	results.push(ScrapedSearchResult {
103	title,
104	url,
105	snippet,
106	});
107	}
108
109	results
110	}
111
112	/// Parse Bing HTML SERP results. Known spam-domain hits are omitted.
113	pub fn parse_bing_results(html: &str, max_results: usize) -> Vec<ScrapedSearchResult> {
114	let mut results = Vec::new();
115	for cap in get_bing_result_re().captures_iter(html) {
116	if results.len() >= max_results {
117	break;
118	}
119	let Some(block) = cap.get(1).map(\|m\| m.as_str()) else {
120	continue;
121	};
122	let Some(title_cap) = get_bing_title_re().captures(block) else {
123	continue;
124	};
125	let href = title_cap.get(1).map(\|m\| m.as_str()).unwrap_or("");
126	let title_raw = title_cap.get(2).map(\|m\| m.as_str()).unwrap_or("");
127	let title = normalize_text(title_raw);
128	if title.is_empty() {
129	continue;
130	}
131	let snippet = get_bing_snippet_re()
132	.captures(block)
133	.and_then(\|snippet_cap\| snippet_cap.get(1))
134	.map(\|m\| normalize_text(m.as_str()))
135	.filter(\|s\| !s.is_empty());
136
137	let url = normalize_bing_url(href);
138	if is_known_spam_url(&url) {
139	continue;
140	}
141	results.push(ScrapedSearchResult {
142	title,
143	url,
144	snippet,
145	});
146	}
147	results
148	}
149
150	/// Detect DuckDuckGo bot-challenge interstitial HTML.
151	pub fn is_duckduckgo_challenge(html: &str) -> bool {
152	html.contains("anomaly-modal") \|\| html.contains("Unfortunately, bots use DuckDuckGo too")
153	}
154
155	/// Evidence-based filter for the domain family in #964. A broad same-domain
156	/// ratio is not a spam signal: site-scoped searches and documentation hosts
157	/// legitimately return many results from one registrable domain.
158	fn is_known_spam_url(url: &str) -> bool {
159	const KNOWN_SPAM_DOMAINS: &[&str] = &["forumgratuit.org"];
160
161	let Ok(parsed) = reqwest::Url::parse(url) else {
162	return false;
163	};
164	let Some(host) = parsed.host_str() else {
165	return false;
166	};
167	KNOWN_SPAM_DOMAINS
168	.iter()
169	.any(\|domain\| host == *domain \|\| host.ends_with(&format!(".{domain}")))
170	}
171
172	fn normalize_duckduckgo_url(href: &str) -> String {
173	if let Some(uddg) = extract_query_param(href, "uddg") {
174	let decoded = percent_decode(&uddg);
175	if !decoded.is_empty() {
176	return decoded;
177	}
178	}
179	if href.starts_with("//") {
180	return format!("https:{href}");
181	}
182	if href.starts_with('/') {
183	return format!("https://duckduckgo.com{href}");
184	}
185	href.to_string()
186	}
187
188	/// Normalize a Bing SERP result href, unwrapping `/ck/a?...&u=<base64>` redirects.
189	pub fn normalize_bing_url(href: &str) -> String {
190	// Bing wraps every SERP result URL in a `/ck/a?...&u=<base64>` click-tracking
191	// redirect, and in the raw HTML the separators are `&` entities. Without
192	// decoding entities first, `extract_query_param` looks for `u` but the actual
193	// key is `amp;u`, so the real URL is never recovered and callers receive Bing's
194	// tracking URL instead of the cited source. Decode entities before parsing.
195	let href = decode_html_entities(href);
196	let href = href.as_str();
197	if let Some(encoded) = extract_query_param(href, "u") {
198	let decoded = percent_decode(&encoded);
199	let token = decoded.strip_prefix("a1").unwrap_or(&decoded);
200	let mut padded = token.replace('-', "+").replace('_', "/");
201	while !padded.len().is_multiple_of(4) {
202	padded.push('=');
203	}
204	if let Ok(bytes) = general_purpose::STANDARD.decode(padded)
205	&& let Ok(url) = String::from_utf8(bytes)
206	&& (url.starts_with("http://") \|\| url.starts_with("https://"))
207	{
208	return url;
209	}
210	}
211	if href.starts_with("//") {
212	return format!("https:{href}");
213	}
214	if href.starts_with('/') {
215	return format!("https://www.bing.com{href}");
216	}
217	href.to_string()
218	}
219
220	fn normalize_text(text: &str) -> String {
221	let stripped = strip_html_tags(text);
222	let decoded = decode_html_entities(&stripped);
223	decoded.split_whitespace().collect::<Vec<_>>().join(" ")
224	}
225
226	fn strip_html_tags(text: &str) -> String {
227	get_tag_re().replace_all(text, "").to_string()
228	}
229
230	/// Decode common HTML named and numeric character references.
231	pub fn decode_html_entities(text: &str) -> String {
232	static ENTITY_RE: OnceLock<Regex> = OnceLock::new();
233	let re = ENTITY_RE.get_or_init(\|\| {
234	Regex::new(r"&(?:#(\d+)\|#x([0-9A-Fa-f]+)\|([a-zA-Z]+));").expect("HTML entity regex")
235	});
236
237	re.replace_all(text, \|caps: &regex::Captures\| {
238	if let Some(dec) = caps.get(1) {
239	return dec
240	.as_str()
241	.parse::<u32>()
242	.ok()
243	.and_then(std::char::from_u32)
244	.unwrap_or('\u{FFFD}')
245	.to_string();
246	}
247	if let Some(hex) = caps.get(2) {
248	return u32::from_str_radix(hex.as_str(), 16)
249	.ok()
250	.and_then(std::char::from_u32)
251	.unwrap_or('\u{FFFD}')
252	.to_string();
253	}
254	let named = caps.get(3).map(\|m\| m.as_str());
255	match named {
256	Some("amp") => "&",
257	Some("lt") => "<",
258	Some("gt") => ">",
259	Some("quot") => "\"",
260	Some("apos") => "'",
261	Some("nbsp") => " ",
262	Some("copy") => "\u{00A9}",
263	Some("reg") => "\u{00AE}",
264	Some("mdash") => "\u{2014}",
265	Some("ndash") => "\u{2013}",
266	Some("lsquo") => "\u{2018}",
267	Some("rsquo") => "\u{2019}",
268	Some("ldquo") => "\u{201C}",
269	Some("rdquo") => "\u{201D}",
270	Some("hellip") => "\u{2026}",
271	_ => return caps.get(0).map(\|m\| m.as_str()).unwrap_or("").to_string(),
272	}
273	.to_string()
274	})
275	.to_string()
276	}
277
278	/// Percent-decode a URL component. `+` becomes space (query-string convention).
279	pub fn percent_decode(input: &str) -> String {
280	let bytes = input.as_bytes();
281	let mut out = Vec::new();
282	let mut i = 0;
283	while i < bytes.len() {
284	match bytes[i] {
285	b'%' if i + 2 < bytes.len() => {
286	let hex = &input[i + 1..i + 3];
287	if let Ok(val) = u8::from_str_radix(hex, 16) {
288	out.push(val);
289	i += 3;
290	continue;
291	}
292	out.push(bytes[i]);
293	}
294	b'+' => out.push(b' '),
295	_ => out.push(bytes[i]),
296	}
297	i += 1;
298	}
299	String::from_utf8_lossy(&out).to_string()
300	}
301
302	fn extract_query_param(url: &str, key: &str) -> Option<String> {
303	let query = url.split_once('?')?.1;
304	for part in query.split('&') {
305	let mut iter = part.splitn(2, '=');
306	let name = iter.next().unwrap_or("");
307	if name == key {
308	return iter.next().map(str::to_string);
309	}
310	}
311	None
312	}
313
314	#[cfg(test)]
315	mod tests {
316	use super::*;
317
318	// Regression guard: Bing /ck/a redirect hrefs are HTML-entity-encoded
319	// (`&`). normalize_bing_url must decode entities before extracting the
320	// `u=` base64 payload, otherwise the real URL is never recovered and the
321	// result remains a Bing tracking URL instead of the cited source.
322	#[test]
323	fn bing_ckurl_with_html_entities_decodes_real_url() {
324	let href = "https://www.bing.com/ck/a?!&&p=abc&u=a1aHR0cHM6Ly9ydXN0LWxhbmcub3JnLw&ntb=1";
325	assert_eq!(normalize_bing_url(href), "https://rust-lang.org/");
326	}
327
328	#[test]
329	fn parses_bing_results_and_decodes_redirect_url() {
330	let html = r#"
331	<ol>
332	<li class="b_algo">
333	<h2><a href="https://www.bing.com/ck/a?u=a1aHR0cHM6Ly9leGFtcGxlLmNvbS9wYXRoP3E9MQ">Example & Result</a></h2>
334	<div class="b_caption"><p>A <strong>useful</strong> snippet.</p></div>
335	</li>
336	</ol>
337	"#;
338
339	let results = parse_bing_results(html, 5);
340
341	assert_eq!(results.len(), 1);
342	assert_eq!(results[0].title, "Example & Result");
343	assert_eq!(results[0].url, "https://example.com/path?q=1");
344	assert_eq!(results[0].snippet.as_deref(), Some("A useful snippet."));
345	}
346
347	#[test]
348	fn parses_duckduckgo_results() {
349	let html = r#"
350	<a class="result__a" href="https://example.com/rust">Rust & async</a>
351	<a class="result__snippet">A <b>useful</b> snippet.</a>
352	<a class="result__a" href="//docs.rs/tokio">Tokio</a>
353	<div class="result__snippet">Runtime docs</div>
354	"#;
355	let results = parse_duckduckgo_results(html, 5);
356	assert_eq!(results.len(), 2);
357	assert_eq!(results[0].title, "Rust & async");
358	assert_eq!(results[0].url, "https://example.com/rust");
359	assert_eq!(results[0].snippet.as_deref(), Some("A useful snippet."));
360	assert_eq!(results[1].url, "https://docs.rs/tokio");
361	}
362
363	#[test]
364	fn known_spam_filter_is_domain_specific() {
365	assert!(is_known_spam_url("https://astralia.forumgratuit.org/page1"));
366	assert!(!is_known_spam_url("https://forumgratuit.org.example/a"));
367	assert!(!is_known_spam_url("https://docs.example.com/a"));
368	}
369
370	#[test]
371	fn legitimate_same_domain_results_are_preserved() {
372	let html = r#"
373	<a class="result__a" href="https://docs.example.com/a">A</a>
374	<a class="result__snippet">s</a>
375	<a class="result__a" href="https://docs.example.com/b">B</a>
376	<a class="result__snippet">s</a>
377	<a class="result__a" href="https://docs.example.com/c">C</a>
378	<a class="result__snippet">s</a>
379	<a class="result__a" href="https://docs.example.com/d">D</a>
380	<a class="result__snippet">s</a>
381	"#;
382	assert_eq!(parse_duckduckgo_results(html, 10).len(), 4);
383	}
384
385	#[test]
386	fn public_suffix_and_private_suffix_hosts_are_not_misgrouped() {
387	let html = r#"
388	<a class="result__a" href="https://alpha.example.co.uk/a">A</a>
389	<a class="result__snippet">s</a>
390	<a class="result__a" href="https://beta.example.co.uk/b">B</a>
391	<a class="result__snippet">s</a>
392	<a class="result__a" href="https://alice.github.io/c">C</a>
393	<a class="result__snippet">s</a>
394	<a class="result__a" href="https://bob.github.io/d">D</a>
395	<a class="result__snippet">s</a>
396	"#;
397	assert_eq!(parse_duckduckgo_results(html, 10).len(), 4);
398	}
399
400	#[test]
401	fn max_results_three_keeps_legitimate_same_domain_results() {
402	let html = r#"
403	<a class="result__a" href="https://docs.example.com/a">A</a>
404	<a class="result__snippet">s</a>
405	<a class="result__a" href="https://docs.example.com/b">B</a>
406	<a class="result__snippet">s</a>
407	<a class="result__a" href="https://docs.example.com/c">C</a>
408	<a class="result__snippet">s</a>
409	<a class="result__a" href="https://docs.example.com/d">D</a>
410	<a class="result__snippet">s</a>
411	"#;
412	assert_eq!(parse_duckduckgo_results(html, 3).len(), 3);
413	}
414
415	#[test]
416	fn parse_duckduckgo_filters_known_spam_domain() {
417	// Shared path used by web_run and web_search: known spam → empty.
418	let html = r#"
419	<a class="result__a" href="https://astralia.forumgratuit.org/a">A</a>
420	<a class="result__snippet">s</a>
421	<a class="result__a" href="https://russia.forumgratuit.org/b">B</a>
422	<a class="result__snippet">s</a>
423	<a class="result__a" href="https://other.forumgratuit.org/c">C</a>
424	<a class="result__snippet">s</a>
425	<a class="result__a" href="https://hello.forumgratuit.org/d">D</a>
426	<a class="result__snippet">s</a>
427	<a class="result__a" href="https://world.forumgratuit.org/e">E</a>
428	<a class="result__snippet">s</a>
429	"#;
430	assert!(parse_duckduckgo_results(html, 10).is_empty());
431	}
432
433	#[test]
434	fn parse_bing_filters_known_spam_domain() {
435	let html = r#"
436	<ol>
437	<li class="b_algo">
438	<h2><a href="https://astralia.forumgratuit.org/a">A</a></h2>
439	<div class="b_caption"><p>s</p></div>
440	</li>
441	<li class="b_algo">
442	<h2><a href="https://russia.forumgratuit.org/b">B</a></h2>
443	<div class="b_caption"><p>s</p></div>
444	</li>
445	<li class="b_algo">
446	<h2><a href="https://other.forumgratuit.org/c">C</a></h2>
447	<div class="b_caption"><p>s</p></div>
448	</li>
449	<li class="b_algo">
450	<h2><a href="https://hello.forumgratuit.org/d">D</a></h2>
451	<div class="b_caption"><p>s</p></div>
452	</li>
453	<li class="b_algo">
454	<h2><a href="https://world.forumgratuit.org/e">E</a></h2>
455	<div class="b_caption"><p>s</p></div>
456	</li>
457	</ol>
458	"#;
459	assert!(parse_bing_results(html, 10).is_empty());
460	}
461
462	#[test]
463	fn decode_html_entities_handles_named_entities() {
464	assert_eq!(decode_html_entities("&"), "&");
465	assert_eq!(decode_html_entities("<"), "<");
466	assert_eq!(decode_html_entities(">"), ">");
467	assert_eq!(decode_html_entities("""), "\"");
468	assert_eq!(decode_html_entities("'"), "'");
469	assert_eq!(decode_html_entities(" "), " ");
470	assert_eq!(decode_html_entities("©"), "\u{00A9}");
471	assert_eq!(decode_html_entities("—"), "\u{2014}");
472	}
473
474	#[test]
475	fn decode_html_entities_handles_decimal_numeric_references() {
476	assert_eq!(decode_html_entities("A"), "A");
477	assert_eq!(decode_html_entities("<"), "<");
478	assert_eq!(decode_html_entities("–"), "\u{2013}");
479	}
480
481	#[test]
482	fn decode_html_entities_handles_hex_numeric_references() {
483	assert_eq!(decode_html_entities("A"), "A");
484	assert_eq!(decode_html_entities("<"), "<");
485	assert_eq!(decode_html_entities("—"), "\u{2014}");
486	}
487
488	#[test]
489	fn decode_html_entities_passthrough_unknown() {
490	assert_eq!(decode_html_entities("&unknown;"), "&unknown;");
491	}
492
493	#[test]
494	fn decode_html_entities_mixed_content() {
495	let input = "Hello & welcome to "Rust's world" — enjoy!";
496	let expected = "Hello & welcome to \"Rust's world\" \u{2014} enjoy!";
497	assert_eq!(decode_html_entities(input), expected);
498	}
499
500	#[test]
501	fn percent_decode_handles_utf8_multibyte_sequences() {
502	// Percent-encoded CJK: %E4%B8%AA%E4%BA%BA = 个人 (each glyph is 3 UTF-8 bytes).
503	assert_eq!(percent_decode("Hello %E4%B8%AA%E4%BA%BA"), "Hello 个人");
504	assert_eq!(percent_decode("%E7%B4%A0%E6%9D%90"), "素材");
505	// Percent-encoded UTF-8 inside a URL path (DuckDuckGo `uddg=` redirect shape).
506	assert_eq!(
507	percent_decode("https://example.com/%E9%A1%B5%E9%9D%A2"),
508	"https://example.com/页面"
509	);
510	// Raw UTF-8 in the input passes through unchanged.
511	assert_eq!(percent_decode("查询 keyword"), "查询 keyword");
512	// Query-string convention: `+` becomes space; `%20` becomes space.
513	assert_eq!(percent_decode("foo+bar%20baz"), "foo bar baz");
514	}
515
516	#[test]
517	fn is_duckduckgo_challenge_detects_interstitial() {
518	assert!(is_duckduckgo_challenge(
519	"Unfortunately, bots use DuckDuckGo too."
520	));
521	assert!(is_duckduckgo_challenge(
522	r#"<div class="anomaly-modal">challenge</div>"#
523	));
524	assert!(!is_duckduckgo_challenge(
525	r#"<a class="result__a" href="https://example.com">ok</a>"#
526	));
527	}
528	}
529

529 lines RUST