| 1 | // Run: tsx src/__tests__/search-sources.test.ts |
| 2 | |
| 3 | import { |
| 4 | formatSearchFootnotesMarkdown, |
| 5 | mergeSearchSources, |
| 6 | parseSearchSources, |
| 7 | searchSourcesFromHistory, |
| 8 | } from "../lib/searchSources"; |
| 9 | import { normalizeSearchSources } from "../lib/searchSourcesPresentation"; |
| 10 | import { historyMessagesToItems, initialState, reducer } from "../lib/useController"; |
| 11 | import type { HistoryMessage, WireEvent } from "../lib/types"; |
| 12 | |
| 13 | let passed = 0; |
| 14 | let failed = 0; |
| 15 | |
| 16 | function eq(a: unknown, b: unknown, label: string) { |
| 17 | if (a === b) { |
| 18 | process.stdout.write(` PASS ${label}\n`); |
| 19 | passed += 1; |
| 20 | } else { |
| 21 | process.stdout.write(` FAIL ${label}: expected ${JSON.stringify(b)}, got ${JSON.stringify(a)}\n`); |
| 22 | failed += 1; |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | console.log("\nsearch footnotes"); |
| 27 | |
| 28 | eq( |
| 29 | formatSearchFootnotesMarkdown([{ title: "Change Log", url: "https://api-docs.deepseek.com/updates/" }, { title: "No URL" }]), |
| 30 | "\n- **Change Log**\n <https://api-docs.deepseek.com/updates/>\n- **No URL**\n", |
| 31 | "footnotes reuse the title-and-autolink list", |
| 32 | ); |
| 33 | eq(formatSearchFootnotesMarkdown([{ title: "bad", url: "javascript:alert(1)" }]), "\n- **bad**\n", "unsafe URLs are dropped"); |
| 34 | eq(parseSearchSources("新闻本文\nhttps://example.com/a\nNo URL").length, 2, "output parser keeps title-only hits"); |
| 35 | eq(parseSearchSources("新闻本文\nhttps://example.com/a")[0]?.title, "新闻本文", "output parser keeps the title"); |
| 36 | eq(parseSearchSources("新闻本文\r\nhttps://example.com/a\r\n")[0]?.url, "https://example.com/a", "output parser tolerates CRLF line endings"); |
| 37 | const independent = parseSearchSources(JSON.stringify({ summary: "A summary, not a source title", sources: [{ title: "Docs", url: "https://example.com/docs" }, { title: "Unsafe", url: "javascript:alert(1)" }, null] })); |
| 38 | eq(independent.length, 1, "independent search parses only safe structured sources"); |
| 39 | eq(independent[0]?.title, "Docs", "search summary does not become a source title"); |
| 40 | eq(parseSearchSources('{"summary":"No matches","sources":[]}').length, 0, "empty native results stay empty"); |
| 41 | const degraded = parseSearchSources("- **新闻本文**\n <https://example.com/a>"); |
| 42 | eq(degraded.length, 1, "degraded footnote-markdown dump still resolves to one source"); |
| 43 | eq(degraded[0]?.title, "新闻本文", "degraded dump strips the bullet and bold markers from the title"); |
| 44 | eq(degraded[0]?.url, "https://example.com/a", "degraded dump unwraps the autolink URL"); |
| 45 | eq(parseSearchSources("<https://example.com/a>")[0]?.url, "https://example.com/a", "autolink-only lines parse as URL sources"); |
| 46 | eq(mergeSearchSources([{ title: "A", url: "https://a.example" }], [{ title: "A", url: "https://a.example" }]).length, 1, "duplicate hits collapse"); |
| 47 | |
| 48 | const normalizedInput = [ |
| 49 | { title: "Tracked", url: "https://example.com/article?utm_source=search&gclid=abc&part=1#section" }, |
| 50 | { title: "Same canonical URL", url: "https://example.com/article?part=1" }, |
| 51 | { title: "Google redirect", url: "https://www.google.com/url?q=https%3A%2F%2Fdocs.example.com%2Fguide%3Futm_medium%3Dcpc%26x%3D1" }, |
| 52 | { title: "Broken redirect", url: "https://page.sm.cn/blm/midpage-317/index?id=11" }, |
| 53 | { title: "Unsafe", url: "javascript:alert(1)" }, |
| 54 | { title: " ", url: "https://empty-title.example" }, |
| 55 | ]; |
| 56 | const normalized = normalizeSearchSources(normalizedInput); |
| 57 | eq(normalized.visible.length, 2, "normalization keeps only valid unique HTTP(S) sources"); |
| 58 | eq(normalized.visible[0]?.href, "https://example.com/article?part=1", "tracking parameters and fragments are removed"); |
| 59 | eq(normalized.visible[1]?.href, "https://docs.example.com/guide?x=1", "google redirect target is unwrapped and cleaned"); |
| 60 | eq(normalized.visible[0]?.hostname, "example.com", "source projection exposes the hostname"); |
| 61 | eq(normalized.visible[0]?.displayUrl, "example.com/article?part=1", "source projection exposes a compact URL"); |
| 62 | eq(normalized.hiddenCount, 4, "invalid, duplicate, and missing-title sources are counted as hidden"); |
| 63 | eq(normalizedInput[0]?.url, "https://example.com/article?utm_source=search&gclid=abc&part=1#section", "normalization does not mutate raw search data"); |
| 64 | |
| 65 | const history = historyMessagesToItems([{ |
| 66 | role: "assistant", |
| 67 | content: "answer only", |
| 68 | serverSearch: [{ |
| 69 | id: "s1", |
| 70 | query: "bitcoin", |
| 71 | results: [{ title: "新闻本文", url: "https://example.com/a" }], |
| 72 | }], |
| 73 | }] as HistoryMessage[], "h-").items; |
| 74 | const answer = history.find((item) => item.kind === "assistant"); |
| 75 | eq(answer?.kind === "assistant" ? answer.text : "", "answer only", "answer text stays model-only"); |
| 76 | eq(answer?.kind === "assistant" ? answer.searchSources?.[0]?.title : "", "新闻本文", "history hydrates footnotes on the answer"); |
| 77 | |
| 78 | function ev(s: typeof initialState, e: WireEvent) { |
| 79 | return reducer(s, { type: "event", e }); |
| 80 | } |
| 81 | |
| 82 | let live = ev(initialState, { kind: "turn_started" }); |
| 83 | live = ev(live, { |
| 84 | kind: "tool_result", |
| 85 | tool: { id: "s1", name: "web_search", readOnly: true, output: "新闻本文\nhttps://example.com/a" }, |
| 86 | } as WireEvent); |
| 87 | live = ev(live, { kind: "text", text: "answer only" }); |
| 88 | live = ev(live, { kind: "message", text: "answer only" }); |
| 89 | const liveAnswer = live.items.find((item) => item.kind === "assistant"); |
| 90 | eq(liveAnswer?.kind === "assistant" ? liveAnswer.text : "", "answer only", "live answer stays model-only"); |
| 91 | eq(liveAnswer?.kind === "assistant" ? liveAnswer.searchSources?.[0]?.title : "", "新闻本文", "live tool result attaches footnotes to the answer"); |
| 92 | live = ev(live, { kind: "tool_dispatch", tool: { id: "after-live", name: "bash", args: "{}", readOnly: false } } as WireEvent); |
| 93 | live = ev(live, { kind: "tool_result", tool: { id: "after-live", name: "bash", readOnly: false, output: "ok" } } as WireEvent); |
| 94 | live = ev(live, { kind: "stream_attempt", streamAttempt: { id: "after-live", action: "begin", attempt: 1, max: 1 } } as WireEvent); |
| 95 | live = ev(live, { kind: "message", text: "later answer" } as WireEvent); |
| 96 | const laterLiveAnswer = live.items.filter((item) => item.kind === "assistant")[1]; |
| 97 | eq(laterLiveAnswer?.kind === "assistant" ? laterLiveAnswer.searchSources?.length ?? 0 : -1, 0, "sources attached to an active answer do not enter the pending buffer"); |
| 98 | |
| 99 | let segmented = ev(initialState, { kind: "turn_started", turnId: "search-ownership" } as WireEvent); |
| 100 | segmented = ev(segmented, { kind: "message", text: "searching" } as WireEvent); |
| 101 | segmented = ev(segmented, { |
| 102 | kind: "tool_dispatch", |
| 103 | tool: { id: "search-1", name: "web_search", args: '{"query":"bitcoin"}', readOnly: true }, |
| 104 | } as WireEvent); |
| 105 | segmented = ev(segmented, { |
| 106 | kind: "tool_result", |
| 107 | tool: { id: "search-1", name: "web_search", readOnly: true, output: "Source A\nhttps://example.com/a" }, |
| 108 | } as WireEvent); |
| 109 | segmented = ev(segmented, { |
| 110 | kind: "tool_dispatch", |
| 111 | tool: { id: "search-2", name: "web_search", args: '{"query":"ethereum"}', readOnly: true }, |
| 112 | } as WireEvent); |
| 113 | segmented = ev(segmented, { |
| 114 | kind: "tool_result", |
| 115 | tool: { id: "search-2", name: "web_search", readOnly: true, output: "Source B\nhttps://example.com/b" }, |
| 116 | } as WireEvent); |
| 117 | segmented = ev(segmented, { kind: "stream_attempt", streamAttempt: { id: "answer-1", action: "begin", attempt: 1, max: 2 } } as WireEvent); |
| 118 | segmented = ev(segmented, { kind: "message", text: "search answer" } as WireEvent); |
| 119 | segmented = ev(segmented, { |
| 120 | kind: "tool_dispatch", |
| 121 | tool: { id: "shell-1", name: "bash", args: '{"command":"true"}', readOnly: false }, |
| 122 | } as WireEvent); |
| 123 | segmented = ev(segmented, { |
| 124 | kind: "tool_result", |
| 125 | tool: { id: "shell-1", name: "bash", readOnly: false, output: "ok" }, |
| 126 | } as WireEvent); |
| 127 | segmented = ev(segmented, { kind: "stream_attempt", streamAttempt: { id: "answer-2", action: "begin", attempt: 1, max: 2 } } as WireEvent); |
| 128 | segmented = ev(segmented, { kind: "message", text: "final answer" } as WireEvent); |
| 129 | const segmentedAnswers = segmented.items.filter((item) => item.kind === "assistant"); |
| 130 | eq(segmentedAnswers.length, 3, "multi-round search flow keeps three assistant segments"); |
| 131 | eq(segmentedAnswers[1]?.kind === "assistant" ? segmentedAnswers[1].searchSources?.[0]?.title : "", "Source A", "search sources attach to the immediately following answer"); |
| 132 | eq(segmentedAnswers[1]?.kind === "assistant" ? segmentedAnswers[1].searchSources?.[1]?.title : "", "Source B", "multiple pending searches accumulate before the answer starts"); |
| 133 | eq(segmentedAnswers[2]?.kind === "assistant" ? segmentedAnswers[2].searchSources?.length ?? 0 : -1, 0, "consumed search sources do not leak into later assistant segments"); |
| 134 | eq(segmented.pendingSearchSources, undefined, "allocating an answer consumes the pending search-source buffer"); |
| 135 | eq(searchSourcesFromHistory([{ results: [{ title: "A" }] }])[0]?.title, "A", "history helper reads structured hits"); |
| 136 | |
| 137 | if (failed) { |
| 138 | process.stdout.write(`\n${failed} failed, ${passed} passed\n`); |
| 139 | process.exit(1); |
| 140 | } |
| 141 | process.stdout.write(`\n${passed} passed\n`); |
| 142 |