| 1 | package retrieval |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | ) |
| 7 | |
| 8 | func TestTokensV2CodeSymbolsAndCJK(t *testing.T) { |
| 9 | got := strings.Join(TokensV2("RetryPolicy.MaxBackoffMs ticket_1827 数据库"), " ") |
| 10 | for _, want := range []string{"retrypolicy", "retry", "policy", "maxbackoffms", "max", "backoff", "ticket_1827", "ticket", "1827", "数", "据", "库", "数据", "据库"} { |
| 11 | if !strings.Contains(" "+got+" ", " "+want+" ") { |
| 12 | t.Fatalf("TokensV2 missing %q in %q", want, got) |
| 13 | } |
| 14 | } |
| 15 | if strings.Contains(" "+got+" ", " ms ") { |
| 16 | // "Ms" splits to a single meaningful segment only when len > 1; "ms" is |
| 17 | // fine — assert single letters never appear instead. |
| 18 | t.Log("ms segment present (acceptable)") |
| 19 | } |
| 20 | for _, tok := range TokensV2("A B C") { |
| 21 | if len(tok) == 0 { |
| 22 | t.Fatal("empty token emitted") |
| 23 | } |
| 24 | } |
| 25 | } |
| 26 | |
| 27 | func TestRankV2FieldWeightsBeatBodyMentions(t *testing.T) { |
| 28 | docs := []FieldedDoc{ |
| 29 | {ID: "titled", Fields: map[string]string{"title": "Package manager", "body": "We migrated to pnpm."}}, |
| 30 | {ID: "mention", Fields: map[string]string{"title": "Sprint notes", "body": "Someone asked about the package manager once; also weather, lunch, and a package of stickers."}}, |
| 31 | } |
| 32 | hits := RankV2("package manager", docs) |
| 33 | if len(hits) != 2 || hits[0].ID != "titled" { |
| 34 | t.Fatalf("field-weighted doc must outrank body mentions: %+v", hits) |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | func TestRankV2FindsCodeSymbolBySegment(t *testing.T) { |
| 39 | docs := []FieldedDoc{ |
| 40 | {ID: "sym", Fields: map[string]string{"body": "Configure RetryPolicy.MaxBackoffMs in policy/retry.toml."}}, |
| 41 | {ID: "other", Fields: map[string]string{"body": "Unrelated fact about deploys."}}, |
| 42 | } |
| 43 | hits := RankV2("max backoff configuration", docs) |
| 44 | if len(hits) == 0 || hits[0].ID != "sym" { |
| 45 | t.Fatalf("segment query must find the camel-case symbol: %+v", hits) |
| 46 | } |
| 47 | } |
| 48 |