| 1 | package builtin |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "runtime" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | |
| 12 | "golang.org/x/text/encoding/simplifiedchinese" |
| 13 | |
| 14 | fileenc "reasonix/internal/fileutil/encoding" |
| 15 | ) |
| 16 | |
| 17 | // TestReadFileStreamsLargeGB18030 proves GB18030 content far past the 256KB |
| 18 | // detection sample still decodes correctly via the streaming read path. |
| 19 | func TestReadFileStreamsLargeGB18030(t *testing.T) { |
| 20 | dir := t.TempDir() |
| 21 | path := filepath.Join(dir, "big.gbk") |
| 22 | var sb strings.Builder |
| 23 | for range 20000 { |
| 24 | sb.WriteString("第一行中文 line one 你好世界\n") |
| 25 | } |
| 26 | sb.WriteString("终点标记 THE-END\n") |
| 27 | enc, err := simplifiedchinese.GB18030.NewEncoder().String(sb.String()) |
| 28 | if err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | if err := os.WriteFile(path, []byte(enc), 0o644); err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | args, _ := json.Marshal(map[string]any{"path": path, "offset": 19999, "limit": 2}) |
| 35 | out, err := readFile{}.Execute(context.Background(), args) |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | if !strings.Contains(out, "终点标记 THE-END") || !strings.Contains(out, "你好世界") { |
| 40 | t.Fatalf("deep GB18030 content not decoded correctly:\n%s", out) |
| 41 | } |
| 42 | } |
| 43 | |
| 44 | // TestReadFileLargeBoundedMemory guards against re-slurping the whole file: a |
| 45 | // small read of a large file must allocate far less than the file size. |
| 46 | func TestReadFileLargeBoundedMemory(t *testing.T) { |
| 47 | dir := t.TempDir() |
| 48 | path := filepath.Join(dir, "big.txt") |
| 49 | var sb strings.Builder |
| 50 | for range 130000 { // ~8 MB, no NUL |
| 51 | sb.WriteString("a line of perfectly ordinary text in a large utf-8 file\n") |
| 52 | } |
| 53 | if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | args, _ := json.Marshal(map[string]any{"path": path, "limit": 5}) |
| 57 | |
| 58 | runtime.GC() |
| 59 | var m0, m1 runtime.MemStats |
| 60 | runtime.ReadMemStats(&m0) |
| 61 | out, err := readFile{}.Execute(context.Background(), args) |
| 62 | runtime.ReadMemStats(&m1) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | if alloc := m1.TotalAlloc - m0.TotalAlloc; alloc > 4<<20 { |
| 67 | t.Fatalf("read allocated %d bytes for a 5-line read of an ~8MB file — slurp regression", alloc) |
| 68 | } |
| 69 | if !strings.Contains(out, "1→a line") { |
| 70 | t.Fatalf("unexpected output: %q", out[:min(80, len(out))]) |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | func TestReadFileLargeUTF16UsesStreamingDecoder(t *testing.T) { |
| 75 | var sb strings.Builder |
| 76 | for range 100000 { |
| 77 | sb.WriteString("a line of ordinary UTF-16 text with a searchable marker\n") |
| 78 | } |
| 79 | content := sb.String() |
| 80 | cases := []struct { |
| 81 | name string |
| 82 | kind fileenc.Kind |
| 83 | }{ |
| 84 | {"le-bom", fileenc.UTF16LE}, {"be-bom", fileenc.UTF16BE}, |
| 85 | {"le-no-bom", fileenc.UTF16LENoBOM}, {"be-no-bom", fileenc.UTF16BENoBOM}, |
| 86 | } |
| 87 | for _, tc := range cases { |
| 88 | t.Run(tc.name, func(t *testing.T) { |
| 89 | path := filepath.Join(t.TempDir(), "big-utf16.txt") |
| 90 | if err := os.WriteFile(path, fileenc.Encode(content, tc.kind), 0o644); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | args, _ := json.Marshal(map[string]any{"path": path, "limit": 5}) |
| 94 | runtime.GC() |
| 95 | var m0, m1 runtime.MemStats |
| 96 | runtime.ReadMemStats(&m0) |
| 97 | out, err := readFile{}.Execute(context.Background(), args) |
| 98 | runtime.ReadMemStats(&m1) |
| 99 | if err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | if alloc := m1.TotalAlloc - m0.TotalAlloc; alloc > 4<<20 { |
| 103 | t.Fatalf("UTF-16 read allocated %d bytes for a five-line window", alloc) |
| 104 | } |
| 105 | if !strings.Contains(out, "UTF-16 text") || strings.Contains(out, "\x00") { |
| 106 | t.Fatalf("unexpected streamed UTF-16 output: %q", out) |
| 107 | } |
| 108 | }) |
| 109 | } |
| 110 | } |
| 111 |