| 1 | // Package encoding detects and converts file encodings for the built-in |
| 2 | // file tools. The detection cascade (BOM → strict UTF-8 → GB18030 → lossy |
| 3 | // UTF-8) mirrors v1's file-encoding.ts and keeps CJK Windows files editable |
| 4 | // without silently mangling their bytes. |
| 5 | package encoding |
| 6 | |
| 7 | import ( |
| 8 | "bytes" |
| 9 | "encoding/binary" |
| 10 | "os" |
| 11 | "unicode/utf8" |
| 12 | |
| 13 | "golang.org/x/text/encoding/simplifiedchinese" |
| 14 | "golang.org/x/text/encoding/unicode" |
| 15 | "golang.org/x/text/transform" |
| 16 | ) |
| 17 | |
| 18 | // Kind identifies a detected file encoding. |
| 19 | type Kind int |
| 20 | |
| 21 | const ( |
| 22 | // UTF8 is plain UTF-8 without a BOM — the common case. |
| 23 | UTF8 Kind = iota |
| 24 | // UTF8BOM is UTF-8 with a leading BOM (EF BB BF). |
| 25 | UTF8BOM |
| 26 | // UTF16LE is UTF-16 Little-Endian with a BOM (FF FE). |
| 27 | UTF16LE |
| 28 | // UTF16BE is UTF-16 Big-Endian with a BOM (FE FF). |
| 29 | UTF16BE |
| 30 | // GB18030 is the Chinese national standard charset (superset of GBK). |
| 31 | GB18030 |
| 32 | // LossyUTF8 is not valid UTF-8 and not valid GB18030 — decoded lossily |
| 33 | // as UTF-8 with replacement characters so the model sees something. |
| 34 | LossyUTF8 |
| 35 | // UTF16LENoBOM is UTF-16 Little-Endian without a BOM — common for source |
| 36 | // files saved by Windows tools. Detected heuristically from the NUL-byte |
| 37 | // pattern; written back without a BOM to preserve the original bytes. |
| 38 | UTF16LENoBOM |
| 39 | // UTF16BENoBOM is UTF-16 Big-Endian without a BOM. |
| 40 | UTF16BENoBOM |
| 41 | ) |
| 42 | |
| 43 | var utf8BOM = []byte{0xEF, 0xBB, 0xBF} |
| 44 | |
| 45 | // Detect returns the encoding kind for the given raw file bytes. The same |
| 46 | // bytes should then be passed to Decode for conversion to UTF-8. |
| 47 | func Detect(data []byte) (Kind, []byte) { |
| 48 | switch { |
| 49 | case len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF: |
| 50 | return UTF8BOM, data |
| 51 | case len(data) >= 2 && data[0] == 0xFF && data[1] == 0xFE: |
| 52 | return UTF16LE, data |
| 53 | case len(data) >= 2 && data[0] == 0xFE && data[1] == 0xFF: |
| 54 | return UTF16BE, data |
| 55 | } |
| 56 | // BOM-less UTF-16 must be tried before utf8.Valid: its low bytes plus 0x00 |
| 57 | // high bytes are all valid UTF-8 code units, so a naive check would tag a |
| 58 | // UTF-16 source file as UTF-8 and surface the embedded NULs as garbage. |
| 59 | if k, ok := DetectUTF16NoBOM(data); ok { |
| 60 | return k, data |
| 61 | } |
| 62 | if utf8.Valid(data) { |
| 63 | return UTF8, data |
| 64 | } |
| 65 | // Try GB18030 — it is a strict superset of GBK and rejects truly |
| 66 | // invalid byte sequences, so a successful decode is a reliable signal. |
| 67 | dec := simplifiedchinese.GB18030.NewDecoder() |
| 68 | if _, _, err := transform.Bytes(dec, data); err == nil { |
| 69 | return GB18030, data |
| 70 | } |
| 71 | return LossyUTF8, data |
| 72 | } |
| 73 | |
| 74 | // DetectQuick checks only for BOM prefixes in the first few bytes. This is |
| 75 | // the fast path for peek-based binary rejection: BOM-prefixed files (UTF-16, |
| 76 | // UTF-8 BOM) skip the NUL-byte check since 0x00 is normal in UTF-16. Returns |
| 77 | // UTF8 for non-BOM content (the caller should fall through to full Detect |
| 78 | // after verifying no NUL bytes). |
| 79 | func DetectQuick(peek []byte) Kind { |
| 80 | switch { |
| 81 | case len(peek) >= 3 && peek[0] == 0xEF && peek[1] == 0xBB && peek[2] == 0xBF: |
| 82 | return UTF8BOM |
| 83 | case len(peek) >= 2 && peek[0] == 0xFF && peek[1] == 0xFE: |
| 84 | return UTF16LE |
| 85 | case len(peek) >= 2 && peek[0] == 0xFE && peek[1] == 0xFF: |
| 86 | return UTF16BE |
| 87 | } |
| 88 | return UTF8 |
| 89 | } |
| 90 | |
| 91 | // DetectUTF16NoBOM heuristically recognises BOM-less UTF-16 from the NUL-byte |
| 92 | // distribution: ASCII-range text encodes one byte of payload and one 0x00 per |
| 93 | // code unit, so the NULs cluster on odd offsets (LE) or even offsets (BE). It |
| 94 | // requires a strong skew — one parity heavily NUL, the other almost none — so |
| 95 | // genuine binary (NULs on both parities) and plain UTF-8 (no NULs) fall through. |
| 96 | func DetectUTF16NoBOM(b []byte) (Kind, bool) { |
| 97 | n := len(b) |
| 98 | if n < 16 { |
| 99 | return UTF8, false |
| 100 | } |
| 101 | n &^= 1 // examine an even-length window so parity counts are comparable |
| 102 | var evenNUL, oddNUL int |
| 103 | for i := range n { |
| 104 | if b[i] != 0 { |
| 105 | continue |
| 106 | } |
| 107 | if i%2 == 0 { |
| 108 | evenNUL++ |
| 109 | } else { |
| 110 | oddNUL++ |
| 111 | } |
| 112 | } |
| 113 | half := n / 2 |
| 114 | switch { |
| 115 | case oddNUL*10 >= half*3 && evenNUL*20 <= half: |
| 116 | return UTF16LENoBOM, true |
| 117 | case evenNUL*10 >= half*3 && oddNUL*20 <= half: |
| 118 | return UTF16BENoBOM, true |
| 119 | } |
| 120 | return UTF8, false |
| 121 | } |
| 122 | |
| 123 | // Decode converts data from the given encoding to UTF-8 bytes. |
| 124 | func Decode(data []byte, enc Kind) []byte { |
| 125 | switch enc { |
| 126 | case UTF8BOM: |
| 127 | return data[3:] |
| 128 | case UTF16LE: |
| 129 | return decodeUTF16(data[2:], binary.LittleEndian) |
| 130 | case UTF16BE: |
| 131 | return decodeUTF16(data[2:], binary.BigEndian) |
| 132 | case UTF16LENoBOM: |
| 133 | return decodeUTF16(data, binary.LittleEndian) |
| 134 | case UTF16BENoBOM: |
| 135 | return decodeUTF16(data, binary.BigEndian) |
| 136 | case GB18030: |
| 137 | out, _, err := transform.Bytes(simplifiedchinese.GB18030.NewDecoder(), data) |
| 138 | if err != nil { |
| 139 | return data // should not happen after Detect, but be safe |
| 140 | } |
| 141 | return out |
| 142 | } |
| 143 | // UTF8 and LossyUTF8 both pass through — LossyUTF8 is already |
| 144 | // "best effort" and Go strings can hold arbitrary bytes. |
| 145 | return data |
| 146 | } |
| 147 | |
| 148 | // DecodeToUTF8 converts raw text-like file bytes to UTF-8 using Reasonix's |
| 149 | // shared detection cascade. It is intended for user-editable structured files |
| 150 | // (TOML, JSON, dotenv, Markdown) before handing the content to strict parsers. |
| 151 | func DecodeToUTF8(data []byte) []byte { |
| 152 | enc, raw := Detect(data) |
| 153 | return Decode(raw, enc) |
| 154 | } |
| 155 | |
| 156 | // ReadFileUTF8 reads path and decodes text-like content to UTF-8. |
| 157 | func ReadFileUTF8(path string) ([]byte, error) { |
| 158 | data, err := os.ReadFile(path) |
| 159 | if err != nil { |
| 160 | return nil, err |
| 161 | } |
| 162 | return DecodeToUTF8(data), nil |
| 163 | } |
| 164 | |
| 165 | // Decoder returns a streaming transform.Transformer for the given encoding, |
| 166 | // suitable for wrapping an io.Reader via dec.Reader(r). Returns nil for UTF-8 |
| 167 | // and LossyUTF8 (no transformation needed — the caller should read directly). |
| 168 | func Decoder(enc Kind) transform.Transformer { |
| 169 | switch enc { |
| 170 | case UTF8BOM: |
| 171 | // UTF-8 BOM just needs the 3-byte prefix stripped; the content is |
| 172 | // already valid UTF-8. Callers handle BOM stripping via Decode. |
| 173 | return nil |
| 174 | case GB18030: |
| 175 | return simplifiedchinese.GB18030.NewDecoder() |
| 176 | case UTF16LE: |
| 177 | return unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder() |
| 178 | case UTF16BE: |
| 179 | return unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM).NewDecoder() |
| 180 | case UTF16LENoBOM: |
| 181 | return unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder() |
| 182 | case UTF16BENoBOM: |
| 183 | return unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewDecoder() |
| 184 | } |
| 185 | // UTF8 and LossyUTF8 need no transformation. |
| 186 | return nil |
| 187 | } |
| 188 | |
| 189 | // Encode converts a UTF-8 string back to the given file encoding. |
| 190 | // UTF8 and LossyUTF8 produce plain UTF-8 bytes. |
| 191 | func Encode(text string, enc Kind) []byte { |
| 192 | switch enc { |
| 193 | case UTF8BOM: |
| 194 | return append(utf8BOM, []byte(text)...) |
| 195 | case UTF16LE: |
| 196 | return encodeUTF16(text, binary.LittleEndian, true) |
| 197 | case UTF16BE: |
| 198 | return encodeUTF16(text, binary.BigEndian, true) |
| 199 | case UTF16LENoBOM: |
| 200 | return encodeUTF16(text, binary.LittleEndian, false) |
| 201 | case UTF16BENoBOM: |
| 202 | return encodeUTF16(text, binary.BigEndian, false) |
| 203 | case GB18030: |
| 204 | out, _, err := transform.Bytes(simplifiedchinese.GB18030.NewEncoder(), []byte(text)) |
| 205 | if err != nil { |
| 206 | return []byte(text) |
| 207 | } |
| 208 | return out |
| 209 | } |
| 210 | return []byte(text) |
| 211 | } |
| 212 | |
| 213 | // decodeUTF16 converts UTF-16 bytes (BOM already stripped) to UTF-8. |
| 214 | func decodeUTF16(b []byte, order binary.ByteOrder) []byte { |
| 215 | u := make([]uint16, 0, len(b)/2) |
| 216 | for i := 0; i+1 < len(b); i += 2 { |
| 217 | u = append(u, order.Uint16(b[i:i+2])) |
| 218 | } |
| 219 | return []byte(string(utf16Decode(u))) |
| 220 | } |
| 221 | |
| 222 | // encodeUTF16 converts a UTF-8 string to UTF-16 bytes, with a BOM when withBOM. |
| 223 | func encodeUTF16(text string, order binary.ByteOrder, withBOM bool) []byte { |
| 224 | runes := []rune(text) |
| 225 | encoded := utf16Encode(runes) |
| 226 | |
| 227 | var buf bytes.Buffer |
| 228 | if withBOM { |
| 229 | var bom [2]byte |
| 230 | if order == binary.LittleEndian { |
| 231 | bom[0], bom[1] = 0xFF, 0xFE |
| 232 | } else { |
| 233 | bom[0], bom[1] = 0xFE, 0xFF |
| 234 | } |
| 235 | buf.Write(bom[:]) |
| 236 | } |
| 237 | for _, u := range encoded { |
| 238 | var b [2]byte |
| 239 | order.PutUint16(b[:], u) |
| 240 | buf.Write(b[:]) |
| 241 | } |
| 242 | return buf.Bytes() |
| 243 | } |
| 244 | |
| 245 | // utf16Decode converts UTF-16 code units to runes, handling surrogate pairs. |
| 246 | func utf16Decode(u []uint16) []rune { |
| 247 | var out []rune |
| 248 | for i := 0; i < len(u); i++ { |
| 249 | c := u[i] |
| 250 | if c >= 0xD800 && c <= 0xDBFF && i+1 < len(u) { |
| 251 | c2 := u[i+1] |
| 252 | if c2 >= 0xDC00 && c2 <= 0xDFFF { |
| 253 | out = append(out, rune(c-0xD800)<<10|rune(c2-0xDC00)+0x10000) |
| 254 | i++ |
| 255 | continue |
| 256 | } |
| 257 | } |
| 258 | out = append(out, rune(c)) |
| 259 | } |
| 260 | return out |
| 261 | } |
| 262 | |
| 263 | // utf16Encode converts runes to UTF-16 code units, producing surrogates for |
| 264 | // supplementary plane characters. |
| 265 | func utf16Encode(runes []rune) []uint16 { |
| 266 | var out []uint16 |
| 267 | for _, r := range runes { |
| 268 | if r >= 0x10000 && r <= 0x10FFFF { |
| 269 | r -= 0x10000 |
| 270 | out = append(out, uint16(0xD800+(r>>10)), uint16(0xDC00+(r&0x3FF))) |
| 271 | } else { |
| 272 | out = append(out, uint16(r)) |
| 273 | } |
| 274 | } |
| 275 | return out |
| 276 | } |
| 277 |