Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

38.7 KB
Raw Blame
//! Classifies file contents as text or binary and decodes text of any
//! supported encoding into UTF-8, both whole and as a stream.

use std::io::{self, Read};

use anyhow::Result;
use chardetng::EncodingDetector;
use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE};
use futures_lite::future::yield_now;

/// Number of leading bytes that [`analyze_byte_content`] inspects.
pub const FILE_ANALYSIS_BYTES: usize = 1024;

const DECODING_BLOCK_BYTES: usize = 64 * 1024;

/// Text produced by [`decode_text`], with what is needed to write it back via [`encode_text`].
pub struct DecodedText {
    /// The decoded UTF-8 text.
    pub text: String,
    /// The encoding the bytes were decoded from.
    pub encoding: &'static Encoding,
    /// Whether the bytes started with a byte order mark.
    pub has_bom: bool,
}

pub fn read_byte_header(reader: &mut dyn Read) -> io::Result<(Vec<u8>, bool)> {
    let mut header = vec![0; FILE_ANALYSIS_BYTES];
    let mut length = 0;
    while length < header.len() {
        let read = match reader.read(&mut header[length..]) {
            Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
            result => result?,
        };
        if read == 0 {
            header.truncate(length);
            return Ok((header, true));
        }
        length += read;
    }
    Ok((header, false))
}

/// Inspects the start of a file.
/// Returns the BOM encoding, if any, and otherwise the [`ByteContent`] classification of the prefix.
pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
    if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
        return (Some(encoding), ByteContent::Unknown);
    }
    (None, analyze_byte_content(prefix))
}

/// Decodes a whole file into UTF-8, detecting its encoding.
/// Fails when the bytes are classified as [`ByteContent::Binary`].
pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {
    let (bom_encoding, byte_content) = decode_byte_header(&bytes);
    if let Some(encoding) = bom_encoding {
        let (text, _) = encoding.decode_with_bom_removal(&bytes);
        return Ok(DecodedText {
            text: text.into_owned(),
            encoding,
            has_bom: true,
        });
    }

    if let Some(encoding) = byte_content.encoding() {
        let (text, _, _) = encoding.decode(&bytes);
        return Ok(DecodedText {
            text: text.into_owned(),
            encoding,
            has_bom: false,
        });
    }
    anyhow::ensure!(
        byte_content != ByteContent::Binary,
        "Binary files cannot be decoded into text"
    );

    match String::from_utf8(bytes) {
        Ok(text) if !text.contains('\x1b') => Ok(DecodedText {
            text,
            encoding: UTF_8,
            has_bom: false,
        }),
        Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),
        Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),
    }
}

pub async fn detect_encoding(reader: &mut (dyn Read + Send)) -> io::Result<&'static Encoding> {
    let mut detector = EncodingDetector::new();
    let mut block = vec![0; DECODING_BLOCK_BYTES];
    let mut length = 0;
    loop {
        let read = match reader.read(&mut block[length..]) {
            Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
            result => result?,
        };
        if read == 0 {
            detector.feed(&block[..length], true);
            return Ok(detector.guess(None, true));
        }
        length += read;
        if length == block.len() {
            detector.feed(&block, false);
            yield_now().await;
            length = 0;
        }
    }
}

/// Inverse of [`decode_text`]: encodes UTF-8 text into `encoding`, prepending a BOM when asked.
pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {
    if encoding == UTF_8 && !has_bom {
        return text.into_bytes();
    }

    // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.
    if encoding == UTF_16BE {
        let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
        if has_bom {
            bytes.extend_from_slice(&[0xFE, 0xFF]);
        }
        bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));
        return bytes;
    }

    if encoding == UTF_16LE {
        let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
        if has_bom {
            bytes.extend_from_slice(&[0xFF, 0xFE]);
        }
        bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));
        return bytes;
    }

    let (encoded, _, _) = encoding.encode(&text);
    if has_bom && encoding == UTF_8 {
        let mut bytes = Vec::with_capacity(encoded.len() + 3);
        bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);
        bytes.extend_from_slice(&encoded);
        bytes
    } else {
        encoded.into_owned()
    }
}

/// A [`Read`] adapter that transcodes its source into UTF-8 block by block, with bounded memory.
pub struct DecodingReader<'a> {
    inner: &'a mut (dyn Read + Send),
    decoder: Decoder,
    input: Vec<u8>,
    output: Vec<u8>,
    output_position: usize,
    finished: bool,
}

impl<'a> DecodingReader<'a> {
    pub fn new(inner: &'a mut (dyn Read + Send), encoding: &'static Encoding) -> Self {
        Self {
            inner,
            decoder: encoding.new_decoder(),
            input: vec![0; DECODING_BLOCK_BYTES],
            output: Vec::new(),
            output_position: 0,
            finished: false,
        }
    }

    pub fn encoding(&self) -> &'static Encoding {
        self.decoder.encoding()
    }

    fn fill_output(&mut self) -> io::Result<()> {
        self.output.clear();
        self.output_position = 0;
        let read = self.inner.read(&mut self.input)?;
        self.finished = read == 0;
        decode_block(
            &mut self.decoder,
            &self.input[..read],
            self.finished,
            &mut self.output,
        )
    }
}

impl Read for DecodingReader<'_> {
    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
        if buffer.is_empty() {
            return Ok(0);
        }
        while self.output_position == self.output.len() {
            if self.finished {
                return Ok(0);
            }
            match self.fill_output() {
                Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
                result => result?,
            }
        }

        let available = &self.output[self.output_position..];
        let count = available.len().min(buffer.len());
        buffer[..count].copy_from_slice(&available[..count]);
        self.output_position += count;
        Ok(count)
    }
}

/// Verdict of [`analyze_byte_content`] on a file prefix.
#[derive(Debug, PartialEq)]
pub enum ByteContent {
    /// UTF-16 little-endian text without a BOM.
    Utf16Le,
    /// UTF-16 big-endian text without a BOM.
    Utf16Be,
    /// Not text; the file must not be decoded or searched.
    Binary,
    /// Text in UTF-8 or some yet unknown encoding.
    Unknown,
}

impl ByteContent {
    /// The encoding this classification pins down, if any.
    pub fn encoding(&self) -> Option<&'static Encoding> {
        match self {
            Self::Utf16Le => Some(UTF_16LE),
            Self::Utf16Be => Some(UTF_16BE),
            Self::Binary | Self::Unknown => None,
        }
    }
}

/// Classifies the first [`FILE_ANALYSIS_BYTES`] of a file.
///
/// Heuristic check using null byte distribution plus a generic text-likeness
/// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise
/// distinguishes between text-like and binary-like content.
pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent {
    if bytes.len() < 2 {
        return ByteContent::Unknown;
    }

    if has_binary_header(bytes, KNOWN_BINARY_HEADERS) {
        return ByteContent::Binary;
    }

    let limit = bytes.len().min(FILE_ANALYSIS_BYTES);
    let mut even_null_count = 0usize;
    let mut odd_null_count = 0usize;
    let mut non_text_like_count = 0usize;

    for (i, &byte) in bytes[..limit].iter().enumerate() {
        if byte == 0 {
            if i % 2 == 0 {
                even_null_count += 1;
            } else {
                odd_null_count += 1;
            }
            non_text_like_count += 1;
            continue;
        }

        let is_text_like = match byte {
            b'\t' | b'\n' | b'\r' | 0x0C => true,
            0x20..=0x7E | 0x80..=0xBF | 0xC2..=0xF4 => true,
            _ => false,
        };

        if !is_text_like {
            non_text_like_count += 1;
        }
    }

    let total_null_count = even_null_count + odd_null_count;
    let has_significant_nulls = total_null_count > 0 && total_null_count >= limit / 16;
    let nulls_skew_to_even = even_null_count > odd_null_count * 4;
    let nulls_skew_to_odd = odd_null_count > even_null_count * 4;

    if has_significant_nulls {
        let sample = &bytes[..limit];

        // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first)
        // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first)

        if nulls_skew_to_even && is_plausible_utf16_text(sample, false) {
            return ByteContent::Utf16Be;
        }

        if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) {
            return ByteContent::Utf16Le;
        }

        return ByteContent::Binary;
    }

    if has_binary_header(bytes, ADDITIONAL_BINARY_HEADERS)
        || is_bitmap_header(bytes)
        || is_gguf_header(bytes)
        || is_safetensors_header(bytes)
    {
        return ByteContent::Binary;
    }

    if total_null_count == 0 || non_text_like_count * 100 < limit * 8 {
        ByteContent::Unknown
    } else {
        ByteContent::Binary
    }
}

fn decode_block(
    decoder: &mut Decoder,
    block: &[u8],
    last: bool,
    output: &mut Vec<u8>,
) -> io::Result<()> {
    let capacity = decoder
        .max_utf8_buffer_length(block.len())
        .ok_or_else(|| io::Error::new(io::ErrorKind::OutOfMemory, "decoded block is too large"))?;
    let start = output.len();
    output.resize(start + capacity, 0);
    let (result, consumed, written, _) = decoder.decode_to_utf8(block, &mut output[start..], last);
    output.truncate(start + written);
    if result != CoderResult::InputEmpty || consumed != block.len() {
        return Err(io::Error::other("decoder did not consume the whole block"));
    }
    Ok(())
}

fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {
    let mut detector = EncodingDetector::new();
    detector.feed(&bytes, true);
    let encoding = detector.guess(None, true);
    let (text, _, _) = encoding.decode(&bytes);
    DecodedText {
        text: text.into_owned(),
        encoding,
        has_bom: false,
    }
}

const KNOWN_BINARY_HEADERS: &[&[u8]] = &[
    b"%PDF-",             // PDF
    b"GIF87a",            // GIF87a
    b"GIF89a",            // GIF89a
    b"ID3",               // MP3 with ID3v2 tag
    b"IWAD",              // Doom IWAD archive
    b"OggS",              // OGG (Vorbis, Opus, FLAC)
    b"PK\x03\x04",        // ZIP local header
    b"PK\x05\x06",        // ZIP end of central directory
    b"PK\x07\x08",        // ZIP spanning/splitting
    b"PWAD",              // Doom PWAD archive
    b"RIFF",              // WAV, AVI, WebP
    b"fLaC",              // FLAC
    b"\x89PNG\r\n\x1a\n", // PNG
    b"\xFF\xD8\xFF",      // JPEG
    b"\xFF\xF2",          // MP3 frame sync (MPEG2 Layer3)
    b"\xFF\xF3",          // MP3 frame sync (MPEG2 Layer3)
    b"\xFF\xFA",          // MP3 frame sync (MPEG1 Layer3)
    b"\xFF\xFB",          // MP3 frame sync (MPEG1 Layer3)
];

const ADDITIONAL_BINARY_HEADERS: &[&[u8]] = &[
    b"\x00\x00\x00\x0CJXL \r\n\x87\n", // JPEG XL container
    b"\x00\x00\x00\x0CjP  ",           // JPEG 2000
    b"\x00\x00\x01\x00",               // Windows ICO
    b"\x00\x00\x02\x00",               // Windows CUR
    b"\x00\x01\x00\x00\x00",           // TrueType font
    b"\x00asm",                        // WebAssembly
    b"\x04\x22\x4D\x18",               // LZ4 frame
    b"\x1A\x45\xDF\xA3",               // Matroska, WebM
    b"\x1F\x8B\x08",                   // gzip
    b"7z\xBC\xAF\x27\x1C",             // 7-Zip
    b"8BPS\x00",                       // Photoshop PSD
    b"DDS |\x00\x00\x00",              // DirectDraw Surface
    b"II*\x00",                        // TIFF little-endian
    b"Kaydara FBX Binary  \x00",       // FBX binary
    b"MM\x00*",                        // TIFF big-endian
    b"MSCF\x00\x00\x00\x00",           // Microsoft Cabinet
    b"OTTO\x00",                       // OpenType font with CFF outlines
    b"Rar!\x1A\x07",                   // RAR
    b"SQLite format 3\x00",            // SQLite database
    b"glTF\x01\x00\x00\x00",           // Binary glTF 1
    b"glTF\x02\x00\x00\x00",           // Binary glTF 2
    b"\x7FELF",                        // ELF
    b"\x89HDF\r\n\x1a\n",              // HDF5
    b"\x93NUMPY\x01\x00",              // NumPy array
    b"\x93NUMPY\x02\x00",
    b"\x93NUMPY\x03\x00",
    b"\x95\x04\x12\xDE",                 // gettext MO big-endian
    b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04", // pcap big-endian
    b"\xABKTX 11\xBB\r\n\x1A\n",         // KTX texture
    b"\xABKTX 20\xBB\r\n\x1A\n",
    b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1", // OLE2 compound file (legacy MS Office)
    b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00", // pcap little-endian
    b"\xDE\x12\x04\x95",                 // gettext MO little-endian
    b"\xFD7zXZ\x00",                     // xz
];

fn has_binary_header(bytes: &[u8], headers: &[&[u8]]) -> bool {
    let index = headers.partition_point(|header| *header <= bytes);
    headers[..index]
        .last()
        .is_some_and(|header| bytes.starts_with(header))
}

fn is_gguf_header(bytes: &[u8]) -> bool {
    bytes.starts_with(b"GGUF") && bytes.get(5..8) == Some(&[0, 0, 0])
}

fn is_bitmap_header(bytes: &[u8]) -> bool {
    bytes.starts_with(b"BM") && bytes.get(6..10) == Some(&[0, 0, 0, 0])
}

fn is_safetensors_header(bytes: &[u8]) -> bool {
    bytes.get(3..8) == Some(&[0, 0, 0, 0, 0]) && bytes.get(8) == Some(&b'{')
}

// Null byte skew alone is not enough to identify UTF-16 -- binary formats with
// small 16-bit values (like PCM audio) produce the same pattern. Decode the
// bytes as UTF-16 and reject if too many code units land in control character
// ranges or form unpaired surrogates, which real text almost never contains.
fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {
    let mut suspicious_count = 0usize;
    let mut word_like_count = 0usize;
    let mut total = 0usize;

    let mut i = 0;
    while let Some(code_unit) = read_u16(bytes, i, little_endian) {
        total += 1;

        match code_unit {
            0x0009 | 0x000A | 0x000C | 0x000D => {}
            0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {
                word_like_count += 1;
            }
            // C0/C1 control characters and non-characters
            0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,
            0xD800..=0xDBFF => {
                let next_offset = i + 2;
                let has_low_surrogate = read_u16(bytes, next_offset, little_endian)
                    .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));
                if has_low_surrogate {
                    total += 1;
                    word_like_count += 2;
                    i += 2;
                } else {
                    suspicious_count += 1;
                }
            }
            // Lone low surrogate without a preceding high surrogate
            0xDC00..=0xDFFF => suspicious_count += 1,
            0x0100.. => word_like_count += 1,
            _ => {}
        }

        i += 2;
    }

    if total == 0 {
        return false;
    }

    // Real UTF-16 text has near-zero control characters; binary data with
    // small 16-bit values typically exceeds 5%. 2% provides a safe margin.
    let low_control_ratio = suspicious_count * 100 < total * 2;

    // Binary formats that interleave short ASCII fragments with small
    // length/type fields (e.g. game asset formats) can dodge the control
    // character check above while barely containing any real words: their
    // code units land on ASCII punctuation and Latin-1 symbol values rather
    // than letters, digits, or spaces. Real text is overwhelmingly made of
    // word characters, so require a minimum share of them. Code units above
    // the Latin-1 range (and surrogate pairs) also count as word-like so
    // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,
    // are still recognized -- tag bytes paired with a zero byte can never
    // land there.
    let enough_word_chars = word_like_count * 100 >= total * 30;

    low_control_ratio && enough_word_chars
}

fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {
    let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?];
    if little_endian {
        return Some(u16::from_le_bytes(pair));
    }
    Some(u16::from_be_bytes(pair))
}

#[cfg(test)]
mod tests {
    use super::*;
    use futures_lite::future::{block_on, poll_once};

    #[test]
    fn decodes_and_encodes_legacy_text() {
        for (expected, encoding) in [
            (
                String::from("“NUMPY is useful for array processing.”\n"),
                encoding_rs::WINDOWS_1252,
            ),
            (
                String::from(
                    "«KTX est un format de texture déjà utilisé à côté des images françaises.»\n",
                ),
                encoding_rs::WINDOWS_1252,
            ),
            (
                String::from(
                    "〔迷宫〕这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n",
                ),
                encoding_rs::GBK,
            ),
            (
                String::from(
                    "€\x02£\x02Symboles monétaires déjà utilisés à côté des factures françaises.\n",
                ),
                encoding_rs::WINDOWS_1252,
            ),
            (
                String::from(
                    "\x1fќерка\x1fОва е обичен македонски текст за проверка на кодирањето.\n",
                ),
                encoding_rs::WINDOWS_1251,
            ),
            (
                format!(
                    "я{}{}",
                    "x".repeat(1024 * 1024),
                    "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(1000)
                ),
                encoding_rs::WINDOWS_1251,
            ),
            (
                String::from("ясно, что это обычный русский текст. Сегодня хорошая погода.\n"),
                encoding_rs::WINDOWS_1251,
            ),
            (
                String::from("ящик содержит обычный русский текст. Сегодня хорошая погода.\n"),
                encoding_rs::WINDOWS_1251,
            ),
            (
                format!(
                    "\x1b[0m{}{}",
                    "x".repeat(1024 * 1024),
                    " déjà été à côté français ".repeat(1000)
                ),
                encoding_rs::WINDOWS_1252,
            ),
        ] {
            let (bytes, _, _) = encoding.encode(&expected);
            let decoded = decode_text(bytes.clone().into_owned()).unwrap();

            assert_eq!(decoded.encoding, encoding);
            assert_eq!(decoded.text, expected);
            assert!(!decoded.has_bom);
            assert_eq!(
                encode_text(decoded.text, decoded.encoding, decoded.has_bom),
                bytes.as_ref()
            );
        }
    }

    #[test]
    fn preserves_unicode_boms() {
        let expected = "七个人上山,下个月再来。";
        for encoding in [UTF_8, UTF_16LE, UTF_16BE] {
            let bytes = encode_text(expected.to_owned(), encoding, true);
            assert_eq!(
                decode_byte_header(&bytes),
                (Some(encoding), ByteContent::Unknown)
            );
            let decoded = decode_text(bytes.clone()).unwrap();

            assert_eq!(decoded.text, expected);
            assert_eq!(decoded.encoding, encoding);
            assert!(decoded.has_bom);
            assert_eq!(
                encode_text(decoded.text, decoded.encoding, decoded.has_bom),
                bytes
            );
        }
    }

    #[test]
    fn detects_known_binary_headers() {
        let cases: &[(&[u8], &str)] = &[
            (b"II*\x00\x08\x00\x00\x00", "TIFF little endian"),
            (b"MM\x00*\x00\x00\x00\x08", "TIFF big endian"),
            (b"glTF\x02\x00\x00\x00\x10\x00\x00\x00", "glTF binary"),
            (b"\x7FELF\x02\x01\x01\x00", "ELF"),
            (b"\x1F\x8B\x08\x00", "gzip"),
            (b"SQLite format 3\x00", "SQLite"),
            (b"\x93NUMPY\x01\x00\x76\x00{'descr': '|u1'", "NumPy 1"),
            (b"\x93NUMPY\x02\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 2"),
            (b"\x93NUMPY\x03\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 3"),
            (b"\xABKTX 11\xBB\r\n\x1A\n\x04\x03\x02\x01", "KTX 1"),
            (b"\xABKTX 20\xBB\r\n\x1A\n\x00\x00\x00\x00", "KTX 2"),
            (
                b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x00\x01",
                "pcap big endian",
            ),
            (
                b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x01\x00\x00\x00",
                "pcap little endian",
            ),
            (b"GGUF\x03\x00\x00\x00", "GGUF"),
            (b"BM\x36\x10\x0E\x00\x00\x00\x00\x00\x36\x00\x00\x00", "BMP"),
            (
                b"\x90\x01\x00\x00\x00\x00\x00\x00{\"__metadata__\"",
                "safetensors",
            ),
        ];
        for (header, label) in cases {
            let mut bytes = header.to_vec();
            bytes.resize(FILE_ANALYSIS_BYTES, b'A');
            assert_eq!(analyze_byte_content(&bytes), ByteContent::Binary, "{label}");
            assert_eq!(
                decode_text(bytes).err().map(|error| error.to_string()),
                Some(String::from("Binary files cannot be decoded into text")),
                "{label}"
            );
        }
        assert_eq!(
            analyze_byte_content(b"\x28\xB5\x2F\xFD\x20\x00\x01\x00\x00"),
            ByteContent::Binary
        );
        assert_eq!(
            analyze_byte_content(
                b"\x28\xB5\x2F\xFD\x20\x20\x01\x01\x00needle-abcdefghijklmnopqrstuvwxy"
            ),
            ByteContent::Binary
        );
        for text in [
            "FLV\x01video/x-flv\x01Flash video\n",
            "v/1\x01api/users\x01version one route\n",
            "!<arch>\nThis documents the archive identifier.\n",
            "glTF Sample Models\n",
            "OTTO GmbH\n",
            "GGUF is a format\n",
            "wOFF = 1\n",
            "wOF2 = 1\n",
        ] {
            assert_eq!(
                analyze_byte_content(text.as_bytes()),
                ByteContent::Unknown,
                "{text:?}"
            );
        }
    }

    #[test]
    fn binary_headers_are_sorted_and_prefix_free() {
        for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {
            for pair in headers.windows(2) {
                assert!(pair[0] < pair[1], "{pair:?}");
                assert_eq!(pair[1].strip_prefix(pair[0]), None, "{pair:?}");
            }
        }
    }

    #[test]
    fn binary_header_lookup_matches_linear_scan() {
        for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {
            let check = |bytes: &[u8]| {
                let expected = headers.iter().any(|header| bytes.starts_with(header));
                assert_eq!(has_binary_header(bytes, headers), expected, "{bytes:?}");
            };

            for header in headers {
                for length in 0..=header.len() {
                    check(&header[..length]);
                }
                for suffix in [0, u8::MAX] {
                    let mut bytes = header.to_vec();
                    bytes.push(suffix);
                    check(&bytes);
                }
                for index in 0..header.len() {
                    let mut bytes = header.to_vec();
                    for byte in 0..=u8::MAX {
                        bytes[index] = byte;
                        check(&bytes);
                    }
                }
            }
        }
    }

    #[test]
    fn classifies_nul_free_content() {
        let mut protobuf_like = Vec::new();
        while protobuf_like.len() < FILE_ANALYSIS_BYTES {
            protobuf_like.extend_from_slice(b"\x0a\x0c/conv1/Conv_0\x12\x02\x08\x01\x1a\x04Conv\x22\x06\x08\x03\x10\x05\x18\x07");
        }
        let random_like = pseudo_random_bytes(FILE_ANALYSIS_BYTES)
            .into_iter()
            .map(|byte| if byte == 0 { 1 } else { byte })
            .collect::<Vec<_>>();
        let russian_text = "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(20);
        let (russian, _, _) = encoding_rs::WINDOWS_1251.encode(&russian_text);
        let chinese_text =
            "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20);
        let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);
        let ansi_log =
            "\x1b[32mINFO\x1b[0m \x1b[1m[server]\x1b[0m ok id=\x1b[33m42\x1b[0m\n".repeat(40);
        let control_delimited = "alpha\x01beta\x01gamma\x01delta\x02\n".repeat(60);
        let overstrike =
            "N\x08NA\x08AM\x08ME\x08E\n       ls - list directory contents\n".repeat(30);

        let cases: [(&str, &[u8], ByteContent); 7] = [
            ("protobuf-like", &protobuf_like, ByteContent::Unknown),
            ("random without NUL", &random_like, ByteContent::Unknown),
            ("windows-1251", &russian, ByteContent::Unknown),
            ("gbk", &chinese, ByteContent::Unknown),
            ("ansi log", ansi_log.as_bytes(), ByteContent::Unknown),
            (
                "control delimited",
                control_delimited.as_bytes(),
                ByteContent::Unknown,
            ),
            (
                "nroff overstrike",
                overstrike.as_bytes(),
                ByteContent::Unknown,
            ),
        ];
        for (label, bytes, expected) in cases {
            assert_eq!(analyze_byte_content(bytes), expected, "{label}");
        }

        let thai = "สวัสดีชาวโลกนี่คือข้อความทดสอบภาษาไทย";
        for encoding in [UTF_16LE, UTF_16BE] {
            let bytes = encode_text(String::from(thai), encoding, false);
            assert_eq!(decode_byte_header(&bytes), (None, ByteContent::Unknown));
            assert_eq!(decode_text(bytes.clone()).unwrap().encoding, UTF_8);
            assert_eq!(encoding.decode(&bytes).0, thai);
        }
    }

    #[test]
    fn streaming_decoder_matches_whole_file_decoding() {
        let chinese_text =
            "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20_000);
        let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);
        assert!(chinese.len() > 1024 * 1024);
        let (russian, _, _) =
            encoding_rs::WINDOWS_1251.encode("Съешь же ещё этих мягких французских булок\n");
        let mut ascii_then_russian = "plain ascii line\n".repeat(5_000).into_bytes();
        ascii_then_russian.extend_from_slice(&russian);
        let mut utf16 = vec![0xFF, 0xFE];
        utf16.extend(
            "utf-16 text\n"
                .repeat(100)
                .encode_utf16()
                .flat_map(u16::to_le_bytes),
        );

        let interior_bom = "prefix  déjà été à côté français ".repeat(10);
        let (interior_bom_bytes, _, _) = encoding_rs::WINDOWS_1252.encode(&interior_bom);
        assert_eq!(
            decode_text(interior_bom_bytes.to_vec()).unwrap().text,
            interior_bom
        );

        for (label, bytes) in [
            ("large gbk", chinese.into_owned()),
            ("ascii prefix then windows-1251", ascii_then_russian),
            ("utf-16 with bom", utf16),
            ("pure ascii", b"just ascii\n".repeat(3)),
            ("empty", Vec::new()),
            ("truncated utf-16", b"\xFF\xFEa\0b".to_vec()),
            (
                "short utf-16le",
                encode_text(String::from("needleé"), UTF_16LE, false),
            ),
            (
                "short utf-16be",
                encode_text(String::from("needleé"), UTF_16BE, false),
            ),
            (
                "utf-16be fullwidth asterisk",
                encode_text(
                    String::from("* This is a fullwidth asterisk followed by ordinary text.\n"),
                    UTF_16BE,
                    false,
                ),
            ),
            (
                "utf-16be pickle prefix",
                encode_text(
                    String::from("考试 instructions for tomorrow\n"),
                    UTF_16BE,
                    false,
                ),
            ),
            (
                "utf-16be ftyp",
                encode_text(
                    String::from("AB晴祰 ordinary UTF-16 text\n"),
                    UTF_16BE,
                    false,
                ),
            ),
            (
                "interior BOM in windows-1252",
                interior_bom_bytes.into_owned(),
            ),
            (
                "interior UTF-8 BOM",
                "prefix \u{feff}suffix\x1b[0m".as_bytes().to_vec(),
            ),
        ] {
            let expected = decode_text(bytes.clone()).unwrap();
            for read_limit in [usize::MAX, 8 * 1024, 7] {
                let mut source = ShortReads {
                    inner: io::Cursor::new(bytes.clone()),
                    limit: 1,
                };
                let (header, _) = read_byte_header(&mut source).unwrap();
                let (bom_encoding, byte_content) = decode_byte_header(&header);
                assert_ne!(byte_content, ByteContent::Binary, "{label}");
                source.inner.set_position(0);
                source.limit = read_limit;
                let encoding = bom_encoding
                    .or(byte_content.encoding())
                    .unwrap_or_else(|| block_on(detect_encoding(&mut source)).unwrap());
                source.inner.set_position(0);
                let mut reader = DecodingReader::new(&mut source, encoding);
                let mut streamed = String::new();
                reader.read_to_string(&mut streamed).unwrap();
                assert_eq!(
                    streamed, expected.text,
                    "{label}, read_limit = {read_limit}"
                );
                assert_eq!(
                    reader.encoding(),
                    expected.encoding,
                    "{label}, read_limit = {read_limit}"
                );
            }
        }
    }

    #[test]
    fn encoding_detection_yields_between_blocks() {
        let mut source = io::Cursor::new(vec![b'a'; 2 * DECODING_BLOCK_BYTES]);
        assert!(block_on(poll_once(detect_encoding(&mut source))).is_none());
        assert_eq!(source.position(), DECODING_BLOCK_BYTES as u64);
    }

    /// reproduction of issue #50785
    fn build_pcm16_wav_bytes() -> Vec<u8> {
        let header: Vec<u8> = vec![
            /*  RIFF header  */
            0x52, 0x49, 0x46, 0x46, // "RIFF"
            0xc6, 0xcf, 0x00, 0x00, // file size: 8
            0x57, 0x41, 0x56, 0x45, // "WAVE"
            /*  fmt chunk  */
            0x66, 0x6d, 0x74, 0x20, // "fmt "
            0x10, 0x00, 0x00, 0x00, // chunk size: 16
            0x01, 0x00, // format: PCM (1)
            0x01, 0x00, // channels: 1 (mono)
            0x80, 0x3e, 0x00, 0x00, // sample rate: 16000
            0x00, 0x7d, 0x00, 0x00, // byte rate: 32000
            0x02, 0x00, // block align: 2
            0x10, 0x00, // bits per sample: 16
            /*  LIST chunk  */
            0x4c, 0x49, 0x53, 0x54, // "LIST"
            0x1a, 0x00, 0x00, 0x00, // chunk size: 26
            0x49, 0x4e, 0x46, 0x4f, // "INFO"
            0x49, 0x53, 0x46, 0x54, // "ISFT"
            0x0d, 0x00, 0x00, 0x00, // sub-chunk size: 13
            0x4c, 0x61, 0x76, 0x66, 0x36, 0x32, 0x2e, 0x33, // "Lavf62.3"
            0x2e, 0x31, 0x30, 0x30, 0x00, // ".100\0"
            /* padding byte for word alignment */
            0x00, // data chunk header
            0x64, 0x61, 0x74, 0x61, // "data"
            0x80, 0xcf, 0x00, 0x00, // chunk size
        ];

        let mut bytes = header;

        // fill remaining space up to `FILE_ANALYSIS_BYTES` with synthetic PCM
        let audio_bytes_needed = FILE_ANALYSIS_BYTES - bytes.len();
        for i in 0..(audio_bytes_needed / 2) {
            let sample = (i & 0xFF) as u8;
            bytes.push(sample); // low byte: varies
            bytes.push(0x00); // high byte: zero for small values
        }

        bytes
    }

    #[test]
    fn test_pcm16_wav_detected_as_binary() {
        let wav_bytes = build_pcm16_wav_bytes();
        assert_eq!(wav_bytes.len(), FILE_ANALYSIS_BYTES);

        let result = analyze_byte_content(&wav_bytes);
        assert_eq!(
            result,
            ByteContent::Binary,
            "PCM 16-bit WAV should be detected as Binary via RIFF header"
        );
    }

    #[test]
    fn test_le16_binary_not_misdetected_as_utf16le() {
        let mut bytes = b"FAKE".to_vec();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            let sample = (bytes.len() & 0xFF) as u8;
            bytes.push(sample);
            bytes.push(0x00);
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        let result = analyze_byte_content(&bytes);
        assert_eq!(
            result,
            ByteContent::Binary,
            "LE 16-bit binary with control characters should be detected as Binary"
        );
    }

    #[test]
    fn test_be16_binary_not_misdetected_as_utf16be() {
        let mut bytes = b"FAKE".to_vec();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.push(0x00);
            let sample = (bytes.len() & 0xFF) as u8;
            bytes.push(sample);
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        let result = analyze_byte_content(&bytes);
        assert_eq!(
            result,
            ByteContent::Binary,
            "BE 16-bit binary with control characters should be detected as Binary"
        );
    }

    // Mimics binary formats that interleave short ASCII fragments with small
    // length/type fields (as seen in some game/asset binary formats, e.g.
    // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's
    // null-byte pattern for ASCII, but the low bytes are mostly non-word
    // "tag" values rather than real letters/digits/spaces.
    fn build_tag_interleaved_binary_bytes() -> Vec<u8> {
        let mut bytes = Vec::new();
        let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];
        let mut i = 0;
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.push(tags[i % tags.len()]);
            bytes.push(0x00);
            i += 1;
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);
        bytes
    }

    #[test]
    fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {
        let bytes = build_tag_interleaved_binary_bytes();
        assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);

        let result = analyze_byte_content(&bytes);
        assert_eq!(
            result,
            ByteContent::Binary,
            "binary data with sparse non-word low bytes and null high bytes \
             should not be misdetected as UTF-16LE text"
        );
    }

    #[test]
    fn test_utf16le_text_detected_as_utf16le() {
        let text = "Hello, world! This is a UTF-16 test string. ";
        let mut bytes = Vec::new();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
    }

    #[test]
    fn test_utf16be_text_detected_as_utf16be() {
        let text = "Hello, world! This is a UTF-16 test string. ";
        let mut bytes = Vec::new();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
    }

    #[test]
    fn test_utf16le_cyrillic_text_detected_as_utf16le() {
        let text = "Привет, мир! Это тестовая строка в UTF-16. ";
        let mut bytes = Vec::new();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
    }

    #[test]
    fn test_utf16be_greek_text_detected_as_utf16be() {
        let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";
        let mut bytes = Vec::new();
        while bytes.len() < FILE_ANALYSIS_BYTES {
            bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
        }
        bytes.truncate(FILE_ANALYSIS_BYTES);

        assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
    }

    #[test]
    fn test_known_binary_headers() {
        let cases: &[(&[u8], &str)] = &[
            (b"RIFF\x00\x00\x00\x00WAVE", "WAV"),
            (b"RIFF\x00\x00\x00\x00AVI ", "AVI"),
            (b"OggS\x00\x02", "OGG"),
            (b"fLaC\x00\x00", "FLAC"),
            (b"ID3\x03\x00", "MP3 ID3v2"),
            (b"\xFF\xFB\x90\x00", "MP3 MPEG1 Layer3"),
            (b"\xFF\xF3\x90\x00", "MP3 MPEG2 Layer3"),
        ];

        for (header, label) in cases {
            let mut bytes = header.to_vec();
            bytes.resize(FILE_ANALYSIS_BYTES, 0x41); // pad with 'A'
            assert_eq!(
                analyze_byte_content(&bytes),
                ByteContent::Binary,
                "{label} should be detected as Binary"
            );
        }
    }

    struct ShortReads<R> {
        inner: R,
        limit: usize,
    }

    impl<R: Read> Read for ShortReads<R> {
        fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
            let len = buffer.len().min(self.limit);
            self.inner.read(&mut buffer[..len])
        }
    }

    fn pseudo_random_bytes(len: usize) -> Vec<u8> {
        let mut state = 0x2545_F491_4F6C_DD1Du64;
        (0..len)
            .map(|_| {
                state = state
                    .wrapping_mul(6364136223846793005)
                    .wrapping_add(1442695040888963407);
                (state >> 56) as u8
            })
            .collect()
    }
}