Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

49.0 KB
Raw Blame
use std::borrow::Cow;

use encoding_rs::Encoding;

use super::*;
use moli_charset_parser::HTML_META_CHARSET_PRESCAN_LIMIT;

fn gbk_bytes(input: &str) -> Vec<u8> {
    encoding_rs::GBK.encode(input).0.into_owned()
}

fn test_legacy_encoding_detector(
    bytes: &[u8],
    url_hint: Option<&str>,
) -> Option<&'static Encoding> {
    (!bytes.is_empty() && url_hint == Some("https://legacy.example/"))
        .then_some(encoding_rs::IBM866)
}

fn unexpected_legacy_encoding_detector(
    _bytes: &[u8],
    _url_hint: Option<&str>,
) -> Option<&'static Encoding> {
    panic!("declared encoding must take precedence over heuristic detection")
}

#[test]
fn xml_document_ignores_html_meta_across_chunk_splits() {
    for declaration in ["", "<?xml version='1.0' encoding='UTF-8'?>"] {
        for meta in [
            "<meta charset='windows-1252'/>",
            "<meta http-equiv='content-type' content='text/html; charset=windows-1252'/>",
        ] {
            let source = format!("{declaration}<root>{meta}<value>café</value></root>");
            assert_xml_document_decodes_across_chunk_splits(
                &[],
                source.as_bytes(),
                &source,
                "UTF-8",
            );
        }
    }
}

fn assert_xml_document_decodes_across_chunk_splits(
    headers: &[(String, Vec<u8>)],
    bytes: &[u8],
    source: &str,
    encoding: &str,
) {
    for split in 0..=bytes.len() {
        let mut decoder = DocumentStreamingDecoder::new_xml_document(headers);
        let mut decoded = decoder.push(&bytes[..split]).concat();
        decoded.push_str(&decoder.push(&bytes[split..]).concat());
        decoded.push_str(&decoder.finish().unwrap_or_default());
        assert_eq!(decoded, source, "{encoding}: split {split}");
        assert_eq!(decoder.document_encoding_name(), encoding, "split {split}");
    }
    let mut decoder = DocumentStreamingDecoder::new_xml_document(headers);
    let mut decoded = String::new();
    for byte in bytes {
        decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat());
    }
    decoded.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(decoded, source, "{encoding}: single-byte chunks");
    assert_eq!(decoder.document_encoding_name(), encoding);
}

#[test]
fn xml_document_preserves_encoding_precedence_across_chunk_splits() {
    let source = "<?xml version='1.0' encoding='windows-1252'?>\
                  <root><meta charset='UTF-8'/><value>café</value></root>";
    let legacy = encoding_rs::WINDOWS_1252.encode(source).0.into_owned();
    assert_xml_document_decodes_across_chunk_splits(&[], &legacy, source, "windows-1252");

    let headers = [(
        "Content-Type".to_owned(),
        b"application/xml; charset=utf-8".to_vec(),
    )];
    assert_xml_document_decodes_across_chunk_splits(&headers, source.as_bytes(), source, "UTF-8");

    let headers = [(
        "Content-Type".to_owned(),
        b"application/xml; charset=windows-1252".to_vec(),
    )];
    let bom_body = [b"\xef\xbb\xbf".as_slice(), source.as_bytes()].concat();
    assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, "UTF-8");

    let source = "<?xml version='1.0' encoding='UTF-16'?>\
                  <root><meta charset='windows-1252'/><value>café</value></root>";
    for (bytes, bom, encoding) in [
        (utf16le_bytes(source), [0xff, 0xfe], "UTF-16LE"),
        (utf16be_bytes(source), [0xfe, 0xff], "UTF-16BE"),
    ] {
        assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding);
        let bom_body = [bom.as_slice(), &bytes].concat();
        assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, encoding);
    }
}

#[test]
fn xml_document_ignores_encoding_attributes_in_processing_instructions() {
    let source = "<?xml-stylesheet encoding='windows-1252'?>\
                  <root><value>café</value></root>";
    assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8");
}

#[test]
fn xml_document_selects_encoding_at_the_end_of_its_declaration() {
    let mut decoder = DocumentStreamingDecoder::new_xml_document(&[]);
    let mut decoded = decoder
        .push(b"<?xml version='1.0' encoding='windows-")
        .concat();
    assert_eq!(decoder.selected_encoding_name(), None);
    decoded.push_str(&decoder.push(b"1252'?>").concat());
    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
    decoded.push_str(&decoder.push(b"<root>caf\xe9</root>").concat());
    decoded.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(
        decoded,
        "<?xml version='1.0' encoding='windows-1252'?><root>café</root>"
    );
}

#[test]
fn xml_document_finishes_empty_or_incomplete_declarations_with_utf8() {
    for source in [
        "",
        "<",
        "<?xml",
        "<?xml version='1.0' encoding='windows-1252'",
    ] {
        assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8");
    }
}

// Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937:
// third_party/blink/renderer/core/html/parser/text_resource_decoder_test.cc,
// XMLDeclPieces. Also exercise the XML/Text policies with the same byte stream.
#[test]
fn chromium_xml_declaration_pieces_follow_the_document_policy() {
    let source = b"<?xml encoding='utf-8'?>foo";
    for (mut decoder, expected_encoding) in [
        (DocumentStreamingDecoder::new(&[]), "UTF-8"),
        (DocumentStreamingDecoder::new_xml_document(&[]), "UTF-8"),
        (
            DocumentStreamingDecoder::new_text_document(
                &[],
                "https://example.test/",
                test_legacy_encoding_detector,
                false,
                None,
            ),
            "windows-1252",
        ),
    ] {
        let mut decoded = String::new();
        for byte in source {
            decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat());
        }
        decoded.push_str(&decoder.finish().unwrap_or_default());
        assert_eq!(decoded.as_bytes(), source);
        assert_eq!(decoder.document_encoding_name(), expected_encoding);
    }
}

// Chromium text_resource_decoder_test.cc, BrokenBOMs (revision above).
#[test]
fn chromium_plain_text_broken_boms_are_flushed_with_the_default_encoding() {
    for (bytes, expected) in [
        (b"\xef\xbb".as_slice(), "ï»"),
        (b"\xff", "ÿ"),
        (b"\xfe", "þ"),
    ] {
        let mut decoder = DocumentStreamingDecoder::new_text_document(
            &[],
            "https://example.test/",
            test_legacy_encoding_detector,
            false,
            None,
        );
        for byte in bytes {
            assert!(decoder.push(std::slice::from_ref(byte)).is_empty());
        }
        assert_eq!(decoder.finish().as_deref(), Some(expected));
        assert_eq!(decoder.document_encoding_name(), "windows-1252");
    }
}

// Adapt Chromium fast/encoding/bom-in-content{,-utf16}.html to the XML policy.
#[test]
fn chromium_bom_in_content_is_preserved_by_xml_decoding() {
    let source = "<?xml version='1.0'?><root>\u{feff}</root>";
    for (bytes, bom, encoding) in [
        (
            source.as_bytes().to_vec(),
            b"\xef\xbb\xbf".as_slice(),
            "UTF-8",
        ),
        (utf16le_bytes(source), b"\xff\xfe", "UTF-16LE"),
        (utf16be_bytes(source), b"\xfe\xff", "UTF-16BE"),
    ] {
        let bytes = [bom, &bytes].concat();
        assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding);
    }
}

#[test]
fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
    let input = "{\"name\":\"Gülçek\"}";
    for split in 0..=input.len() {
        let mut decoder = DocumentStreamingDecoder::new_text_document(
            &[],
            "https://example.test/",
            unexpected_legacy_encoding_detector,
            true,
            Some("GBK"),
        );
        let mut output = decoder.push(&input.as_bytes()[..split]).join("");
        output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
        output.push_str(&decoder.finish().unwrap_or_default());
        assert_eq!(output, input, "split {split}");
        assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
    }
}

#[test]
fn plain_text_document_honors_transport_charset() {
    let headers = [(
        "Content-Type".to_owned(),
        b"text/plain; charset=gbk".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new_text_document(
        &headers,
        "https://example.test/",
        unexpected_legacy_encoding_detector,
        false,
        Some("UTF-8"),
    );
    let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
    output.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(output, "<b>家居</b>");
    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}

#[test]
fn text_document_bom_precedes_transport_charset() {
    let headers = [(
        "Content-Type".to_owned(),
        b"text/plain; charset=gbk".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new_text_document(
        &headers,
        "https://example.test/",
        unexpected_legacy_encoding_detector,
        false,
        Some("windows-1251"),
    );
    assert_eq!(decoder.document_encoding_name(), "GBK");
    assert_eq!(decoder.selected_encoding_name(), None);
    assert!(decoder.push(&[0xef]).is_empty());
    assert!(decoder.push(&[0xbb]).is_empty());
    let mut tail = vec![0xbf];
    tail.extend_from_slice("Gülçek".as_bytes());
    let mut output = decoder.push(&tail).join("");
    output.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(output, "Gülçek");
    assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
    assert_eq!(decoder.document_encoding_name(), "UTF-8");
}

#[test]
fn plain_text_document_retains_legacy_detection_without_a_charset() {
    let mut decoder = DocumentStreamingDecoder::new_text_document(
        &[],
        "https://legacy.example/",
        test_legacy_encoding_detector,
        false,
        None,
    );
    let mut output = decoder.push(&[0x80]).join("");
    output.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(output, "А");
    assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
}

#[test]
fn plain_text_document_inherits_encoding_before_detection_across_chunk_splits() {
    for (input, encoding) in [
        ("plain ASCII", encoding_rs::UTF_8),
        ("<meta charset=gbk>吴姓-姓氏渊源", encoding_rs::UTF_8),
        ("<?xml encoding='utf-8'?>家居", encoding_rs::GBK),
    ] {
        let bytes = encoding.encode(input).0;
        for split in 0..=bytes.len() {
            let mut decoder = DocumentStreamingDecoder::new_text_document(
                &[],
                "https://example.test/",
                unexpected_legacy_encoding_detector,
                false,
                Some(encoding.name()),
            );
            assert_eq!(decoder.document_encoding_name(), encoding.name());
            let mut output = decoder.push(&bytes[..split]).concat();
            output.push_str(&decoder.push(&bytes[split..]).concat());
            output.push_str(&decoder.finish().unwrap_or_default());
            assert_eq!(output, input, "{} split {split}", encoding.name());
            assert_eq!(decoder.selected_encoding_name(), Some(encoding.name()));
        }
    }
}

#[test]
fn invalid_inherited_text_encoding_does_not_disable_detection() {
    let mut decoder = DocumentStreamingDecoder::new_text_document(
        &[],
        "https://legacy.example/",
        test_legacy_encoding_detector,
        false,
        Some("not-an-encoding"),
    );
    let mut output = decoder.push(&[0x80]).concat();
    output.push_str(&decoder.finish().unwrap_or_default());
    assert_eq!(output, "А");
    assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
}

#[test]
fn plain_text_document_ignores_literal_encoding_declarations() {
    for prefix in [
        "<meta charset=utf-8>",
        "<?xml version='1.0' encoding='utf-8'?>",
    ] {
        let mut input = prefix.as_bytes().to_vec();
        input.push(0xe9);
        let mut decoder = DocumentStreamingDecoder::new_text_document(
            &[],
            "https://example.test/",
            test_legacy_encoding_detector,
            false,
            None,
        );
        let mut output = decoder.push(&input).join("");
        output.push_str(&decoder.finish().unwrap_or_default());
        assert_eq!(output, format!("{prefix}é"));
        assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
    }
}

#[test]
fn content_type_charset_is_selected() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=gbk".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(decoder.push(&gbk_bytes("太平洋")), vec!["太平洋"]);
    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}

#[test]
fn meta_charset_is_selected_without_header_charset() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut input = b"<!doctype html><meta charset=\"gbk\"><p>".to_vec();
    input.extend_from_slice(&gbk_bytes("家居"));
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(
        decoder.push(&input),
        vec!["<!doctype html><meta charset=\"gbk\"><p>家居"]
    );
    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}

#[test]
fn meta_charset_can_be_split_across_chunks() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(
        decoder.push(b"<!doctype html><meta char"),
        vec!["<!doctype html><meta char"]
    );
    let mut tail = b"set=\"gbk\"><p>".to_vec();
    tail.extend_from_slice(&gbk_bytes("装修"));

    assert_eq!(decoder.push(&tail), vec!["set=\"gbk\"><p>装修"]);
    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}

#[test]
fn ascii_prefix_streams_while_charset_sniffing_continues() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(
        decoder.push(b"<!doctype html><script src=\"/gate.js\"></script>"),
        vec!["<!doctype html><script src=\"/gate.js\"></script>"]
    );
    assert_eq!(decoder.selected_encoding_name(), None);
    assert_eq!(decoder.finish(), None);
    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
}

#[test]
fn later_meta_charset_decodes_unemitted_non_ascii_after_ascii_prefix() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut decoder = DocumentStreamingDecoder::new(&headers);
    let mut tail = b"<meta charset=\"gbk\"><p>".to_vec();
    tail.extend_from_slice(&gbk_bytes("家居"));

    assert_eq!(decoder.push(b"<!doctype html>"), vec!["<!doctype html>"]);
    assert_eq!(decoder.push(&tail), vec!["<meta charset=\"gbk\"><p>家居"]);
    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}

#[test]
fn meta_charset_after_1024_bytes_still_in_head_is_selected() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut input = vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT];
    input.extend_from_slice(b"<meta charset=\"gbk\"><p>");
    input.extend_from_slice(&gbk_bytes("家居"));
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    let decoded = decoder.push(&input).join("");

    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
    assert!(decoded.contains("家居"));
}

#[test]
fn meta_charset_crossing_1024_byte_boundary_is_selected_while_in_head() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let partial_meta = b"<meta char";
    let mut input = vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - partial_meta.len()];
    input.extend_from_slice(partial_meta);
    input.extend_from_slice(b"set=\"gbk\"><p>");
    input.extend_from_slice(&gbk_bytes("家居"));
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    let decoded = decoder.push(&input).join("");

    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
    assert!(decoded.contains("家居"));
}

#[test]
fn meta_charset_after_1024_bytes_after_head_is_ignored() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut input = b"<body>".to_vec();
    input.extend(vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - input.len()]);
    input.extend_from_slice(b"<meta charset=\"gbk\"><p>");
    input.extend_from_slice(&gbk_bytes("家居"));
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    let decoded = decoder.push(&input).join("");

    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
    assert!(!decoded.contains("家居"));
}

#[test]
fn meta_charset_starting_before_1024_bytes_after_head_is_selected() {
    let headers: Vec<(String, Vec<u8>)> = vec![("Content-Type".to_owned(), b"text/html".to_vec())];
    let mut input = b"</head>".to_vec();
    input.extend(vec![
        b' ';
        HTML_META_CHARSET_PRESCAN_LIMIT - 1 - input.len()
    ]);
    input.extend_from_slice(b"<meta charset=\"gbk\"><p>");
    input.extend_from_slice(&gbk_bytes("家居"));
    let mut decoder = DocumentStreamingDecoder::new_with_fallback(&headers, Some("windows-1252"));

    let decoded = decoder.push(&input).join("");

    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
    assert!(decoded.contains("家居"));
}

#[test]
fn meta_charset_prescan_matches_real_meta_start_tags_only() {
    use moli_charset_parser::sniff_html_meta_charset;

    assert_eq!(
        sniff_html_meta_charset(br#"<metadata charset="gbk"><meta charset="utf-8">"#)
            .map(Encoding::name),
        Some("UTF-8")
    );
    assert_eq!(
        sniff_html_meta_charset(br#"<metaverse charset="gbk"><p>ok</p>"#),
        None
    );
}

#[test]
fn meta_charset_prescan_ignores_script_text_and_requires_pragma_for_content() {
    use moli_charset_parser::sniff_html_meta_charset;

    assert_eq!(
        sniff_html_meta_charset(
            br#"<script>document.write('<meta charset="gbk">')</script><meta charset="utf-8">"#
        )
        .map(Encoding::name),
        Some("UTF-8")
    );
    assert_eq!(
        sniff_html_meta_charset(br#"<meta content="text/html; charset=gbk">"#),
        None
    );
    assert_eq!(
        sniff_html_meta_charset(
            br#"<meta http-equiv="content-type" content="text/html; charset=gbk">"#
        ),
        Some(encoding_rs::GBK)
    );
}

#[test]
fn gbk_multibyte_can_be_split_across_chunks() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=gbk".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert!(decoder.push(&[0xCC]).is_empty());
    assert_eq!(decoder.push(&[0xAB]), vec!["太"]);
}

#[test]
fn bom_wins_over_header_charset() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=gbk".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(decoder.push(&[0xEF]), Vec::<String>::new());
    assert_eq!(decoder.push(&[0xBB]), Vec::<String>::new());
    assert_eq!(decoder.push(&[0xBF, b'o', b'k']), vec!["ok"]);
    assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
}

#[test]
fn unknown_charset_falls_back_to_html_default_on_finish() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=x-unknown".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(decoder.push(b"<p>ok"), vec!["<p>ok"]);
    assert_eq!(decoder.finish(), None);
    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
}

#[test]
fn unlabelled_html_document_without_detector_uses_html_default() {
    let (text, encoding) = decode_html_document(b"\x80\x80 Hello", &[]);

    assert_eq!(encoding, "windows-1252");
    assert_eq!(text, "\u{20ac}\u{20ac} Hello");
}

#[test]
fn injected_legacy_content_detector_receives_url_hint() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=x-unknown".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new_with_legacy_encoding_detector(
        &headers,
        "https://legacy.example/",
        test_legacy_encoding_detector,
    );
    let mut text = decoder.push(b"\x80\x80 Hello").join("");
    if let Some(tail) = decoder.finish() {
        text.push_str(&tail);
    }

    let encoding = decoder
        .selected_encoding_name()
        .expect("finishing must select an encoding");
    assert_eq!(encoding, "IBM866");
    assert_eq!(text, "\u{410}\u{410} Hello");
}

#[test]
fn declared_encodings_precede_injected_legacy_detector() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=gbk".to_vec(),
    )];
    let mut header_decoder = DocumentStreamingDecoder::new_with_legacy_encoding_detector(
        &headers,
        "https://legacy.example/",
        unexpected_legacy_encoding_detector,
    );
    assert_eq!(header_decoder.push(&gbk_bytes("太平洋")), vec!["太平洋"]);

    let mut meta_decoder = DocumentStreamingDecoder::new_with_legacy_encoding_detector(
        &[],
        "https://legacy.example/",
        unexpected_legacy_encoding_detector,
    );
    let mut input = b"<meta charset=\"gbk\">".to_vec();
    input.extend_from_slice(&gbk_bytes("太平洋"));
    assert_eq!(
        meta_decoder.push(&input),
        vec!["<meta charset=\"gbk\">太平洋"]
    );
}

#[test]
fn explicit_fallback_keeps_core_decoder_deterministic() {
    let (text, encoding) =
        decode_html_document_with_fallback(b"\x80\x80 Hello", &[], Some("windows-1252"));

    assert_eq!(encoding, "windows-1252");
    assert_eq!(text, "\u{20ac}\u{20ac} Hello");
}

#[test]
fn no_label_html_document_can_inherit_parent_fallback() {
    let (text, encoding) = decode_html_document_with_fallback(&gbk_bytes("家居"), &[], Some("GBK"));

    assert_eq!(encoding, "GBK");
    assert_eq!(text, "家居");
}

#[test]
fn utf32_little_endian_bom_is_treated_as_utf16le_bom() {
    let (text, encoding) = decode_html_document(&[0xFF, 0xFE, 0x00, 0x00, b'<', 0x00], &[]);

    assert_eq!(encoding, "UTF-16LE");
    assert!(text.starts_with('\0'));
}

#[test]
fn document_decoding_removes_only_one_bom() {
    let (utf8_text, utf8_encoding) =
        decode_html_document(&[0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF], &[]);
    let (utf16le_text, utf16le_encoding) = decode_html_document(&[0xFF, 0xFE, 0xFF, 0xFE], &[]);
    let (utf16be_text, utf16be_encoding) = decode_html_document(&[0xFE, 0xFF, 0xFE, 0xFF], &[]);

    assert_eq!(utf8_encoding, "UTF-8");
    assert_eq!(utf8_text, "\u{feff}");
    assert_eq!(utf16le_encoding, "UTF-16LE");
    assert_eq!(utf16le_text, "\u{feff}");
    assert_eq!(utf16be_encoding, "UTF-16BE");
    assert_eq!(utf16be_text, "\u{feff}");
}

#[test]
fn form_submission_uses_first_valid_accept_charset_label() {
    let encoding = form_submission_encoding(Some("unknown iso-8859-1 gbk"), "GBK");

    assert_eq!(encoding.name(), "windows-1252");
}

#[test]
fn form_submission_falls_back_to_document_character_set() {
    let encoding = form_submission_encoding(None, "gbk");

    assert_eq!(encoding.name(), "GBK");
}

#[test]
fn charset_sentinel_name_matches_ascii_case_insensitively() {
    assert!(is_charset_sentinel_name("_charset_"));
    assert!(is_charset_sentinel_name("_CHARSET_"));
    assert!(is_charset_sentinel_name("_Charset_"));
    assert!(!is_charset_sentinel_name("_charſet_"));
}

#[test]
fn form_urlencoded_serializer_uses_selected_legacy_encoding() {
    let encoded = form_urlencoded_serialize_pairs([("q", "家居")], encoding_rs::GBK);

    assert_eq!(encoded, "q=%BC%D2%BE%D3");
}

#[test]
fn form_urlencoded_serializer_uses_numeric_references_for_unmappable_text() {
    let encoded = form_urlencoded_serialize_pairs([("emoji", "💩")], encoding_rs::WINDOWS_1252);

    assert_eq!(encoded, "emoji=%26%23128169%3B");
}

#[test]
fn form_urlencoded_serializer_handles_iso_2022_jp_stateful_unmappables() {
    let encoded = form_urlencoded_serialize_pairs(
        [("utf16", "ABC~¤•★星🌟星★•¤~XYZ")],
        encoding_rs::ISO_2022_JP,
    );

    assert_eq!(
        encoded,
        "utf16=ABC%7E%26%23164%3B%26%238226%3B%1B%24B%21z%401%1B%28B%26%23127775%3B%1B%24B%401%21z%1B%28B%26%238226%3B%26%23164%3B%7EXYZ"
    );
}

#[test]
fn text_decoder_uses_legacy_charset_label_or_utf8() {
    assert_eq!(
        decode_text_for_legacy_web(&gbk_bytes("家居"), Some("gbk")),
        "家居"
    );
    assert_eq!(
        decode_text_for_legacy_web("plain".as_bytes(), None),
        "plain"
    );
}

#[test]
fn html_document_decoder_returns_selected_encoding() {
    let mut bytes = b"<!doctype html><meta charset=\"shift_jis\"><p>".to_vec();
    bytes.extend_from_slice(&encoding_rs::SHIFT_JIS.encode("目次").0);

    let (text, encoding) = decode_html_document(&bytes, &[]);

    assert_eq!(encoding, "Shift_JIS");
    assert!(text.contains("目次"), "text={text}");
}

#[test]
fn classic_script_decoding_inherits_document_character_set() {
    let script = r#"document.body.textContent = "目次";"#;
    let bytes = encoding_rs::SHIFT_JIS.encode(script).0.into_owned();

    assert_eq!(
        decode_classic_script_source(
            &bytes,
            &[(
                "Content-Type".to_owned(),
                b"application/javascript".to_vec()
            )],
            None,
            Some("shift_jis"),
        ),
        script
    );
}

#[test]
fn classic_script_header_charset_wins_over_document_character_set() {
    let script = r#"document.body.textContent = "Привет";"#;
    let bytes = encoding_rs::WINDOWS_1251.encode(script).0.into_owned();

    assert_eq!(
        decode_classic_script_source(
            &bytes,
            &[(
                "Content-Type".to_owned(),
                b"application/javascript; charset=windows-1251".to_vec(),
            )],
            None,
            Some("shift_jis"),
        ),
        script
    );
}

#[test]
fn classic_script_charset_uses_the_extracted_mime_record() {
    for (values, expected) in [
        (
            vec!["text/plain; charset=windows-1252", "text/javascript"],
            "€",
        ),
        (
            vec!["text/javascript; charset=windows-1252", "text/javascript"],
            "€",
        ),
        (
            vec![
                "text/javascript; charset=windows-1252",
                "invalid",
                "text/javascript",
            ],
            "€",
        ),
        (
            vec![
                "text/javascript; charset=windows-1252",
                "text/plain",
                "text/javascript",
            ],
            "€",
        ),
        (
            vec![
                "text/javascript; charset=shift_jis",
                "text/javascript; charset=windows-1252",
            ],
            "€",
        ),
        (
            vec!["text/javascript; charset=bogus", "text/javascript"],
            "€",
        ),
        (vec!["text/javascript; charset=\" windows-1252 \""], "€"),
    ] {
        for combined in [false, true] {
            let headers: Vec<_> = if combined {
                vec![("Content-Type".to_owned(), values.join(", ").into_bytes())]
            } else {
                values
                    .iter()
                    .map(|value| ("Content-Type".to_owned(), value.as_bytes().to_vec()))
                    .collect()
            };
            assert_eq!(
                decode_classic_script_source("€".as_bytes(), &headers, None, Some("utf-8")),
                expected,
                "combined={combined}: {values:?}"
            );
        }
    }
}

#[test]
fn classic_script_charset_attribute_is_fallback_before_document_character_set() {
    let script = r#"document.body.textContent = "目次";"#;
    let bytes = encoding_rs::SHIFT_JIS.encode(script).0.into_owned();

    assert_eq!(
        decode_classic_script_source(&bytes, &[], Some("shift_jis"), Some("gbk")),
        script
    );
}

#[test]
fn classic_script_charset_uses_encoding_standard_label_preprocessing() {
    let script = r#"document.body.textContent = "目次";"#;
    let bytes = encoding_rs::SHIFT_JIS.encode(script).0.into_owned();

    assert_eq!(
        decode_classic_script_source(&bytes, &[], Some(" shift_jis "), Some("utf-8")),
        script
    );
}

#[test]
fn classic_script_bom_wins_over_labels() {
    let script = r#"document.body.textContent = "目次";"#;
    let mut bytes = vec![0xEF, 0xBB, 0xBF];
    bytes.extend_from_slice(script.as_bytes());

    assert_eq!(
        decode_classic_script_source(
            &bytes,
            &[(
                "Content-Type".to_owned(),
                b"application/javascript; charset=gbk".to_vec(),
            )],
            Some("shift_jis"),
            Some("gbk"),
        ),
        script
    );
}

#[test]
fn utf8_decode_removes_only_a_utf8_bom() {
    assert_eq!(decode_utf8(b"\xef\xbb\xbfdiv {}"), "div {}");
    assert_eq!(decode_utf8(b"\xff\xfed\0i\0v\0"), "��d\0i\0v\0");
}

#[test]
fn url_query_encoder_uses_selected_legacy_encoding() {
    let encoded =
        encode_url_query_for_legacy_web("/search?q=家居&safe=a+b%20c#frag", encoding_rs::GBK);

    assert_eq!(encoded, "/search?q=%BC%D2%BE%D3&safe=a+b%20c#frag");
}

#[test]
fn url_query_encoder_percent_encodes_unmappable_entity_fallback() {
    let encoded = encode_url_query_for_legacy_web("/search?q=ChineseG", encoding_rs::WINDOWS_1252);

    assert_eq!(encoded, "/search?q=Chinese%26%2365319%3B");
}

#[test]
fn url_query_encoder_preserves_query_separators_between_components() {
    let encoded = encode_url_query_for_legacy_web(
        "/search?first=家居&second=💩;third=ok#frag",
        encoding_rs::GBK,
    );

    assert_eq!(
        encoded,
        "/search?first=%BC%D2%BE%D3&second=%26%23128169%3B;third=ok#frag"
    );
}

#[test]
fn url_query_encoder_preserves_ampersand_from_encoded_bytes() {
    let encoded = encode_url_query_for_legacy_web("/search?q=Γ", encoding_rs::ISO_2022_JP);

    assert_eq!(encoded, "/search?q=%1B$B&%23%1B(B");
}

#[test]
fn url_query_encoder_percent_encodes_iso_2022_jp_unmappables() {
    let encoded =
        encode_url_query_for_legacy_web("/search?q=Γ\x0E\x0F\x1Bx", encoding_rs::ISO_2022_JP);

    assert_eq!(
        encoded,
        "/search?q=%1B$B&%23%1B(B%26%2365533%3B%26%2365533%3B%26%2365533%3Bx"
    );
}

#[test]
fn url_query_encoder_handles_iso_2022_jp_stateful_output() {
    let encoded = encode_url_query_for_legacy_web("/search?q=¥‾s\\ミ佩", encoding_rs::ISO_2022_JP);

    assert_eq!(encoded, "/search?q=%1B(J\\~s%1B(B\\%1B$B%_PP%1B(B");
}

#[test]
fn url_query_encoder_leaves_utf8_and_queryless_inputs_borrowed() {
    assert!(matches!(
        encode_url_query_for_legacy_web("/search?q=家居", encoding_rs::UTF_8),
        Cow::Borrowed(_)
    ));
    assert!(matches!(
        encode_url_query_for_legacy_web("/search#家居", encoding_rs::GBK),
        Cow::Borrowed(_)
    ));
    assert!(matches!(
        encode_url_query_for_legacy_web("/search#frag?q=家居", encoding_rs::GBK),
        Cow::Borrowed(_)
    ));
}

#[test]
fn meta_declared_utf16_document_decodes_as_utf8() {
    let (text, encoding) = decode_html_document(
        br#"<!DOCTYPE html><meta charset="utf-16"><p>Hello, world!</p>"#,
        &[],
    );

    assert_eq!(encoding, "UTF-8");
    assert!(text.contains("Hello, world!"), "decoded as {text:?}");
}

/// `encoding_rs` has no UTF-16 encoder — the Encoding Standard replaces it
/// with UTF-8 for output — so UTF-16LE input has to be built by hand.
fn utf16le_bytes(input: &str) -> Vec<u8> {
    input
        .encode_utf16()
        .flat_map(|unit| unit.to_le_bytes())
        .collect()
}

#[test]
fn meta_declared_utf16_does_not_override_a_bom() {
    let mut input = vec![0xFF, 0xFE];
    input.extend_from_slice(&utf16le_bytes("<meta charset=\"utf-16\">hi"));

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "UTF-16LE");
    assert!(text.ends_with("hi"), "decoded as {text:?}");
}

#[test]
fn transport_utf16_still_wins_over_the_meta_rewrite() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=utf-16le".to_vec(),
    )];
    let input = utf16le_bytes("<meta charset=\"utf-16\">hi");

    let (text, encoding) = decode_html_document(&input, &headers);

    assert_eq!(encoding, "UTF-16LE");
    assert!(text.ends_with("hi"), "decoded as {text:?}");
}

#[test]
fn bom_less_utf16le_xml_declaration_is_detected() {
    let input = utf16le_bytes(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",
    );

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "UTF-16LE");
    assert!(text.contains("hi"), "decoded as {text:?}");
}

#[test]
fn transport_charset_wins_over_utf16_xml_signature() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=windows-1252".to_vec(),
    )];
    let input = utf16le_bytes(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",
    );

    let (_, encoding) = decode_html_document(&input, &headers);

    assert_eq!(encoding, "windows-1252");
}

fn utf16be_bytes(input: &str) -> Vec<u8> {
    input
        .encode_utf16()
        .flat_map(|unit| unit.to_be_bytes())
        .collect()
}

fn assert_bom_less_utf16_xml_decodes_across_signature_splits(
    input: &[u8],
    expected_text: &str,
    expected_encoding: &str,
) {
    for split in 1..6 {
        let mut decoder = DocumentStreamingDecoder::new(&[]);
        assert_eq!(
            decoder.push(&input[..split]),
            Vec::<String>::new(),
            "split after signature byte {split} must remain buffered"
        );

        let mut decoded = decoder.push(&input[split..]).concat();
        if let Some(tail) = decoder.finish() {
            decoded.push_str(&tail);
        }

        assert_eq!(
            decoder.selected_encoding_name(),
            Some(expected_encoding),
            "wrong encoding after signature byte {split} split"
        );
        assert_eq!(
            decoded, expected_text,
            "misdecoded document after signature byte {split} split"
        );
    }
}

#[test]
fn bom_less_utf16le_xml_signature_survives_every_streaming_split() {
    let source = "<?xml version=\"1.0\"?><html><head></head><body>little endian</body></html>";
    let input = utf16le_bytes(source);

    assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16LE");
}

#[test]
fn bom_less_utf16be_xml_signature_survives_every_streaming_split() {
    let source = "<?xml version=\"1.0\"?><html><head></head><body>big endian</body></html>";
    let input = utf16be_bytes(source);

    assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16BE");
}

#[test]
fn diverged_utf16_xml_signature_prefix_resumes_ascii_streaming() {
    let mut decoder = DocumentStreamingDecoder::new(&[]);

    assert!(decoder.push(b"<").is_empty());
    assert_eq!(
        decoder.push(b"!doctype html><p>ordinary html"),
        vec!["<!doctype html><p>ordinary html"]
    );
    assert_eq!(decoder.finish(), None);
    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
}

#[test]
fn transport_charset_does_not_wait_for_a_utf16_xml_signature_prefix() {
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=windows-1252".to_vec(),
    )];
    let mut decoder = DocumentStreamingDecoder::new(&headers);

    assert_eq!(decoder.push(b"<"), vec!["<"]);
    assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
}

#[test]
fn bom_less_utf16be_xml_declaration_is_detected() {
    let input = utf16be_bytes(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html><head></head><body>hi</body></html>",
    );

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "UTF-16BE");
    assert!(text.contains("hi"), "decoded as {text:?}");
}

#[test]
fn ascii_xml_declaration_selects_its_encoding_after_meta_prescan() {
    let mut input = br#"<?xml version="1.0" encoding="windows-1251"?><p>"#.to_vec();
    input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);
    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1251");
    assert!(text.ends_with("ж"), "decoded as {text:?}");
}

#[test]
fn ascii_xml_declaration_uses_the_compatibility_grammar() {
    for declaration in [
        b"<?xml encoding='cp1251'>".as_slice(),
        b"<?xmlencoding=\"windows-1251\">",
        b"<?xmla<encoding=\"windows-1251\">",
        b"<?xml encoding\x01=\x00\"windows-1251\">",
    ] {
        assert_eq!(
            decode_html_document(declaration, &[]).1,
            "windows-1251",
            "declaration={declaration:?}"
        );
    }
}

#[test]
fn ascii_xml_declaration_rejects_near_misses() {
    for declaration in [
        b"<?XML encoding=\"windows-1251\">".as_slice(),
        b"<?xml ENCODING=\"windows-1251\">",
        b"<?xml encodingencoding=\"windows-1251\">",
        b"<?xml encoding=encoding=\"windows-1251\">",
        b"<?xml encoding=windows-1251>",
        b"<?xml encoding=\" windows-1251\">",
        b"<?xml>encoding=\"windows-1251\">",
        b"<?xml encoding=\"windows-1251'>",
    ] {
        assert_eq!(
            decode_html_document(declaration, &[]).1,
            "windows-1252",
            "declaration={declaration:?}"
        );
    }
}

#[test]
fn meta_and_transport_encodings_precede_ascii_xml_declarations() {
    let input = br#"<?xml encoding="windows-1251"?><meta charset="windows-1253">"#;
    assert_eq!(decode_html_document(input, &[]).1, "windows-1253");

    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=UTF-8".to_vec(),
    )];
    assert_eq!(decode_html_document(input, &headers).1, "UTF-8");
}

#[test]
fn ascii_xml_declaration_rewrites_utf16_and_preserves_replacement() {
    assert_eq!(
        decode_html_document(br#"<?xml encoding="UTF-16">"#, &[]).1,
        "UTF-8"
    );
    assert_eq!(
        decode_html_document(br#"<?xml encoding="ISO-2022-KR">"#, &[]).1,
        "replacement"
    );
}

#[test]
fn ascii_xml_declaration_can_finish_after_the_prescan_boundary() {
    let mut input = br#"<?xml version="1.0" "#.to_vec();
    input.resize(HTML_META_CHARSET_PRESCAN_LIMIT + 1, b' ');
    input.extend_from_slice(br#"encoding="windows-1251"><p>"#);
    input.extend_from_slice(&encoding_rs::WINDOWS_1251.encode("ж").0);

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1251");
    assert!(text.ends_with("ж"), "decoded as {text:?}");
}

#[test]
fn bom_less_utf16le_non_xml_not_detected() {
    let input = utf16le_bytes("<?y>");

    let (_, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
}

#[test]
fn bom_less_utf16le_uppercase_x_not_detected() {
    let input = utf16le_bytes("<?X>");

    let (_, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
}

#[test]
fn bom_less_utf16le_truncated_not_detected() {
    let input: Vec<u8> = vec![0x3C, 0x00, 0x3F, 0x00];

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
    assert_eq!(text, "<\0?\0");
}

#[test]
fn bom_less_utf16be_non_xml_not_detected() {
    let input = utf16be_bytes("<?y>");

    let (_, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
}

#[test]
fn bom_less_utf16be_uppercase_x_not_detected() {
    let input = utf16be_bytes("<?X>");

    let (_, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
}

#[test]
fn bom_less_utf16be_truncated_not_detected() {
    let input: Vec<u8> = vec![0x00, 0x3C, 0x00, 0x3F];

    let (text, encoding) = decode_html_document(&input, &[]);

    assert_eq!(encoding, "windows-1252");
    assert_eq!(text, "\0<\0?");
}

#[test]
fn header_charset_stays_inside_another_parameters_quoted_string() {
    assert_eq!(
        charset_from_content_type("text/html; boundary=\"; charset=gbk\""),
        None
    );
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; boundary=\"; charset=gbk\"".to_vec(),
    )];
    assert_eq!(
        decode_html_document(b"<p>hi</p>", &headers).1,
        "windows-1252"
    );
}

#[test]
fn header_charset_is_not_displaced_by_an_escaped_quote() {
    assert_eq!(
        charset_from_content_type("text/html; name=\"a\\\"; charset=gbk\"; charset=utf-8")
            .as_deref(),
        Some("utf-8")
    );
}

#[test]
fn header_charset_removes_quoting_backslashes() {
    assert_eq!(
        charset_from_content_type("text/html; charset=\"utf\\-8\"").as_deref(),
        Some("utf-8")
    );
    let headers: Vec<(String, Vec<u8>)> = vec![(
        "Content-Type".to_owned(),
        b"text/html; charset=\"utf\\-8\"".to_vec(),
    )];
    assert_eq!(decode_html_document(b"<p>hi</p>", &headers).1, "UTF-8");
}

#[test]
fn header_charset_matches_chromium_network_tolerances() {
    for header in [
        "text/html; charset=utf-8",
        "text/html;charset=utf-8",
        "TEXT/HTML; CHARSET=UTF-8",
        "text/html; charset=\"utf-8\"",
        "text/html;charset=utf-8;",
        "text/html; charset=\"utf-8",
    ] {
        assert_eq!(
            charset_from_content_type(header)
                .as_deref()
                .and_then(encoding_for_response_charset)
                .map(Encoding::name),
            Some("UTF-8"),
            "header={header}"
        );
    }
    assert_eq!(charset_from_content_type("text/html"), None);
    assert_eq!(charset_from_content_type("text/html; charset="), None);
    assert_eq!(charset_from_content_type("charset=utf-8"), None);
    assert_eq!(
        charset_from_content_type("text/html; charset = utf-8"),
        None
    );
    assert_eq!(
        charset_from_content_type("text/html; charset='utf-8'").as_deref(),
        Some("'utf-8'")
    );
    assert!(
        charset_from_content_type("text/html; charset='utf-8'")
            .as_deref()
            .and_then(encoding_for_response_charset)
            .is_none()
    );
    assert_eq!(
        charset_from_content_type("text/html; charset=gbk; boundary=x").as_deref(),
        Some("gbk")
    );
}

#[test]
fn header_charset_exact_lookup_rejects_non_http_whitespace() {
    for whitespace in ['\u{000b}', '\u{000c}', '\r', '\n'] {
        let expected_label = format!("{whitespace}gbk");
        let headers: Vec<(String, Vec<u8>)> = vec![(
            "Content-Type".to_owned(),
            format!("text/html; charset={whitespace}gbk").into_bytes(),
        )];

        assert_eq!(
            charset_from_headers(&headers).as_deref(),
            Some(expected_label.as_str())
        );
        assert_eq!(encoding_from_response_headers(&headers), None);
        assert_eq!(
            decode_html_document(&gbk_bytes("太平洋"), &headers).1,
            "windows-1252"
        );
    }
}

#[test]
fn encoding_label_lookup_applies_encoding_standard_whitespace_preprocessing() {
    assert_eq!(encoding_for_label("gbk"), Some(encoding_rs::GBK));
    for label in [" gbk", "gbk ", "\tgbk", "gbk\n", "\u{000c}gbk", "gbk\r"] {
        assert_eq!(
            encoding_for_label(label),
            Some(encoding_rs::GBK),
            "label={label:?}"
        );
    }
    assert_eq!(encoding_for_label("\u{000b}gbk"), None);
}

#[test]
fn response_charset_lookup_is_exact_after_http_lws_preprocessing() {
    assert_eq!(encoding_for_response_charset("gbk"), Some(encoding_rs::GBK));
    for label in [
        " gbk",
        "gbk ",
        "\tgbk",
        "gbk\n",
        "\u{000b}gbk",
        "gbk\u{000c}",
    ] {
        assert_eq!(
            encoding_for_response_charset(label),
            None,
            "label={label:?}"
        );
    }
}

#[test]
fn header_charset_preserves_chromium_empty_parameter_precedence() {
    assert_eq!(
        charset_from_content_type("text/html; charset=; charset=gbk").as_deref(),
        Some("gbk")
    );
    assert_eq!(
        charset_from_content_type("text/html; charset=\"\"; charset=gbk"),
        None
    );
}

#[test]
fn header_charset_does_not_reopen_quotes_after_a_value_started() {
    assert_eq!(
        charset_from_content_type("text/html; x=a=\"unterminated; charset=gbk").as_deref(),
        Some("gbk")
    );
    assert_eq!(
        charset_from_content_type("text/html; x=\"ok\"junk=\"unterminated; charset=gbk").as_deref(),
        Some("gbk")
    );
}

#[test]
fn header_charset_recovers_after_a_stray_quote() {
    // WPT MIME case: the `"` does not open a parameter value, so the following
    // `;` still separates parameters and the real charset is found.
    assert_eq!(
        charset_from_content_type("text/html;\";charset=gbk").as_deref(),
        Some("gbk")
    );
}

#[test]
fn header_charset_keeps_an_escaped_quote_as_data() {
    // The quoted value is `utf-8"`, which is not a valid label. Trimming the
    // data quote would manufacture a valid one.
    let label = charset_from_content_type("text/html; charset=\"utf-8\\\"\"")
        .expect("a parameter value is present");

    assert_eq!(label, "utf-8\"");
    assert!(encoding_for_label(&label).is_none());
}

#[test]
fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() {
    use XhrResponseTextKind::{Html, Text, Xml};
    let xml = b"<?xml version='1.0' encoding='windows-1252'?><x>\xff</x>";
    let xml_utf16 = "<?xml version='1.0' encoding='UTF-16'?><x>é</x>";
    let cases = [
        (
            Text,
            None,
            b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
            "機",
            "UTF-8",
        ),
        (
            Text,
            Some(encoding_rs::WINDOWS_1252),
            b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
            "機",
            "UTF-8",
        ),
        (
            Text,
            None,
            b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(),
            "\u{feff}",
            "UTF-8",
        ),
        (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"),
        (
            Text,
            Some(encoding_rs::UTF_8),
            b"\xfe\xff\0A\0\xe9".to_vec(),
            "Aé",
            "UTF-16BE",
        ),
        (
            Text,
            Some(encoding_rs::UTF_16LE),
            b"\xe9\0".to_vec(),
            "é",
            "UTF-16LE",
        ),
        (
            Text,
            Some(encoding_rs::SHIFT_JIS),
            b"\x83e\x83X\x83g".to_vec(),
            "テスト",
            "Shift_JIS",
        ),
        (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"),
        (
            Xml,
            None,
            xml.to_vec(),
            "<?xml version='1.0' encoding='windows-1252'?><x>ÿ</x>",
            "windows-1252",
        ),
        (
            Xml,
            Some(encoding_rs::UTF_8),
            xml.to_vec(),
            "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
            "UTF-8",
        ),
        (
            Text,
            None,
            xml.to_vec(),
            "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
            "UTF-8",
        ),
        (
            Xml,
            None,
            xml_utf16
                .encode_utf16()
                .flat_map(u16::to_le_bytes)
                .collect(),
            xml_utf16,
            "UTF-16LE",
        ),
        (
            Html,
            None,
            b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
            "<meta charset=windows-1252><x>ÿ</x>",
            "windows-1252",
        ),
        (
            Html,
            Some(encoding_rs::UTF_8),
            b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
            "<meta charset=windows-1252><x>\u{fffd}</x>",
            "UTF-8",
        ),
        (
            Html,
            None,
            b"<x>\xe6\xa9\x9f</x>".to_vec(),
            "<x>機</x>",
            "UTF-8",
        ),
    ];
    for (kind, transport, bytes, expected, encoding) in cases {
        for boundary in 0..=bytes.len() {
            let mut decoder = XhrResponseDecoder::new(kind, transport);
            let mut text = decoder.push(&bytes[..boundary]);
            text.push_str(&decoder.push(&bytes[boundary..]));
            text.push_str(&decoder.finish());
            assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}");
            assert_eq!(decoder.encoding_name(), encoding);
        }
    }
}

#[test]
fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() {
    let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
    assert_eq!(decoder.push(b"\xef"), "");
    assert_eq!(decoder.push(b"\xbb"), "");
    assert_eq!(decoder.push(b"\xbf"), "");
    assert_eq!(decoder.push(b"a\xc2"), "a");
    assert_eq!(decoder.finish(), "\u{fffd}");

    let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
    assert_eq!(decoder.push(b"\xff"), "");
    assert_eq!(decoder.push(b"\xfeA"), "");
    assert_eq!(decoder.push(b"\0\xe9"), "A");
    assert_eq!(decoder.push(b"\0"), "é");
    assert_eq!(decoder.finish(), "");
}