Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(parser): expose CDATA from xml5ever

ldm091c281d2c7ae
diff --git a/.gitignore b/.gitignore
index 5d23d60e2..3917fda84 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@
/vendor/*
!/vendor/v8-146.9.0/
!/vendor/v8-146.9.0/**
+!/vendor/xml5ever-0.39.0/
+!/vendor/xml5ever-0.39.0/**
/.moli
__pycache__
*.pyc
diff --git a/Cargo.lock b/Cargo.lock
index 490088f3e..d42ec3fa5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5707,8 +5707,6 @@ dependencies = [
[[package]]
name = "xml5ever"
version = "0.39.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ab627f34ff61b80d756180d556f9c68801d836d271b3b8c094504ceca69d221"
dependencies = [
"log",
"markup5ever 0.39.0",
diff --git a/Cargo.toml b/Cargo.toml
index 98322953b..515823f0b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -75,11 +75,13 @@ members = [
"moli-http-cache",
"moli-wpt-compat"
]
+exclude = ["vendor/xml5ever-0.39.0"]
[workspace.package]
license = "MIT OR Apache-2.0"
[patch.crates-io]
+xml5ever = { path = "vendor/xml5ever-0.39.0" }
curl = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
curl-sys = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" }
diff --git a/moli-parser/src/xml.rs b/moli-parser/src/xml.rs
index 49fc02299..b2dacfeb3 100644
--- a/moli-parser/src/xml.rs
+++ b/moli-parser/src/xml.rs
@@ -12,7 +12,7 @@ use xml5ever::{
driver::{XmlParseOpts, parse_document as parse_xml_document},
interface::{ElementFlags as XmlElementFlags, QuirksMode as XmlQuirksMode},
tendril::{StrTendril as XmlStrTendril, TendrilSink as XmlTendrilSink},
- tree_builder::{NodeOrText as XmlNodeOrText, TreeSink as XmlTreeSink},
+ tree_builder::{NodeOrText as XmlNodeOrText, TreeSink as XmlTreeSinkBase, XmlTreeSink},
};
use xmlparser::{
ElementEnd as XmlElementEnd, Token as XmlTokenizerToken, Tokenizer as XmlTokenizer,
@@ -21,7 +21,6 @@ use xmlparser::{
use super::{html_chunks, xml_tree_viewer::transform_document_to_xml_tree_view};
use moli_dom::native::{Attribute as NativeAttribute, DomHost, NativeDom, NativeNodeId, Node};
-pub(super) const CDATA_MARKER_PI_TARGET: &str = "moli-cdata";
pub(super) const XMLNS_NAMESPACE: &str = "http://www.w3.org/2000/xmlns/";
#[derive(Debug, Clone, Default)]
@@ -66,7 +65,6 @@ impl DerefMut for XmlDomHost<'_> {
struct XmlLiveTreeSinkTarget<'a> {
dom_host: XmlDomHost<'a>,
document_handle: NativeNodeId,
- cdata_sections: Vec<String>,
namespace_declarations: VecDeque<Vec<XmlNamespaceDeclaration>>,
source_has_unclosed_element_at_eof: bool,
}
@@ -96,10 +94,8 @@ impl XmlParser {
) -> NativeDom {
let source_has_unclosed_element_at_eof = xml_source_has_unclosed_element_at_eof(&xml);
let namespace_declarations = xml_namespace_declarations(&xml);
- let (xml, cdata_sections) = xml_with_cdata_markers(&xml);
let sink = XmlDocumentSink::new(XmlLiveTreeSinkTarget::new_owned(
final_url,
- cdata_sections,
namespace_declarations,
source_has_unclosed_element_at_eof,
));
@@ -131,8 +127,7 @@ impl XmlParser {
let target = XmlLiveTreeSinkTarget::new_borrowed(dom_host, document_handle)?
.with_namespace_declarations(namespace_declarations)
.with_source_has_unclosed_element_at_eof(source_has_unclosed_element_at_eof);
- let (xml, cdata_sections) = xml_with_cdata_markers(xml);
- let sink = XmlDocumentSink::new(target.with_cdata_sections(cdata_sections));
+ let sink = XmlDocumentSink::new(target);
let mut parser = parse_xml_document(sink, XmlParseOpts::default());
for chunk in html_chunks(&xml) {
parser.process(XmlStrTendril::from(chunk));
@@ -199,38 +194,6 @@ pub(super) fn xml_source_has_unclosed_element_at_eof(source: &str) -> bool {
pending_element.is_some() || !open_elements.is_empty()
}
-fn xml_with_cdata_markers(xml: &str) -> (String, Vec<String>) {
- let mut output = String::with_capacity(xml.len());
- let mut cdata_sections = Vec::new();
- let mut copied_until = 0;
-
- for token in XmlTokenizer::from(xml) {
- let Ok(XmlTokenizerToken::Cdata { text, span }) = token else {
- continue;
- };
- let range = span.range();
- if range.start < copied_until {
- continue;
- }
- output.push_str(&xml[copied_until..range.start]);
- let index = cdata_sections.len();
- cdata_sections.push(text.as_str().to_owned());
- output.push_str("<?");
- output.push_str(CDATA_MARKER_PI_TARGET);
- output.push(' ');
- output.push_str(&index.to_string());
- output.push_str("?>");
- copied_until = range.end;
- }
-
- if cdata_sections.is_empty() {
- return (xml.to_owned(), cdata_sections);
- }
-
- output.push_str(&xml[copied_until..]);
- (output, cdata_sections)
-}
-
/// xml5ever consumes namespace declarations while resolving qualified names,
/// so its tree sink never receives the declarations as attributes. The DOM,
/// however, exposes `xmlns` declarations as Attr nodes in the XMLNS namespace.
@@ -361,7 +324,6 @@ impl<'a> XmlDocumentSink<'a> {
impl XmlLiveTreeSinkTarget<'static> {
fn new_owned(
final_url: Url,
- cdata_sections: Vec<String>,
namespace_declarations: VecDeque<Vec<XmlNamespaceDeclaration>>,
source_has_unclosed_element_at_eof: bool,
) -> Self {
@@ -370,7 +332,6 @@ impl XmlLiveTreeSinkTarget<'static> {
Self {
dom_host: XmlDomHost::Owned(Box::new(dom_host)),
document_handle,
- cdata_sections,
namespace_declarations,
source_has_unclosed_element_at_eof,
}
@@ -378,7 +339,6 @@ impl XmlLiveTreeSinkTarget<'static> {
fn finish_document(mut self) -> NativeDom {
self.record_unclosed_source_element_error();
- self.restore_cdata_marker_processing_instructions();
let XmlDomHost::Owned(dom_host) = self.dom_host else {
unreachable!("owned XML parsing must finish with an owned DOM host")
};
@@ -399,7 +359,6 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
Some(Self {
dom_host: XmlDomHost::Borrowed(dom_host),
document_handle,
- cdata_sections: Vec::new(),
namespace_declarations: VecDeque::new(),
source_has_unclosed_element_at_eof: false,
})
@@ -413,11 +372,6 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
self
}
- fn with_cdata_sections(mut self, cdata_sections: Vec<String>) -> Self {
- self.cdata_sections = cdata_sections;
- self
- }
-
fn with_source_has_unclosed_element_at_eof(
mut self,
source_has_unclosed_element_at_eof: bool,
@@ -428,7 +382,6 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
fn finish_live_tree(mut self) {
self.record_unclosed_source_element_error();
- self.restore_cdata_marker_processing_instructions();
}
fn record_unclosed_source_element_error(&mut self) {
@@ -494,6 +447,14 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
)
}
+ fn create_cdata(&mut self, text: String) -> XmlParseHandle {
+ XmlParseHandle::new(
+ self.dom_host
+ .create_cdata_section_for_document(self.document_handle, &text),
+ None,
+ )
+ }
+
fn create_processing_instruction(&mut self, target: String, data: String) -> XmlParseHandle {
XmlParseHandle::new(
self.dom_host.create_processing_instruction_for_document(
@@ -680,6 +641,10 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
.map(|handle| XmlParseHandle::new(handle, None))
}
+ fn template_contents_id(&self, node_id: NativeNodeId) -> Option<NativeNodeId> {
+ self.dom_host.parser_template_contents_handle(node_id)
+ }
+
fn add_attrs_if_missing(&mut self, node_id: NativeNodeId, attrs: Vec<XmlAttribute>) {
let Some(node) = self.dom_host.node_mut(node_id) else {
return;
@@ -720,50 +685,9 @@ impl<'a> XmlLiveTreeSinkTarget<'a> {
let _ = self.dom_host.append_child(new_parent_id, child_id);
}
}
-
- fn restore_cdata_marker_processing_instructions(&mut self) {
- self.restore_cdata_marker_processing_instructions_under(self.document_handle);
- }
-
- fn restore_cdata_marker_processing_instructions_under(&mut self, parent_id: NativeNodeId) {
- let child_ids = self.dom_host.child_handles(parent_id).collect::<Vec<_>>();
- for child_id in child_ids {
- if let Some(data) = self.cdata_marker_data(child_id) {
- let cdata = self
- .dom_host
- .create_cdata_section_for_document(self.document_handle, &data);
- let _ = self
- .dom_host
- .insert_before(parent_id, cdata, Some(child_id));
- let _ = self.dom_host.remove_child(parent_id, child_id);
- } else {
- self.restore_cdata_marker_processing_instructions_under(child_id);
- }
- }
- if let Some(template_contents) = self.template_contents_id(parent_id) {
- self.restore_cdata_marker_processing_instructions_under(template_contents);
- }
- }
-
- fn template_contents_id(&self, node_id: NativeNodeId) -> Option<NativeNodeId> {
- self.dom_host.parser_template_contents_handle(node_id)
- }
-
- fn cdata_marker_data(&self, node_id: NativeNodeId) -> Option<String> {
- let processing_instruction = self
- .dom_host
- .node(node_id)?
- .data()
- .as_processing_instruction()?;
- if processing_instruction.target() != CDATA_MARKER_PI_TARGET {
- return None;
- }
- let index = processing_instruction.data().trim().parse::<usize>().ok()?;
- self.cdata_sections.get(index).cloned()
- }
}
-impl<'host> XmlTreeSink for XmlDocumentSink<'host> {
+impl<'host> XmlTreeSinkBase for XmlDocumentSink<'host> {
type Handle = XmlParseHandle;
type Output = XmlLiveTreeSinkTarget<'host>;
type ElemName<'a>
@@ -881,6 +805,12 @@ impl<'host> XmlTreeSink for XmlDocumentSink<'host> {
}
}
+impl XmlTreeSink for XmlDocumentSink<'_> {
+ fn create_cdata(&self, text: XmlStrTendril) -> Self::Handle {
+ self.target.borrow_mut().create_cdata(text.to_string())
+ }
+}
+
#[cfg(test)]
mod tests {
use super::{XmlParser, xml_source_has_unclosed_element_at_eof};
@@ -1209,7 +1139,7 @@ mod tests {
}
#[test]
- fn xml_parser_restores_cdata_inside_html_template_contents() {
+ fn xml_parser_preserves_cdata_inside_html_template_contents() {
let parser = XmlParser;
let dom = parser.parse(
Url::parse("https://example.test/html-template-cdata.xml").unwrap(),
diff --git a/moli-parser/src/xml_stream.rs b/moli-parser/src/xml_stream.rs
index 424d681bf..6769a0d79 100644
--- a/moli-parser/src/xml_stream.rs
+++ b/moli-parser/src/xml_stream.rs
@@ -33,8 +33,7 @@ use crate::{
live_target::{ParserRuntimeDomSinks, ParserStreamHtmlTreeSinkTarget},
stream::prepare_parser_script_handoff_for_static_document,
xml::{
- CDATA_MARKER_PI_TARGET, XmlNamespaceDeclaration, xml_namespace_declarations,
- xml_source_has_unclosed_element_at_eof,
+ XmlNamespaceDeclaration, xml_namespace_declarations, xml_source_has_unclosed_element_at_eof,
},
xml_tree_viewer::transform_parser_target_to_xml_tree_view,
};
@@ -48,7 +47,6 @@ use crate::{
pub struct XmlDocumentStream {
parser: XmlParserSession,
input: XmlParserInputStream,
- preprocessor: XmlInputPreprocessor,
raw_source: String,
scanned_namespace_element_count: usize,
eof_declared: bool,
@@ -76,7 +74,6 @@ struct XmlStreamDocumentSink {
struct XmlParserStreamTarget {
common: ParserStreamHtmlTreeSinkTarget,
namespace_declarations: VecDeque<Vec<XmlNamespaceDeclaration>>,
- cdata_sections: Vec<String>,
source_has_unclosed_element_at_eof: bool,
present_unstyled_top_level_document: bool,
}
@@ -88,13 +85,6 @@ struct XmlStreamParseHandle {
suppress_append: bool,
}
-#[derive(Default)]
-struct XmlInputPreprocessor {
- pending: String,
- next_cdata_index: usize,
- completed_cdata_sections: Vec<String>,
-}
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RawXmlParserStep {
Script(NativeNodeId),
@@ -128,14 +118,12 @@ impl XmlDocumentStream {
let target = XmlParserStreamTarget {
common,
namespace_declarations: VecDeque::new(),
- cdata_sections: Vec::new(),
source_has_unclosed_element_at_eof: false,
present_unstyled_top_level_document,
};
Self {
parser: XmlParserSession::new(target, XmlParseOpts::default()),
input: XmlParserInputStream::default(),
- preprocessor: XmlInputPreprocessor::default(),
raw_source: String::new(),
scanned_namespace_element_count: 0,
eof_declared: false,
@@ -151,26 +139,16 @@ impl XmlDocumentStream {
}
self.raw_source.push_str(&chunk);
self.refresh_namespace_declarations();
- let transformed = self.preprocessor.push(&chunk, false);
- self.transfer_completed_cdata_sections();
- if !transformed.is_empty() {
- self.input.end_segments.push_back(transformed);
- }
+ self.input.end_segments.push_back(chunk);
}
- /// Declares source EOF and flushes any prefix retained by the CDATA
- /// boundary preprocessor. Calling this more than once is a no-op.
+ /// Declares source EOF. Calling this more than once is a no-op.
pub fn declare_eof(&mut self) {
if self.eof_declared {
return;
}
self.eof_declared = true;
self.refresh_namespace_declarations();
- let trailing = self.preprocessor.push("", true);
- self.transfer_completed_cdata_sections();
- if !trailing.is_empty() {
- self.input.end_segments.push_back(trailing);
- }
self.parser
.sink()
.borrow_mut()
@@ -199,7 +177,6 @@ impl XmlDocumentStream {
for segment in &self.input.end_segments {
pending.push_str(segment);
}
- pending.push_str(&self.preprocessor.pending);
pending
}
@@ -367,17 +344,6 @@ impl XmlDocumentStream {
.extend(newly_complete);
}
- fn transfer_completed_cdata_sections(&mut self) {
- let sections = std::mem::take(&mut self.preprocessor.completed_cdata_sections);
- if !sections.is_empty() {
- self.parser
- .sink()
- .borrow_mut()
- .cdata_sections
- .extend(sections);
- }
- }
-
fn take_next_owned_input(&mut self, max_bytes: usize) -> String {
if self.parser.has_buffered_input() {
return String::new();
@@ -737,12 +703,6 @@ impl TreeSink for XmlStreamDocumentSink {
let target_text = target.to_string();
let data_text = data.to_string();
let mut parser_target = self.target.borrow_mut();
- if target_text == CDATA_MARKER_PI_TARGET
- && let Ok(index) = data_text.trim().parse::<usize>()
- && let Some(cdata) = parser_target.cdata_sections.get(index).cloned()
- {
- return XmlStreamParseHandle::new(parser_target.common.create_cdata_section(cdata));
- }
let handle = parser_target
.common
.create_processing_instruction(target_text.clone(), data_text);
@@ -886,6 +846,17 @@ impl TreeSink for XmlStreamDocumentSink {
}
}
+impl xml5ever::tree_builder::XmlTreeSink for XmlStreamDocumentSink {
+ fn create_cdata(&self, text: StrTendril) -> Self::Handle {
+ XmlStreamParseHandle::new(
+ self.target
+ .borrow_mut()
+ .common
+ .create_cdata_section(text.to_string()),
+ )
+ }
+}
+
fn xml_name_to_html(name: &XmlQualName) -> HtmlQualName {
HtmlQualName::new(
name.prefix
@@ -903,59 +874,6 @@ fn xml_attribute_to_html(attribute: XmlAttribute) -> HtmlAttribute {
}
}
-impl XmlInputPreprocessor {
- fn push(&mut self, input: &str, eof: bool) -> String {
- const CDATA_OPEN: &str = "<![CDATA[";
- const CDATA_CLOSE: &str = "]]>";
-
- self.pending.push_str(input);
- let mut output = String::new();
- loop {
- let Some(start) = self.pending.find(CDATA_OPEN) else {
- if eof {
- output.push_str(&self.pending);
- self.pending.clear();
- } else {
- let keep = longest_suffix_prefix_len(&self.pending, CDATA_OPEN);
- let emit_len = self.pending.len().saturating_sub(keep);
- output.push_str(&self.pending[..emit_len]);
- self.pending.drain(..emit_len);
- }
- break;
- };
- output.push_str(&self.pending[..start]);
- self.pending.drain(..start);
- let content_start = CDATA_OPEN.len();
- let Some(relative_end) = self.pending[content_start..].find(CDATA_CLOSE) else {
- if eof {
- output.push_str(&self.pending);
- self.pending.clear();
- }
- break;
- };
- let content_end = content_start + relative_end;
- let cdata = self.pending[content_start..content_end].to_owned();
- output.push_str("<?");
- output.push_str(CDATA_MARKER_PI_TARGET);
- output.push(' ');
- output.push_str(&self.next_cdata_index.to_string());
- output.push_str("?>");
- self.completed_cdata_sections.push(cdata);
- self.next_cdata_index = self.next_cdata_index.saturating_add(1);
- self.pending.drain(..(content_end + CDATA_CLOSE.len()));
- }
- output
- }
-}
-
-fn longest_suffix_prefix_len(value: &str, prefix: &str) -> usize {
- let max = value.len().min(prefix.len().saturating_sub(1));
- (1..=max)
- .rev()
- .find(|length| value.ends_with(&prefix[..*length]))
- .unwrap_or_default()
-}
-
fn split_xml_parser_input_prefix(input: String, max_bytes: usize) -> (String, Option<String>) {
if max_bytes == 0 || input.len() <= max_bytes {
return (input, None);
@@ -1123,6 +1041,53 @@ mod tests {
);
}
+ #[test]
+ fn incremental_xml_keeps_cdata_syntax_in_comment_and_pi() {
+ let mut stream = XmlDocumentStream::new(test_url());
+ for chunk in ["<root><!-- <![CDA", "TA[not closed -->"] {
+ stream.append_to_end(chunk.to_owned());
+ while stream.has_pending_input() {
+ let outcome = stream.pump_next_parser_step(2);
+ assert!(matches!(outcome.result, ParserPumpStep::InputDrained));
+ }
+ }
+
+ let snapshot = stream.snapshot_parser_stream_document();
+ assert_eq!(
+ snapshot
+ .nodes()
+ .find_map(Node::as_comment)
+ .map(|comment| comment.data()),
+ Some(" <![CDATA[not closed "),
+ "a fake CDATA opener inside a closed comment must not retain later input"
+ );
+
+ for chunk in [
+ "<?moli-cdata 0?><?probe <![CDATA[pi]]>?>",
+ "<child><![CDA",
+ "TA[real]]></child></root>",
+ ] {
+ stream.append_to_end(chunk.to_owned());
+ }
+ let document = stream.finish();
+ let instructions = document
+ .nodes()
+ .filter_map(Node::as_processing_instruction)
+ .map(|instruction| (instruction.target(), instruction.data()))
+ .collect::<Vec<_>>();
+ assert_eq!(
+ instructions,
+ [("moli-cdata", "0"), ("probe", "<![CDATA[pi]]>")]
+ );
+ assert_eq!(
+ document
+ .nodes()
+ .find_map(Node::as_cdata_section)
+ .map(|section| section.data()),
+ Some("real")
+ );
+ }
+
#[test]
fn incremental_xml_does_not_suspend_for_non_executable_script_namespace() {
let mut stream = XmlDocumentStream::new(test_url());
diff --git a/vendor/xml5ever-0.39.0/Cargo.toml b/vendor/xml5ever-0.39.0/Cargo.toml
new file mode 100644
index 000000000..ebcf2564b
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/Cargo.toml
@@ -0,0 +1,70 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2021"
+rust-version = "1.71.0"
+name = "xml5ever"
+version = "0.39.0"
+authors = ["The xml5ever project developers"]
+build = false
+exclude = ["xml5lib-tests/*"]
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "Push based streaming parser for XML."
+homepage = "https://github.com/servo/html5ever/blob/main/xml5ever/README.md"
+documentation = "https://docs.rs/xml5ever"
+readme = "README.md"
+keywords = [
+ "xml",
+ "xml5",
+ "parser",
+ "parsing",
+]
+categories = [
+ "parser-implementations",
+ "web-programming",
+]
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/servo/html5ever"
+
+[features]
+serde = ["markup5ever/serde"]
+trace_tokenizer = []
+
+[lib]
+name = "xml5ever"
+path = "src/lib.rs"
+
+[[example]]
+name = "simple_xml_tokenizer"
+path = "examples/simple_xml_tokenizer.rs"
+
+[[example]]
+name = "xml_tokenizer"
+path = "examples/xml_tokenizer.rs"
+
+[[bench]]
+name = "xml5ever"
+path = "benches/xml5ever.rs"
+harness = false
+
+[dependencies.log]
+version = "0.4"
+
+[dependencies.markup5ever]
+version = "0.39"
+
+[dev-dependencies.criterion]
+version = "0.8"
diff --git a/vendor/xml5ever-0.39.0/LICENSE-APACHE b/vendor/xml5ever-0.39.0/LICENSE-APACHE
new file mode 100644
index 000000000..16fe87b06
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/LICENSE-APACHE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/vendor/xml5ever-0.39.0/LICENSE-MIT b/vendor/xml5ever-0.39.0/LICENSE-MIT
new file mode 100644
index 000000000..6e4510217
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/LICENSE-MIT
@@ -0,0 +1,25 @@
+Copyright (c) 2014 The html5ever Project Developers
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/vendor/xml5ever-0.39.0/MOLI_VENDORING.md b/vendor/xml5ever-0.39.0/MOLI_VENDORING.md
new file mode 100644
index 000000000..f24addd40
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/MOLI_VENDORING.md
@@ -0,0 +1,8 @@
+# xml5ever 0.39.0
+
+This directory contains the source of the crates.io `xml5ever` 0.39.0 release.
+It is patched locally because Moli needs the tokenizer to preserve CDATA as a
+distinct token and the XML tree builder to surface CDATA nodes to its sink.
+
+The upstream source is licensed under MIT or Apache-2.0; see `LICENSE-MIT` and
+`LICENSE-APACHE` in this directory.
diff --git a/vendor/xml5ever-0.39.0/README.md b/vendor/xml5ever-0.39.0/README.md
new file mode 100644
index 000000000..e5cb1f757
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/README.md
@@ -0,0 +1,72 @@
+# xml5ever
+
+![http://www.apache.org/licenses/LICENSE-2.0](https://img.shields.io/badge/license-Apache-blue.svg) ![https://opensource.org/licenses/MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+[![Docs.rs](https://docs.rs/xml5ever/badge.svg)](https://docs.rs/xml5ever)
+[![](https://img.shields.io/crates/v/xml5ever.svg)](https://crates.io/crates/xml5ever)
+
+[API documentation](https://docs.rs/xml5ever)
+
+**Warning:** This library is alpha quality, so no guarantees are given.
+
+This crate provides a push based XML parser library that trades well-formedness for error recovery.
+
+xml5ever is based largely on [html5ever](https://github.com/servo/html5ever) parser, so if you have experience with html5ever you will be familiar with xml5ever.
+
+The library is dual licensed under MIT and Apache license.
+
+# Why you should use xml5ever
+
+Main use case for this library is when XML is badly formatted, usually from bad XML
+templates. XML5 tries to handle most common errors, in a manner similar to HTML5.
+
+## When you should use it?
+
+ - You aren't interested in well-formed documents.
+ - You need to get some info from your data even if it has errors (although not all possible errors are handled).
+ - You want to features like character references or XML namespaces.
+
+## When you shouldn't use it
+
+ - You need to have your document validated.
+ - You require DTD support.
+ - You require an easy to use parser, with lots of extensions (e.g. XPath, XQuery).
+ - You require a battle tested, industry proven solution.
+
+# Installation
+
+Add xml5ever as a dependency:
+
+```bash
+cargo add xml5ever
+```
+
+# Getting started
+
+Here is a very simple RcDom backed parser:
+
+```rust
+
+ let input = "<xml></xml>".to_tendril();
+
+ // To parse XML into a tree form, we need a TreeSink
+ // luckily xml5ever comes with a static RC backed tree represetation.
+ let dom: RcDom = parse(std::iter::once(input), Default::default());
+
+ // Do something with dom
+
+```
+The thing that does actual parsing is the `parse` function. It expects an iterator that can be converted into `StrTendril`, so you can use `std::iter::once(input)` or `Some(input).into_iter()` (where `input` is `StrTendril` like structure).
+
+# Working on xml5ever
+
+To build examples and tests you need to do something along the lines of:
+
+```rust
+ git submodule update --init # to fetch xml5lib-tests
+ cargo build
+ cargo test
+```
+
+This will fetch tests from outside repository and it will invoke cargo to
+build and test the crate. If you need docs checkout either [API docs](https://docs.rs/xml5ever) or run `cargo docs`
+to generate documentation.
diff --git a/vendor/xml5ever-0.39.0/benches/xml5ever.rs b/vendor/xml5ever-0.39.0/benches/xml5ever.rs
new file mode 100644
index 000000000..8380ab323
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/benches/xml5ever.rs
@@ -0,0 +1,77 @@
+#[macro_use]
+extern crate criterion;
+extern crate markup5ever;
+extern crate xml5ever;
+
+use std::fs;
+use std::path::PathBuf;
+
+use criterion::Criterion;
+
+use markup5ever::buffer_queue::BufferQueue;
+use xml5ever::tendril::*;
+use xml5ever::tokenizer::{ProcessResult, Token, TokenSink, XmlTokenizer};
+
+struct Sink;
+
+impl TokenSink for Sink {
+ type Handle = ();
+
+ fn process_token(&self, token: Token) -> ProcessResult<()> {
+ // Don't use the token, but make sure we don't get
+ // optimized out entirely.
+ std::hint::black_box(token);
+ ProcessResult::Continue
+ }
+}
+
+fn run_bench(c: &mut Criterion, name: &str) {
+ let mut path = PathBuf::from("./");
+ path.push("data/bench/");
+ path.push(name);
+ let mut file = fs::File::open(&path).expect("can't open file");
+
+ // Read the file and treat it as an infinitely repeating sequence of characters.
+ let mut file_input = ByteTendril::new();
+ file.read_to_tendril(&mut file_input)
+ .expect("can't read file");
+ let file_input: StrTendril = file_input.try_reinterpret().unwrap();
+ let size = file_input.len();
+ let mut stream = file_input.chars().cycle();
+
+ // Break the input into chunks of 1024 chars (= a few kB).
+ // This simulates reading from the network.
+ let mut input = vec![];
+ let mut total = 0usize;
+ while total < size {
+ // The by_ref() call is important, otherwise we get wrong results!
+ // See rust-lang/rust#18045.
+ let sz = std::cmp::min(1024, size - total);
+ input.push(stream.by_ref().take(sz).collect::<String>().to_tendril());
+ total += sz;
+ }
+
+ let test_name = format!("xml tokenizing {name}");
+
+ c.bench_function(&test_name, move |b| {
+ b.iter(|| {
+ let tok = XmlTokenizer::new(Sink, Default::default());
+ let buffer = BufferQueue::default();
+ // We are doing clone inside the bench function, this is not ideal, but possibly
+ // necessary since our iterator consumes the underlying buffer.
+ for buf in input.clone().into_iter() {
+ buffer.push_back(buf);
+ let _ = tok.feed(&buffer);
+ }
+ let _ = tok.feed(&buffer);
+ tok.end();
+ })
+ });
+}
+
+fn xml5ever_benchmarks(c: &mut Criterion) {
+ run_bench(c, "strong.xml");
+}
+
+criterion_group!(benches, xml5ever_benchmarks);
+criterion_main!(benches);
diff --git a/vendor/xml5ever-0.39.0/examples/README.md b/vendor/xml5ever-0.39.0/examples/README.md
new file mode 100644
index 000000000..b1763fc7e
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/examples/README.md
@@ -0,0 +1,223 @@
+# Examples
+
+The examples have been designed with [`cargo-script`](https://github.com/DanielKeep/cargo-script) in mind.
+
+Here I'll just give broad overview how to install [`cargo script`] for Rust 1.5. For more details, check out [cargo-script repository](https://github.com/DanielKeep/cargo-script).
+
+ cargo install cargo-script
+
+
+# Token printer
+
+The basis of xml5ever is its tokenizer and tree builder. Roughly speaking tokenizer
+takes input and returns a set of tokens like comment, processing instruction, start
+tag, end tag, etc.
+
+First let's define our dependencies:
+
+```toml
+ [dependencies]
+ xml5ever = "0.2.0"
+ tendril = "0.1.3"
+```
+
+With dependencies declared, we can now make a simple tokenizer sink. First step is to
+define a [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html). [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html) are traits that received stream of [`Tokens`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/enum.Token.html).
+
+In our case we'll define a unit struct (i.e. a struct without any fields).
+
+```rust
+ struct SimpleTokenPrinter;
+```
+
+To make `SimpleTokenPrinter` a [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html), we need to implement [process_token](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html#tymethod.process_token) method.
+
+```rust
+ impl TokenSink for SimpleTokenPrinter {
+ fn process_token(&mut self, token: Token) {
+ match token {
+ CharacterTokens(b) => {
+ println!("TEXT: {}", &*b);
+ },
+ NullCharacterToken => print!("NULL"),
+ TagToken(tag) => {
+ println!("{:?} {} ", tag.kind, &*tag.name.local);
+ },
+ ParseError(err) => {
+ println!("ERROR: {}", err);
+ },
+ PIToken(Pi{ref target, ref data}) => {
+ println!("PI : <?{} {}?>", &*target, &*data);
+ },
+ CommentToken(ref comment) => {
+ println!("<!--{:?}-->", &*comment);
+ },
+ EOFToken => {
+ println!("EOF");
+ },
+ DoctypeToken(Doctype{ref name, ref public_id, ..}) => {
+ println!("<!DOCTYPE {:?} {:?}>", &*name, &*public_id);
+ }
+ }
+ }
+ }
+```
+
+Now, we need some input to process. For input we'll use `stdin`. However, xml5ever `tokenize_to` method only takes `StrTendril`. So we need to construct a
+[`ByteTendril`](https://docs.rs/tendril/latest/tendril/type.ByteTendril.html) using `ByteTendril::new()`, then read the `stdin` using [`read_to_tendril`](https://docs.rs/tendril/latest/tendril/trait.ReadExt.html#tymethod.read_to_tendril) extension.
+
+Once that is set, to make `SimpleTokenPrinter` parse the input, call,
+`tokenize_to` with it as the first parameter, input wrapped in Option for second parameter and XmlToke.
+
+```rust
+ fn main() {
+ let sink = SimpleTokenPrinter;
+
+ // We need a ByteTendril to read a file
+ let mut input = ByteTendril::new();
+ // Using SliceExt.read_to_tendril we read stdin
+ io::stdin().read_to_tendril(&mut input).unwrap();
+ // For xml5ever we need StrTendril, so we reinterpret it
+ // into StrTendril.
+ //
+ // You might wonder, how does `try_reinterpret` know we
+ // need StrTendril and the answer is type inference based
+ // on `tokenize_xml_to` signature.
+ let input = input.try_reinterpret().unwrap();
+ // Here we create and run tokenizer
+ let mut tok = XmlTokenizer::new(sink, Default::default());
+ // We pass input to parsed.
+ tok.feed(input);
+
+ // tok.end must be invoked for final bytes to be processed.
+ tok.end();
+ }
+```
+
+NOTE: `unwrap` causes panic, it's only OK to use in simple examples.
+
+For full source code check out: [`examples/simple_xml_tokenizer.rs`](https://github.com/servo/html5ever/blob/main/xml5ever/examples/simple_xml_tokenizer.rs)
+
+Once we have successfully compiled the example we run the example with inline
+xml
+
+```bash
+ cargo script simple_xml_tokenizer.rs <<< "<xml>Text with <b>bold words</b>!</xml>"
+```
+
+or by sending an [`examples/example.xml`](https://github.com/servo/html5ever/blob/main/xml5ever/examples/example.xml) located in same folder as examples.
+
+```bash
+ cargo script simple_xml_tokenizer.rs < example.xml
+```
+
+# Tree printer
+
+To actually get an XML document tree from the xml5ever, you need to use a `TreeSink`.
+`TreeSink` is in many way similar to the TokenSink. Basically, TokenSink takes data
+and returns list of tokens, while TreeSink takes tokens and returns a tree of parsed
+XML document. Do note, that this is a simplified explanation and consult
+documentation for more info.
+
+Ok, with that in mind, let's build us a TreePrinter. For example if we get an XML
+file like:
+
+```xml
+ <student>
+ <first-name>Bobby</first-name>
+ <last-name>Tables</last-name>
+ </student>
+```
+
+We'd want a structure similar to this:
+
+```
+#document
+ student
+ first-name
+ #text Bobby
+ last-name
+ #text Tables
+
+```
+We won't print anything other than element names and text fields. So comments,
+doctypes and other such elements are ignored.
+
+First part is similar to making SimpleTokenPrinter:
+
+```rust
+ // We need to allocate an input tendril for xml5ever
+ let mut input = ByteTendril::new();
+ // Using SliceExt.read_to_tendril functions we can read stdin
+ io::stdin().read_to_tendril(&mut input).unwrap();
+ let input = input.try_reinterpret().unwrap();
+```
+
+This time, we need an implementation of [`TreeSink`](https://docs.rs/xml5ever/latest/xml5ever/tree_builder/trait.TreeSink.html). xml5ever comes with a
+built-in `TreeSink` implementation called [`RcDom`](https://docs.rs/markup5ever_rcdom/latest/markup5ever_rcdom/struct.RcDom.html). To process input into
+a `TreeSink` we use the following line:
+
+```rust
+ let dom: RcDom = parse(one_input(input), Default::default());
+```
+
+Let's analyze it a bit. First there is `let dom: RcDom`. We need this part,
+because the type inferencer can't infer which TreeSink implementation we mean
+in this scenario.
+
+Function [`one_input`](https://ygg01.github.io/docs/xml5ever/xml5ever/fn.one_input.html) is a convenience function that turns any value into an iterator. In this case
+it converts a StrTendril into an Iterator over itself.
+
+Ok, so now that we parsed our tree what with it? Well, for that we might need some
+kind of function that will help us traverse it. We shall call that function `walk`.
+
+```rust
+ fn walk(prefix: &str, handle: Handle) {
+ let node = handle.borrow();
+
+ // We print out the prefix before we start
+ print!("{}", prefix);
+ // We are only interested in following nodes:
+ // Document, Text and Element, so our match
+ // reflects that.
+ match node.node {
+ Document
+ => println!("#document"),
+
+ Text(ref text) => {
+ println!("#text {}", text.escape_default())
+ },
+
+ Element(ref name, _) => {
+ println!("{}", name.local);
+ },
+
+ _ => {},
+
+ }
+
+ // We increase indent in child nodes
+ let new_indent = {
+ let mut temp = String::new();
+ temp.push_str(prefix);
+ temp.push_str(" ");
+ temp
+ };
+
+ for child in node.children.iter()
+ // In order to avoid weird indentation, we filter
+ // only Text/Element nodes.
+ // We don't need to filter Document since its guaranteed
+ // child elements don't contain documents
+ .filter(|child| match child.borrow().node {
+ Text(_) | Element (_, _) => true,
+ _ => false,
+ }
+ ) {
+ // Recursion - Yay!
+ walk(&new_indent, child.clone());
+ }
+ }
+```
+
+For full source code check out: [`examples/xml_tree_printer.rs`](https://github.com/servo/html5ever/blob/main/rcdom/examples/xml_tree_printer.rs)
diff --git a/vendor/xml5ever-0.39.0/examples/example.xml b/vendor/xml5ever-0.39.0/examples/example.xml
new file mode 100644
index 000000000..834be5b92
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/examples/example.xml
@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE xml PUBLIC "http://www.wc3.org/">
+<student><first-name>Bobby</first-name><last-name>Tables</last-name></student>
diff --git a/vendor/xml5ever-0.39.0/examples/simple_xml_tokenizer.rs b/vendor/xml5ever-0.39.0/examples/simple_xml_tokenizer.rs
new file mode 100644
index 000000000..20893a8c5
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/examples/simple_xml_tokenizer.rs
@@ -0,0 +1,84 @@
+#!/usr/bin/env run-cargo-script
+//! This is a regular crate doc comment, but it also contains a partial
+//! Cargo manifest. Note the use of a *fenced* code block, and the
+//! `cargo` "language".
+//!
+//! ```cargo
+//! [dependencies]
+//! xml5ever = "0.1.1"
+//! tendril = "0.1.3"
+//! markup5ever = "0.7.4"
+//! ```
+extern crate markup5ever;
+extern crate xml5ever;
+
+use std::io;
+
+use markup5ever::buffer_queue::BufferQueue;
+use xml5ever::tendril::{ByteTendril, ReadExt};
+use xml5ever::tokenizer::{Doctype, Pi, ProcessResult, Token, TokenSink, XmlTokenizer};
+
+struct SimpleTokenPrinter;
+
+impl TokenSink for SimpleTokenPrinter {
+ type Handle = ();
+
+ fn process_token(&self, token: Token) -> ProcessResult<()> {
+ match token {
+ Token::Characters(b) => {
+ println!("TEXT: {}", &*b);
+ },
+ Token::Cdata(b) => {
+ println!("CDATA: {}", &*b);
+ },
+ Token::NullCharacter => print!("NULL"),
+ Token::Tag(tag) => {
+ println!("{:?} {} ", tag.kind, &*tag.name.local);
+ },
+ Token::ParseError(err) => {
+ println!("ERROR: {err}");
+ },
+ Token::ProcessingInstruction(Pi {
+ ref target,
+ ref data,
+ }) => {
+ println!("PI : <?{target} {data}?>");
+ },
+ Token::Comment(ref comment) => {
+ println!("<!--{comment:?}-->");
+ },
+ Token::EndOfFile => {
+ println!("EOF");
+ },
+ Token::Doctype(Doctype {
+ ref name,
+ ref public_id,
+ ..
+ }) => {
+ println!("<!DOCTYPE {name:?} {public_id:?}>");
+ },
+ };
+ ProcessResult::Continue
+ }
+}
+
+fn main() {
+ // Our implementation of TokenSink
+ let sink = SimpleTokenPrinter;
+
+ // We need a ByteTendril to read a file
+ let mut input = ByteTendril::new();
+
+ // Using SliceExt.read_to_tendril we can read stdin
+ io::stdin().read_to_tendril(&mut input).unwrap();
+ // For xml5ever we need StrTendril, so we reinterpret it
+ // into StrTendril.
+
+ // Load input into BufferQueue
+ let input_buffer = BufferQueue::default();
+ input_buffer.push_back(input.try_reinterpret().unwrap());
+ // Here we create and run tokenizer
+ let tok = XmlTokenizer::new(sink, Default::default());
+ let _ = tok.feed(&input_buffer);
+ tok.end();
+}
diff --git a/vendor/xml5ever-0.39.0/examples/xml_tokenizer.rs b/vendor/xml5ever-0.39.0/examples/xml_tokenizer.rs
new file mode 100644
index 000000000..e3f1d8a89
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/examples/xml_tokenizer.rs
@@ -0,0 +1,119 @@
+#!/usr/bin/env run-cargo-script
+//! This is a regular crate doc comment, but it also contains a partial
+//! Cargo manifest. Note the use of a *fenced* code block, and the
+//! `cargo` "language".
+//!
+//! ```cargo
+//! [dependencies]
+//! xml5ever = "0.2.0"
+//! tendril = "0.1.3"
+//! markup5ever = "0.7.4"
+//! ```
+extern crate markup5ever;
+extern crate xml5ever;
+
+use std::cell::Cell;
+use std::io;
+
+use markup5ever::buffer_queue::BufferQueue;
+use xml5ever::tendril::{ByteTendril, ReadExt};
+use xml5ever::tokenizer::{
+ EmptyTag, EndTag, Pi, ProcessResult, ShortTag, StartTag, Token, TokenSink, XmlTokenizer,
+ XmlTokenizerOpts,
+};
+
+#[derive(Clone)]
+struct TokenPrinter {
+ in_char_run: Cell<bool>,
+}
+
+impl TokenPrinter {
+ fn is_char(&self, is_char: bool) {
+ match (self.in_char_run.get(), is_char) {
+ (false, true) => print!("CHAR : \""),
+ (true, false) => println!("\""),
+ _ => (),
+ }
+ self.in_char_run.set(is_char);
+ }
+
+ fn do_char(&self, c: char) {
+ self.is_char(true);
+ print!("{}", c.escape_default().collect::<String>());
+ }
+}
+
+impl TokenSink for TokenPrinter {
+ type Handle = ();
+
+ fn process_token(&self, token: Token) -> ProcessResult<()> {
+ match token {
+ Token::Characters(b) => {
+ for c in b.chars() {
+ self.do_char(c);
+ }
+ },
+ Token::Cdata(b) => {
+ self.is_char(false);
+ println!("CDATA: {}", &*b);
+ },
+ Token::NullCharacter => self.do_char('\0'),
+ Token::Tag(tag) => {
+ self.is_char(false);
+ // This is not proper HTML serialization, of course.
+ match tag.kind {
+ StartTag => print!("TAG : <\x1b[32m{}\x1b[0m", tag.name.local),
+ EndTag => print!("END TAG : <\x1b[31m/{}\x1b[0m", tag.name.local),
+ ShortTag => print!("Short TAG : <\x1b[31m/{}\x1b[0m", tag.name.local),
+ EmptyTag => print!("Empty TAG : <\x1b[31m{}\x1b[0m", tag.name.local),
+ }
+ for attr in tag.attrs.iter() {
+ print!(
+ " \x1b[36m{}\x1b[0m='\x1b[34m{}\x1b[0m'",
+ attr.name.local, attr.value
+ );
+ }
+ if tag.kind == EmptyTag {
+ print!("/");
+ }
+ println!(">");
+ },
+ Token::ParseError(err) => {
+ self.is_char(false);
+ println!("ERROR: {err}");
+ },
+ Token::ProcessingInstruction(Pi { target, data }) => {
+ self.is_char(false);
+ println!("PI : <?{target:?} {data:?}?>");
+ },
+ _ => {
+ self.is_char(false);
+ println!("OTHER: {token:?}");
+ },
+ };
+
+ ProcessResult::Continue
+ }
+}
+
+fn main() {
+ let sink = TokenPrinter {
+ in_char_run: Cell::new(false),
+ };
+ let mut input = ByteTendril::new();
+ io::stdin().read_to_tendril(&mut input).unwrap();
+ let input_buffer = BufferQueue::default();
+ input_buffer.push_back(input.try_reinterpret().unwrap());
+
+ let tok = XmlTokenizer::new(
+ sink,
+ XmlTokenizerOpts {
+ profile: true,
+ exact_errors: true,
+ ..Default::default()
+ },
+ );
+ let _ = tok.feed(&input_buffer);
+ tok.end();
+ tok.sink.is_char(false);
+}
diff --git a/vendor/xml5ever-0.39.0/src/driver.rs b/vendor/xml5ever-0.39.0/src/driver.rs
new file mode 100644
index 000000000..083ce43c6
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/driver.rs
@@ -0,0 +1,91 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use crate::tokenizer::{XmlTokenizer, XmlTokenizerOpts};
+use crate::tree_builder::{XmlTreeBuilder, XmlTreeBuilderOpts, XmlTreeSink};
+
+use std::borrow::Cow;
+
+use crate::tendril;
+use crate::tendril::stream::{TendrilSink, Utf8LossyDecoder};
+use crate::tendril::StrTendril;
+use markup5ever::buffer_queue::BufferQueue;
+use markup5ever::TokenizerResult;
+
+/// All-encompasing parser setting structure.
+#[derive(Clone, Default)]
+pub struct XmlParseOpts {
+ /// Xml tokenizer options.
+ pub tokenizer: XmlTokenizerOpts,
+ /// Xml tree builder .
+ pub tree_builder: XmlTreeBuilderOpts,
+}
+
+/// Parse and send results to a `TreeSink`.
+///
+/// ## Example
+///
+/// ```ignore
+/// let mut sink = MySink;
+/// parse_document(&mut sink, iter::once(my_str), Default::default());
+/// ```
+pub fn parse_document<Sink>(sink: Sink, opts: XmlParseOpts) -> XmlParser<Sink>
+where
+ Sink: XmlTreeSink,
+{
+ let tb = XmlTreeBuilder::new(sink, opts.tree_builder);
+ let tok = XmlTokenizer::new(tb, opts.tokenizer);
+ XmlParser {
+ tokenizer: tok,
+ input_buffer: BufferQueue::default(),
+ }
+}
+
+/// An XML parser,
+/// ready to receive Unicode input through the `tendril::TendrilSink` trait’s methods.
+pub struct XmlParser<Sink>
+where
+ Sink: XmlTreeSink,
+{
+ /// Tokenizer used by XmlParser.
+ pub tokenizer: XmlTokenizer<XmlTreeBuilder<Sink::Handle, Sink>>,
+ /// Input used by XmlParser.
+ pub input_buffer: BufferQueue,
+}
+
+impl<Sink: XmlTreeSink> TendrilSink<tendril::fmt::UTF8> for XmlParser<Sink> {
+ type Output = Sink::Output;
+
+ fn process(&mut self, t: StrTendril) {
+ self.input_buffer.push_back(t);
+ // FIXME: Properly support </script> somehow.
+ while let TokenizerResult::Script(_) = self.tokenizer.feed(&self.input_buffer) {}
+ }
+
+ // FIXME: Is it too noisy to report every character decoding error?
+ fn error(&mut self, desc: Cow<'static, str>) {
+ self.tokenizer.sink.sink.parse_error(desc)
+ }
+
+ fn finish(self) -> Self::Output {
+ self.tokenizer.end();
+ self.tokenizer.sink.sink.finish()
+ }
+}
+
+impl<Sink: XmlTreeSink> XmlParser<Sink> {
+ /// Wrap this parser into a `TendrilSink` that accepts UTF-8 bytes.
+ ///
+ /// Use this when your input is bytes that are known to be in the UTF-8 encoding.
+ /// Decoding is lossy, like `String::from_utf8_lossy`.
+ #[allow(clippy::wrong_self_convention)]
+ pub fn from_utf8(self) -> Utf8LossyDecoder<Self> {
+ Utf8LossyDecoder::new(self)
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/lib.rs b/vendor/xml5ever-0.39.0/src/lib.rs
new file mode 100644
index 000000000..c9e557ff1
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/lib.rs
@@ -0,0 +1,47 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! This crate provides a push based XML parser library that
+//! adheres to XML5 specification. In other words this library
+//! trades well-formedness for error recovery.
+//!
+//! The idea behind this, was to minimize number of errors from
+//! tools that generate XML (e.g. `&#83` won't just return `&#83`
+//! as text, but will parse it into `S` ).
+//! You can check out full specification [here](https://ygg01.github.io/xml5_draft/).
+//!
+//! What this library provides is a solid XML parser that can:
+//!
+//! * Parse somewhat erroneous XML input
+//! * Provide support for [Numeric character references](https://en.wikipedia.org/wiki/Numeric_character_reference).
+//! * Provide partial [XML namespace](http://www.w3.org/TR/xml-names11/) support.
+//! * Provide full set of SVG/MathML entities
+//!
+//! What isn't in scope for this library:
+//!
+//! * Document Type Definition parsing - this is pretty hard to do right and nowadays, its used
+//!
+
+#![crate_name = "xml5ever"]
+#![crate_type = "dylib"]
+#![allow(unexpected_cfgs)]
+#![deny(missing_docs)]
+
+pub use markup5ever::*;
+
+pub(crate) mod macros;
+
+/// Driver
+pub mod driver;
+/// Serializer for XML5.
+pub mod serialize;
+/// XML5 tokenizer - converts input into tokens
+pub mod tokenizer;
+/// XML5 tree builder - converts tokens into a tree like structure
+pub mod tree_builder;
diff --git a/vendor/xml5ever-0.39.0/src/macros.rs b/vendor/xml5ever-0.39.0/src/macros.rs
new file mode 100644
index 000000000..0b786c38a
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/macros.rs
@@ -0,0 +1,18 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+macro_rules! time {
+ ($e:expr) => {{
+ let t0 = ::std::time::Instant::now();
+ let result = $e;
+ let dt = t0.elapsed().as_nanos() as u64;
+ (result, dt)
+ }};
+}
+pub(crate) use time;
diff --git a/vendor/xml5ever-0.39.0/src/serialize/mod.rs b/vendor/xml5ever-0.39.0/src/serialize/mod.rs
new file mode 100644
index 000000000..a2d267102
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/serialize/mod.rs
@@ -0,0 +1,216 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use crate::tree_builder::NamespaceMap;
+use crate::QualName;
+pub use markup5ever::serialize::{AttrRef, Serialize, Serializer, TraversalScope};
+use std::io::{self, Write};
+
+#[derive(Clone)]
+/// Struct for setting serializer options.
+pub struct SerializeOpts {
+ /// Serialize the root node? Default: ChildrenOnly
+ pub traversal_scope: TraversalScope,
+}
+
+impl Default for SerializeOpts {
+ fn default() -> SerializeOpts {
+ SerializeOpts {
+ traversal_scope: TraversalScope::ChildrenOnly(None),
+ }
+ }
+}
+
+/// Method for serializing generic node to a given writer.
+pub fn serialize<Wr, T>(writer: Wr, node: &T, opts: SerializeOpts) -> io::Result<()>
+where
+ Wr: Write,
+ T: Serialize,
+{
+ let mut ser = XmlSerializer::new(writer);
+ node.serialize(&mut ser, opts.traversal_scope)
+}
+
+/// Struct used for serializing nodes into a text that other XML
+/// parses can read.
+///
+/// Serializer contains a set of functions (start_elem, end_elem...)
+/// that make parsing nodes easier.
+pub struct XmlSerializer<Wr> {
+ writer: Wr,
+ namespace_stack: NamespaceMapStack,
+}
+
+#[derive(Debug)]
+struct NamespaceMapStack(Vec<NamespaceMap>);
+
+impl NamespaceMapStack {
+ fn new() -> NamespaceMapStack {
+ NamespaceMapStack(vec![])
+ }
+
+ fn push(&mut self, namespace: NamespaceMap) {
+ self.0.push(namespace);
+ }
+
+ fn pop(&mut self) {
+ self.0.pop();
+ }
+}
+
+/// Writes given text into the Serializer, escaping it,
+/// depending on where the text is written inside the tag or attribute value.
+///
+/// For example
+///```text
+/// <tag>'&-quotes'</tag> becomes <tag>'&amp;-quotes'</tag>
+/// <tag = "'&-quotes'"> becomes <tag = "&apos;&amp;-quotes&apos;"
+///```
+fn write_to_buf_escaped<W: Write>(writer: &mut W, text: &str, attr_mode: bool) -> io::Result<()> {
+ for c in text.chars() {
+ match c {
+ '&' => writer.write_all(b"&amp;"),
+ '\'' if attr_mode => writer.write_all(b"&apos;"),
+ '"' if attr_mode => writer.write_all(b"&quot;"),
+ '<' if !attr_mode => writer.write_all(b"&lt;"),
+ '>' if !attr_mode => writer.write_all(b"&gt;"),
+ c => writer.write_fmt(format_args!("{c}")),
+ }?;
+ }
+ Ok(())
+}
+
+#[inline]
+fn write_qual_name<W: Write>(writer: &mut W, name: &QualName) -> io::Result<()> {
+ if let Some(ref prefix) = name.prefix {
+ writer.write_all(prefix.as_bytes())?;
+ writer.write_all(b":")?;
+ }
+
+ writer.write_all(name.local.as_bytes())?;
+ Ok(())
+}
+
+impl<Wr: Write> XmlSerializer<Wr> {
+ /// Creates a new Serializier from a writer and given serialization options.
+ pub fn new(writer: Wr) -> Self {
+ XmlSerializer {
+ writer,
+ namespace_stack: NamespaceMapStack::new(),
+ }
+ }
+
+ #[inline(always)]
+ fn qual_name(&mut self, name: &QualName) -> io::Result<()> {
+ self.find_or_insert_ns(name);
+ write_qual_name(&mut self.writer, name)
+ }
+
+ #[inline(always)]
+ fn qual_attr_name(&mut self, name: &QualName) -> io::Result<()> {
+ self.find_or_insert_ns(name);
+ write_qual_name(&mut self.writer, name)
+ }
+
+ fn find_uri(&self, name: &QualName) -> bool {
+ let mut found = false;
+ for stack in self.namespace_stack.0.iter().rev() {
+ if let Some(Some(el)) = stack.get(&name.prefix) {
+ found = *el == name.ns;
+ break;
+ }
+ }
+ found
+ }
+
+ fn find_or_insert_ns(&mut self, name: &QualName) {
+ if (name.prefix.is_some() || !name.ns.is_empty()) && !self.find_uri(name) {
+ if let Some(last_ns) = self.namespace_stack.0.last_mut() {
+ last_ns.insert(name);
+ }
+ }
+ }
+}
+
+impl<Wr: Write> Serializer for XmlSerializer<Wr> {
+ /// Serializes given start element into text. Start element contains
+ /// qualified name and an attributes iterator.
+ fn start_elem<'a, AttrIter>(&mut self, name: QualName, attrs: AttrIter) -> io::Result<()>
+ where
+ AttrIter: Iterator<Item = AttrRef<'a>>,
+ {
+ self.namespace_stack.push(NamespaceMap::empty());
+
+ self.writer.write_all(b"<")?;
+ self.qual_name(&name)?;
+ if let Some(current_namespace) = self.namespace_stack.0.last() {
+ for (prefix, url_opt) in current_namespace.get_scope_iter() {
+ self.writer.write_all(b" xmlns")?;
+ if let Some(ref prefix) = *prefix {
+ self.writer.write_all(b":")?;
+ self.writer.write_all(prefix.as_bytes())?;
+ }
+
+ self.writer.write_all(b"=\"")?;
+ let url = if let Some(ref a) = *url_opt {
+ a.as_bytes()
+ } else {
+ b""
+ };
+ self.writer.write_all(url)?;
+ self.writer.write_all(b"\"")?;
+ }
+ }
+ for (name, value) in attrs {
+ self.writer.write_all(b" ")?;
+ self.qual_attr_name(name)?;
+ self.writer.write_all(b"=\"")?;
+ write_to_buf_escaped(&mut self.writer, value, true)?;
+ self.writer.write_all(b"\"")?;
+ }
+ self.writer.write_all(b">")?;
+ Ok(())
+ }
+
+ /// Serializes given end element into text.
+ fn end_elem(&mut self, name: QualName) -> io::Result<()> {
+ self.namespace_stack.pop();
+ self.writer.write_all(b"</")?;
+ self.qual_name(&name)?;
+ self.writer.write_all(b">")
+ }
+
+ /// Serializes comment into text.
+ fn write_comment(&mut self, text: &str) -> io::Result<()> {
+ self.writer.write_all(b"<!--")?;
+ self.writer.write_all(text.as_bytes())?;
+ self.writer.write_all(b"-->")
+ }
+
+ /// Serializes given doctype
+ fn write_doctype(&mut self, name: &str) -> io::Result<()> {
+ self.writer.write_all(b"<!DOCTYPE ")?;
+ self.writer.write_all(name.as_bytes())?;
+ self.writer.write_all(b">")
+ }
+
+ /// Serializes text for a node or an attributes.
+ fn write_text(&mut self, text: &str) -> io::Result<()> {
+ write_to_buf_escaped(&mut self.writer, text, false)
+ }
+
+ /// Serializes given processing instruction.
+ fn write_processing_instruction(&mut self, target: &str, data: &str) -> io::Result<()> {
+ self.writer.write_all(b"<?")?;
+ self.writer.write_all(target.as_bytes())?;
+ self.writer.write_all(b" ")?;
+ self.writer.write_all(data.as_bytes())?;
+ self.writer.write_all(b"?>")
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/tokenizer/char_ref/mod.rs b/vendor/xml5ever-0.39.0/src/tokenizer/char_ref/mod.rs
new file mode 100644
index 000000000..4728a76b6
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tokenizer/char_ref/mod.rs
@@ -0,0 +1,456 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::{TokenSink, XmlTokenizer};
+use crate::data;
+use crate::tendril::StrTendril;
+use log::debug;
+use markup5ever::buffer_queue::BufferQueue;
+use std::borrow::Cow::{self, Borrowed};
+use std::char::from_u32;
+
+use self::State::*;
+pub use self::Status::*;
+
+//§ tokenizing-character-references
+pub struct CharRef {
+ /// The resulting character(s)
+ pub chars: [char; 2],
+
+ /// How many slots in `chars` are valid?
+ pub num_chars: u8,
+}
+
+pub enum Status {
+ Stuck,
+ Progress,
+ Done,
+}
+
+#[derive(Debug)]
+enum State {
+ Begin,
+ Octothorpe,
+ Numeric(u32), // base
+ NumericSemicolon,
+ Named,
+ BogusName,
+}
+
+pub struct CharRefTokenizer {
+ state: State,
+ addnl_allowed: Option<char>,
+ result: Option<CharRef>,
+
+ num: u32,
+ num_too_big: bool,
+ seen_digit: bool,
+ hex_marker: Option<char>,
+
+ name_buf_opt: Option<StrTendril>,
+ name_match: Option<(u32, u32)>,
+ name_len: usize,
+}
+
+impl CharRefTokenizer {
+ // NB: We assume that we have an additional allowed character iff we're
+ // tokenizing in an attribute value.
+ pub fn new(addnl_allowed: Option<char>) -> CharRefTokenizer {
+ CharRefTokenizer {
+ state: Begin,
+ addnl_allowed,
+ result: None,
+ num: 0,
+ num_too_big: false,
+ seen_digit: false,
+ hex_marker: None,
+ name_buf_opt: None,
+ name_match: None,
+ name_len: 0,
+ }
+ }
+
+ // A CharRefTokenizer can only tokenize one character reference,
+ // so this method consumes the tokenizer.
+ pub fn get_result(self) -> CharRef {
+ self.result.expect("get_result called before done")
+ }
+
+ fn name_buf(&self) -> &StrTendril {
+ self.name_buf_opt
+ .as_ref()
+ .expect("name_buf missing in named character reference")
+ }
+
+ fn name_buf_mut(&mut self) -> &mut StrTendril {
+ self.name_buf_opt
+ .as_mut()
+ .expect("name_buf missing in named character reference")
+ }
+
+ fn finish_none(&mut self) -> Status {
+ self.result = Some(CharRef {
+ chars: ['\0', '\0'],
+ num_chars: 0,
+ });
+ Done
+ }
+
+ fn finish_one(&mut self, c: char) -> Status {
+ self.result = Some(CharRef {
+ chars: [c, '\0'],
+ num_chars: 1,
+ });
+ Done
+ }
+}
+
+impl CharRefTokenizer {
+ pub fn step<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ if self.result.is_some() {
+ return Done;
+ }
+
+ debug!("char ref tokenizer stepping in state {:?}", self.state);
+ match self.state {
+ Begin => self.do_begin(tokenizer, input),
+ Octothorpe => self.do_octothorpe(tokenizer, input),
+ Numeric(base) => self.do_numeric(tokenizer, base, input),
+ NumericSemicolon => self.do_numeric_semicolon(tokenizer, input),
+ Named => self.do_named(tokenizer, input),
+ BogusName => self.do_bogus_name(tokenizer, input),
+ }
+ }
+
+ fn do_begin<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ match tokenizer.peek(input) {
+ Some('\t' | '\n' | '\x0C' | ' ' | '<' | '&') => self.finish_none(),
+ Some(c) if Some(c) == self.addnl_allowed => self.finish_none(),
+ Some('#') => {
+ tokenizer.discard_char(input);
+ self.state = Octothorpe;
+ Progress
+ },
+ Some(_) => {
+ self.state = Named;
+ self.name_buf_opt = Some(StrTendril::new());
+ Progress
+ },
+ None => Stuck,
+ }
+ }
+
+ fn do_octothorpe<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ match tokenizer.peek(input) {
+ Some(c @ ('x' | 'X')) => {
+ tokenizer.discard_char(input);
+ self.hex_marker = Some(c);
+ self.state = Numeric(16);
+ },
+ Some(_) => {
+ self.hex_marker = None;
+ self.state = Numeric(10);
+ },
+ None => return Stuck,
+ }
+ Progress
+ }
+
+ fn do_numeric<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ base: u32,
+ input: &BufferQueue,
+ ) -> Status {
+ let Some(c) = tokenizer.peek(input) else {
+ return Stuck;
+ };
+ match c.to_digit(base) {
+ Some(n) => {
+ tokenizer.discard_char(input);
+ self.num = self.num.wrapping_mul(base);
+ if self.num > 0x10FFFF {
+ // We might overflow, and the character is definitely invalid.
+ // We still parse digits and semicolon, but don't use the result.
+ self.num_too_big = true;
+ }
+ self.num = self.num.wrapping_add(n);
+ self.seen_digit = true;
+ Progress
+ },
+
+ None if !self.seen_digit => self.unconsume_numeric(tokenizer, input),
+
+ None => {
+ self.state = NumericSemicolon;
+ Progress
+ },
+ }
+ }
+
+ fn do_numeric_semicolon<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ match tokenizer.peek(input) {
+ Some(';') => tokenizer.discard_char(input),
+ Some(_) => tokenizer.emit_error(Borrowed(
+ "Semicolon missing after numeric character reference",
+ )),
+ None => return Stuck,
+ };
+ self.finish_numeric(tokenizer)
+ }
+
+ fn unconsume_numeric<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ let mut unconsume = StrTendril::from_char('#');
+ if let Some(c) = self.hex_marker {
+ unconsume.push_char(c);
+ }
+
+ tokenizer.unconsume(input, unconsume);
+ tokenizer.emit_error(Borrowed("Numeric character reference without digits"));
+ self.finish_none()
+ }
+
+ fn finish_numeric<Sink: TokenSink>(&mut self, tokenizer: &XmlTokenizer<Sink>) -> Status {
+ fn conv(n: u32) -> char {
+ from_u32(n).expect("invalid char missed by error handling cases")
+ }
+
+ let (c, error) = match self.num {
+ n if (n > 0x10FFFF) || self.num_too_big => ('\u{fffd}', true),
+ 0x00 | 0xD800..=0xDFFF => ('\u{fffd}', true),
+
+ 0x80..=0x9F => match data::C1_REPLACEMENTS[(self.num - 0x80) as usize] {
+ Some(c) => (c, true),
+ None => (conv(self.num), true),
+ },
+
+ 0x01..=0x08 | 0x0B | 0x0D..=0x1F | 0x7F | 0xFDD0..=0xFDEF => (conv(self.num), true),
+
+ n if (n & 0xFFFE) == 0xFFFE => (conv(n), true),
+
+ n => (conv(n), false),
+ };
+
+ if error {
+ let msg = if tokenizer.opts.exact_errors {
+ Cow::from(format!(
+ "Invalid numeric character reference value 0x{:06X}",
+ self.num
+ ))
+ } else {
+ Cow::from("Invalid numeric character reference")
+ };
+ tokenizer.emit_error(msg);
+ }
+
+ self.finish_one(c)
+ }
+
+ fn do_named<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ let Some(c) = tokenizer.get_char(input) else {
+ return Stuck;
+ };
+ self.name_buf_mut().push_char(c);
+ match data::NAMED_ENTITIES.get(&self.name_buf()[..]) {
+ // We have either a full match or a prefix of one.
+ Some(&m) => {
+ if m.0 != 0 {
+ // We have a full match, but there might be a longer one to come.
+ self.name_match = Some(m);
+ self.name_len = self.name_buf().len();
+ }
+ // Otherwise we just have a prefix match.
+ Progress
+ },
+
+ // Can't continue the match.
+ None => self.finish_named(tokenizer, Some(c), input),
+ }
+ }
+
+ fn emit_name_error<Sink: TokenSink>(&mut self, tokenizer: &XmlTokenizer<Sink>) {
+ let msg = if tokenizer.opts.exact_errors {
+ Cow::from(format!("Invalid character reference &{}", self.name_buf()))
+ } else {
+ Cow::from("Invalid character reference")
+ };
+ tokenizer.emit_error(msg);
+ }
+
+ fn unconsume_name<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) {
+ tokenizer.unconsume(input, self.name_buf_opt.take().unwrap());
+ }
+
+ fn finish_named<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ end_char: Option<char>,
+ input: &BufferQueue,
+ ) -> Status {
+ match self.name_match {
+ None => {
+ match end_char {
+ Some(c) if c.is_ascii_alphanumeric() => {
+ // Keep looking for a semicolon, to determine whether
+ // we emit a parse error.
+ self.state = BogusName;
+ return Progress;
+ },
+
+ // Check length because &; is not a parse error.
+ Some(';') if self.name_buf().len() > 1 => self.emit_name_error(tokenizer),
+
+ _ => (),
+ }
+ self.unconsume_name(tokenizer, input);
+ self.finish_none()
+ },
+
+ Some((c1, c2)) => {
+ // We have a complete match, but we may have consumed
+ // additional characters into self.name_buf. Usually
+ // at least one, but several in cases like
+ //
+ // &not => match for U+00AC
+ // &noti => valid prefix for &notin
+ // &notit => can't continue match
+
+ let name_len = self.name_len;
+ assert!(name_len > 0);
+ let last_matched = self.name_buf()[name_len - 1..].chars().next().unwrap();
+
+ // There might not be a next character after the match, if
+ // we had a full match and then hit EOF.
+ let next_after = if name_len == self.name_buf().len() {
+ None
+ } else {
+ Some(self.name_buf()[name_len..].chars().next().unwrap())
+ };
+
+ // "If the character reference is being consumed as part of an
+ // attribute, and the last character matched is not a U+003B
+ // SEMICOLON character (;), and the next character is either a
+ // U+003D EQUALS SIGN character (=) or an alphanumeric ASCII
+ // character, then, for historical reasons, all the characters
+ // that were matched after the U+0026 AMPERSAND character (&)
+ // must be unconsumed, and nothing is returned. However, if
+ // this next character is in fact a U+003D EQUALS SIGN
+ // character (=), then this is a parse error"
+
+ let unconsume_all = match (self.addnl_allowed, last_matched, next_after) {
+ (_, ';', _) => false,
+ (Some(_), _, Some('=')) => {
+ tokenizer.emit_error(Borrowed(
+ "Equals sign after character reference in attribute",
+ ));
+ true
+ },
+ (Some(_), _, Some(c)) if c.is_ascii_alphanumeric() => true,
+ _ => {
+ tokenizer.emit_error(Borrowed(
+ "Character reference does not end with semicolon",
+ ));
+ false
+ },
+ };
+
+ if unconsume_all {
+ self.unconsume_name(tokenizer, input);
+ self.finish_none()
+ } else {
+ tokenizer
+ .unconsume(input, StrTendril::from_slice(&self.name_buf()[name_len..]));
+ self.result = Some(CharRef {
+ chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()],
+ num_chars: if c2 == 0 { 1 } else { 2 },
+ });
+ Done
+ }
+ },
+ }
+ }
+
+ fn do_bogus_name<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) -> Status {
+ let Some(c) = tokenizer.get_char(input) else {
+ return Stuck;
+ };
+ self.name_buf_mut().push_char(c);
+ match c {
+ _ if c.is_ascii_alphanumeric() => return Progress,
+ ';' => self.emit_name_error(tokenizer),
+ _ => (),
+ }
+ self.unconsume_name(tokenizer, input);
+ self.finish_none()
+ }
+
+ pub fn end_of_file<Sink: TokenSink>(
+ &mut self,
+ tokenizer: &XmlTokenizer<Sink>,
+ input: &BufferQueue,
+ ) {
+ while self.result.is_none() {
+ match self.state {
+ Begin => drop(self.finish_none()),
+
+ Numeric(_) if !self.seen_digit => drop(self.unconsume_numeric(tokenizer, input)),
+
+ Numeric(_) | NumericSemicolon => {
+ tokenizer.emit_error(Borrowed("EOF in numeric character reference"));
+ self.finish_numeric(tokenizer);
+ },
+
+ Named => drop(self.finish_named(tokenizer, None, input)),
+
+ BogusName => {
+ self.unconsume_name(tokenizer, input);
+ self.finish_none();
+ },
+
+ Octothorpe => {
+ tokenizer.unconsume(input, StrTendril::from_slice("#"));
+ tokenizer.emit_error(Borrowed("EOF after '#' in character reference"));
+ self.finish_none();
+ },
+ }
+ }
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/tokenizer/interface.rs b/vendor/xml5ever-0.39.0/src/tokenizer/interface.rs
new file mode 100644
index 000000000..15cc569a8
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tokenizer/interface.rs
@@ -0,0 +1,118 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::borrow::Cow;
+
+use crate::tendril::StrTendril;
+use crate::tokenizer::ProcessResult;
+use crate::{Attribute, QualName};
+
+pub use self::TagKind::{EmptyTag, EndTag, ShortTag, StartTag};
+
+/// Tag kind denotes which kind of tag did we encounter.
+#[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)]
+pub enum TagKind {
+ /// Beginning of a tag (e.g. `<a>`).
+ StartTag,
+ /// End of a tag (e.g. `</a>`).
+ EndTag,
+ /// Empty tag (e.g. `<a/>`).
+ EmptyTag,
+ /// Short tag (e.g. `</>`).
+ ShortTag,
+}
+
+/// XML 5 Tag Token
+#[derive(PartialEq, Eq, Debug, Clone)]
+pub struct Tag {
+ /// Token kind denotes which type of token was encountered.
+ /// E.g. if parser parsed `</a>` the token kind would be `EndTag`.
+ pub kind: TagKind,
+ /// Qualified name of the tag.
+ pub name: QualName,
+ /// List of attributes attached to this tag.
+ /// Only valid in start and empty tag.
+ pub attrs: Vec<Attribute>,
+}
+
+impl Tag {
+ /// Sorts attributes in a tag.
+ pub fn equiv_modulo_attr_order(&self, other: &Tag) -> bool {
+ if (self.kind != other.kind) || (self.name != other.name) {
+ return false;
+ }
+
+ let mut self_attrs = self.attrs.clone();
+ let mut other_attrs = other.attrs.clone();
+ self_attrs.sort();
+ other_attrs.sort();
+
+ self_attrs == other_attrs
+ }
+}
+
+/// A `DOCTYPE` token.
+/// Doctype token in XML5 is rather limited for reasons, such as:
+/// security and simplicity. XML5 only supports declaring DTD with
+/// name, public identifier and system identifier
+#[derive(PartialEq, Eq, Clone, Debug, Default)]
+pub struct Doctype {
+ /// Name of DOCTYPE declared
+ pub name: Option<StrTendril>,
+ /// Public identifier of this DOCTYPE.
+ pub public_id: Option<StrTendril>,
+ /// System identifier of this DOCTYPE.
+ pub system_id: Option<StrTendril>,
+}
+
+/// A ProcessingInstruction token.
+#[derive(PartialEq, Eq, Clone, Debug)]
+pub struct Pi {
+ /// What is the name of processing instruction.
+ pub target: StrTendril,
+
+ /// Text of processing instruction.
+ pub data: StrTendril,
+}
+
+/// Describes tokens encountered during parsing of input.
+#[derive(PartialEq, Eq, Debug)]
+pub enum Token {
+ /// Doctype token
+ Doctype(Doctype),
+ /// Token tag founds. This token applies to all
+ /// possible kinds of tags (like start, end, empty tag, etc.).
+ Tag(Tag),
+ /// Processing Instruction token
+ ProcessingInstruction(Pi),
+ /// Comment token.
+ Comment(StrTendril),
+ /// Token that represents a series of characters.
+ Characters(StrTendril),
+ /// Token that represents a CDATA section.
+ Cdata(StrTendril),
+ /// End of File found.
+ EndOfFile,
+ /// NullCharacter encountered.
+ NullCharacter,
+ /// Error happened
+ ParseError(Cow<'static, str>),
+}
+
+/// Types which can receive tokens from the tokenizer.
+pub trait TokenSink {
+ /// Handle to a DOM script element
+ type Handle;
+
+ /// Process a token.
+ fn process_token(&self, token: Token) -> ProcessResult<Self::Handle>;
+
+ /// Signal to the sink that parsing has ended.
+ fn end(&self) {}
+}
diff --git a/vendor/xml5ever-0.39.0/src/tokenizer/mod.rs b/vendor/xml5ever-0.39.0/src/tokenizer/mod.rs
new file mode 100644
index 000000000..515e7654b
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tokenizer/mod.rs
@@ -0,0 +1,1371 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+mod char_ref;
+mod interface;
+mod qname;
+pub mod states;
+
+pub use self::interface::{
+ Doctype, EmptyTag, EndTag, Pi, ShortTag, StartTag, Tag, TagKind, Token, TokenSink,
+};
+pub use crate::{LocalName, Namespace, Prefix};
+
+use crate::macros::time;
+use crate::tendril::StrTendril;
+use crate::{buffer_queue, Attribute, QualName, SmallCharSet};
+use log::debug;
+use markup5ever::{local_name, namespace_prefix, ns, small_char_set, TokenizerResult};
+use std::borrow::Cow::{self, Borrowed};
+use std::cell::{Cell, RefCell, RefMut};
+use std::collections::BTreeMap;
+use std::mem::replace;
+
+use buffer_queue::{BufferQueue, FromSet, NotFromSet, SetResult};
+use char_ref::{CharRef, CharRefTokenizer};
+use qname::QualNameTokenizer;
+use states::{AttrValueKind::*, DoctypeKind, DoctypeKind::*, XmlState};
+
+/// Copy of Tokenizer options, with an impl for `Default`.
+#[derive(Copy, Clone)]
+pub struct XmlTokenizerOpts {
+ /// Report all parse errors described in the spec, at some
+ /// performance penalty? Default: false
+ pub exact_errors: bool,
+
+ /// Discard a `U+FEFF BYTE ORDER MARK` if we see one at the beginning
+ /// of the stream? Default: true
+ pub discard_bom: bool,
+
+ /// Keep a record of how long we spent in each state? Printed
+ /// when `end()` is called. Default: false
+ pub profile: bool,
+
+ /// Initial state override. Only the test runner should use
+ /// a non-`None` value!
+ pub initial_state: Option<XmlState>,
+}
+
+fn process_qname(tag_name: StrTendril) -> QualName {
+ // If tag name can't possibly contain full namespace, skip qualified name
+ // parsing altogether. For a tag to have namespace it must look like:
+ // a:b
+ // Since StrTendril are UTF-8, we know that minimal size in bytes must be
+ // three bytes minimum.
+ let split = if (*tag_name).len() < 3 {
+ None
+ } else {
+ QualNameTokenizer::new((*tag_name).as_bytes()).run()
+ };
+
+ match split {
+ None => QualName::new(None, ns!(), LocalName::from(&*tag_name)),
+ Some(col) => {
+ let len = (*tag_name).len() as u32;
+ let prefix = tag_name.subtendril(0, col);
+ let local = tag_name.subtendril(col + 1, len - col - 1);
+ let ns = ns!(); // Actual namespace URL set in XmlTreeBuilder::bind_qname
+ QualName::new(Some(Prefix::from(&*prefix)), ns, LocalName::from(&*local))
+ },
+ }
+}
+
+fn option_push(opt_str: &mut Option<StrTendril>, c: char) {
+ match *opt_str {
+ Some(ref mut s) => s.push_char(c),
+ None => *opt_str = Some(StrTendril::from_char(c)),
+ }
+}
+
+impl Default for XmlTokenizerOpts {
+ fn default() -> XmlTokenizerOpts {
+ XmlTokenizerOpts {
+ exact_errors: false,
+ discard_bom: true,
+ profile: false,
+ initial_state: None,
+ }
+ }
+}
+/// The Xml tokenizer.
+pub struct XmlTokenizer<Sink> {
+ /// Options controlling the behavior of the tokenizer.
+ opts: XmlTokenizerOpts,
+
+ /// Destination for tokens we emit.
+ pub sink: Sink,
+
+ /// The abstract machine state as described in the spec.
+ state: Cell<XmlState>,
+
+ /// Are we at the end of the file, once buffers have been processed
+ /// completely? This affects whether we will wait for lookahead or not.
+ at_eof: Cell<bool>,
+
+ /// Tokenizer for character references, if we're tokenizing
+ /// one at the moment.
+ char_ref_tokenizer: RefCell<Option<Box<CharRefTokenizer>>>,
+
+ /// Current input character. Just consumed, may reconsume.
+ current_char: Cell<char>,
+
+ /// Should we reconsume the current input character?
+ reconsume: Cell<bool>,
+
+ /// Did we just consume \r, translating it to \n? In that case we need
+ /// to ignore the next character if it's \n.
+ ignore_lf: Cell<bool>,
+
+ /// Discard a U+FEFF BYTE ORDER MARK if we see one? Only done at the
+ /// beginning of the stream.
+ discard_bom: Cell<bool>,
+
+ /// Temporary buffer
+ temp_buf: RefCell<StrTendril>,
+
+ /// Current tag kind.
+ current_tag_kind: Cell<TagKind>,
+
+ /// Current tag name.
+ current_tag_name: RefCell<StrTendril>,
+
+ /// Current tag attributes.
+ current_tag_attrs: RefCell<Vec<Attribute>>,
+
+ /// Current attribute name.
+ current_attr_name: RefCell<StrTendril>,
+
+ /// Current attribute value.
+ current_attr_value: RefCell<StrTendril>,
+
+ current_doctype: RefCell<Doctype>,
+
+ /// Current comment.
+ current_comment: RefCell<StrTendril>,
+
+ /// Current CDATA section.
+ current_cdata: RefCell<StrTendril>,
+
+ /// Current processing instruction target.
+ current_pi_target: RefCell<StrTendril>,
+
+ /// Current processing instruction value.
+ current_pi_data: RefCell<StrTendril>,
+
+ /// Record of how many ns we spent in each state, if profiling is enabled.
+ state_profile: RefCell<BTreeMap<XmlState, u64>>,
+
+ /// Record of how many ns we spent in the token sink.
+ time_in_sink: Cell<u64>,
+}
+
+impl<Sink: TokenSink> XmlTokenizer<Sink> {
+ /// Create a new tokenizer which feeds tokens to a particular `TokenSink`.
+ pub fn new(sink: Sink, opts: XmlTokenizerOpts) -> XmlTokenizer<Sink> {
+ if opts.profile && cfg!(for_c) {
+ panic!("Can't profile tokenizer when built as a C library");
+ }
+
+ let state = *opts.initial_state.as_ref().unwrap_or(&XmlState::Data);
+ let discard_bom = opts.discard_bom;
+ XmlTokenizer {
+ opts,
+ sink,
+ state: Cell::new(state),
+ char_ref_tokenizer: RefCell::new(None),
+ at_eof: Cell::new(false),
+ current_char: Cell::new('\0'),
+ reconsume: Cell::new(false),
+ ignore_lf: Cell::new(false),
+ temp_buf: RefCell::new(StrTendril::new()),
+ discard_bom: Cell::new(discard_bom),
+ current_tag_kind: Cell::new(StartTag),
+ current_tag_name: RefCell::new(StrTendril::new()),
+ current_tag_attrs: RefCell::new(vec![]),
+ current_attr_name: RefCell::new(StrTendril::new()),
+ current_attr_value: RefCell::new(StrTendril::new()),
+ current_comment: RefCell::new(StrTendril::new()),
+ current_cdata: RefCell::new(StrTendril::new()),
+ current_pi_data: RefCell::new(StrTendril::new()),
+ current_pi_target: RefCell::new(StrTendril::new()),
+ current_doctype: RefCell::new(Doctype::default()),
+ state_profile: RefCell::new(BTreeMap::new()),
+ time_in_sink: Cell::new(0),
+ }
+ }
+
+ /// Feed an input string into the tokenizer.
+ pub fn feed(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle> {
+ if input.is_empty() {
+ return TokenizerResult::Done;
+ }
+
+ if self.discard_bom.get() {
+ if let Some(c) = input.peek() {
+ if c == '\u{feff}' {
+ input.next();
+ }
+ } else {
+ return TokenizerResult::Done;
+ }
+ };
+
+ self.run(input)
+ }
+
+ fn process_token(&self, token: Token) -> ProcessResult<Sink::Handle> {
+ if self.opts.profile {
+ let (result, dt) = time!(self.sink.process_token(token));
+ self.time_in_sink.set(self.time_in_sink.get() + dt);
+ result
+ } else {
+ self.sink.process_token(token)
+ }
+ }
+
+ // Get the next input character, which might be the character
+ // 'c' that we already consumed from the buffers.
+ fn get_preprocessed_char(&self, mut c: char, input: &BufferQueue) -> Option<char> {
+ if self.ignore_lf.get() {
+ self.ignore_lf.set(false);
+ if c == '\n' {
+ c = input.next()?;
+ }
+ }
+
+ if c == '\r' {
+ self.ignore_lf.set(true);
+ c = '\n';
+ }
+
+ // Normalize \x00 into \uFFFD
+ if c == '\x00' {
+ c = '\u{FFFD}'
+ }
+
+ // Exclude forbidden Unicode characters
+ if self.opts.exact_errors
+ && match c as u32 {
+ 0x01..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F..=0x9F | 0xFDD0..=0xFDEF => true,
+ n if (n & 0xFFFE) == 0xFFFE => true,
+ _ => false,
+ }
+ {
+ let msg = format!("Bad character {c}");
+ self.emit_error(Cow::Owned(msg));
+ }
+
+ debug!("got character {c}");
+ self.current_char.set(c);
+ Some(c)
+ }
+
+ fn bad_eof_error(&self) {
+ let msg = if self.opts.exact_errors {
+ Cow::from(format!("Saw EOF in state {:?}", self.state))
+ } else {
+ Cow::from("Unexpected EOF")
+ };
+ self.emit_error(msg);
+ }
+
+ fn pop_except_from(&self, input: &BufferQueue, set: SmallCharSet) -> Option<SetResult> {
+ // Bail to the slow path for various corner cases.
+ // This means that `FromSet` can contain characters not in the set!
+ // It shouldn't matter because the fallback `FromSet` case should
+ // always do the same thing as the `NotFromSet` case.
+ if self.opts.exact_errors || self.reconsume.get() || self.ignore_lf.get() {
+ return self.get_char(input).map(FromSet);
+ }
+
+ let d = input.pop_except_from(set);
+ debug!("got characters {d:?}");
+ match d {
+ Some(FromSet(c)) => self.get_preprocessed_char(c, input).map(FromSet),
+
+ // NB: We don't set self.current_char for a run of characters not
+ // in the set. It shouldn't matter for the codepaths that use
+ // this.
+ _ => d,
+ }
+ }
+
+ // Check if the next characters are an ASCII case-insensitive match. See
+ // BufferQueue::eat.
+ //
+ // NB: this doesn't do input stream preprocessing or set the current input
+ // character.
+ fn eat(&self, input: &BufferQueue, pat: &str) -> Option<bool> {
+ input.push_front(replace(&mut *self.temp_buf.borrow_mut(), StrTendril::new()));
+ match input.eat(pat, u8::eq_ignore_ascii_case) {
+ None if self.at_eof.get() => Some(false),
+ None => {
+ let mut temp_buf = self.temp_buf.borrow_mut();
+ while let Some(data) = input.next() {
+ temp_buf.push_char(data);
+ }
+ None
+ },
+ Some(matched) => Some(matched),
+ }
+ }
+
+ /// Run the state machine for as long as we can.
+ pub fn run(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle> {
+ if self.opts.profile {
+ loop {
+ let state = self.state.get();
+ let old_sink = self.time_in_sink.get();
+ let (run, mut dt) = time!(self.step(input));
+ dt -= self.time_in_sink.get() - old_sink;
+ let new = match self.state_profile.borrow_mut().get_mut(&state) {
+ Some(x) => {
+ *x += dt;
+ false
+ },
+ None => true,
+ };
+ if new {
+ // do this here because of borrow shenanigans
+ self.state_profile.borrow_mut().insert(state, dt);
+ }
+ match run {
+ ProcessResult::Continue => continue,
+ ProcessResult::Done => return TokenizerResult::Done,
+ ProcessResult::Script(handle) => return TokenizerResult::Script(handle),
+ }
+ }
+ } else {
+ loop {
+ match self.step(input) {
+ ProcessResult::Continue => continue,
+ ProcessResult::Done => return TokenizerResult::Done,
+ ProcessResult::Script(handle) => return TokenizerResult::Script(handle),
+ }
+ }
+ }
+ }
+
+ //§ tokenization
+ // Get the next input character, if one is available.
+ fn get_char(&self, input: &BufferQueue) -> Option<char> {
+ if self.reconsume.get() {
+ self.reconsume.set(false);
+ Some(self.current_char.get())
+ } else {
+ input
+ .next()
+ .and_then(|c| self.get_preprocessed_char(c, input))
+ }
+ }
+
+ fn bad_char_error(&self) {
+ let msg = if self.opts.exact_errors {
+ let c = self.current_char.get();
+ let state = self.state.get();
+ Cow::from(format!("Saw {c} in state {state:?}"))
+ } else {
+ Cow::from("Bad character")
+ };
+ self.emit_error(msg);
+ }
+
+ fn discard_tag(&self) {
+ *self.current_tag_name.borrow_mut() = StrTendril::new();
+ *self.current_tag_attrs.borrow_mut() = Vec::new();
+ }
+
+ fn create_tag(&self, kind: TagKind, c: char) {
+ self.discard_tag();
+ self.current_tag_name.borrow_mut().push_char(c);
+ self.current_tag_kind.set(kind);
+ }
+
+ // This method creates a PI token and
+ // sets its target to given char
+ fn create_pi(&self, c: char) {
+ *self.current_pi_target.borrow_mut() = StrTendril::new();
+ *self.current_pi_data.borrow_mut() = StrTendril::new();
+ self.current_pi_target.borrow_mut().push_char(c);
+ }
+
+ fn emit_char(&self, c: char) {
+ self.process_token(Token::Characters(StrTendril::from_char(match c {
+ '\0' => '\u{FFFD}',
+ c => c,
+ })));
+ }
+
+ fn emit_short_tag(&self) -> ProcessResult<Sink::Handle> {
+ self.current_tag_kind.set(ShortTag);
+ *self.current_tag_name.borrow_mut() = StrTendril::new();
+ self.emit_current_tag()
+ }
+
+ fn emit_empty_tag(&self) -> ProcessResult<Sink::Handle> {
+ self.current_tag_kind.set(EmptyTag);
+ self.emit_current_tag()
+ }
+
+ fn set_empty_tag(&self) {
+ self.current_tag_kind.set(EmptyTag);
+ }
+
+ fn emit_start_tag(&self) -> ProcessResult<Sink::Handle> {
+ self.current_tag_kind.set(StartTag);
+ self.emit_current_tag()
+ }
+
+ fn emit_current_tag(&self) -> ProcessResult<Sink::Handle> {
+ self.finish_attribute();
+
+ let qname = process_qname(replace(
+ &mut *self.current_tag_name.borrow_mut(),
+ StrTendril::new(),
+ ));
+
+ match self.current_tag_kind.get() {
+ StartTag | EmptyTag => {},
+ EndTag => {
+ if !self.current_tag_attrs.borrow().is_empty() {
+ self.emit_error(Borrowed("Attributes on an end tag"));
+ }
+ },
+ ShortTag => {
+ if !self.current_tag_attrs.borrow().is_empty() {
+ self.emit_error(Borrowed("Attributes on a short tag"));
+ }
+ },
+ }
+
+ let token = Token::Tag(Tag {
+ kind: self.current_tag_kind.get(),
+ name: qname,
+ attrs: self.current_tag_attrs.take(),
+ });
+
+ self.process_token(token)
+ }
+
+ // The string must not contain '\0'!
+ fn emit_chars(&self, b: StrTendril) {
+ self.process_token(Token::Characters(b));
+ }
+
+ // Emits the current Processing Instruction
+ fn emit_pi(&self) -> ProcessResult<<Sink as TokenSink>::Handle> {
+ let token = Token::ProcessingInstruction(Pi {
+ target: replace(&mut *self.current_pi_target.borrow_mut(), StrTendril::new()),
+ data: replace(&mut *self.current_pi_data.borrow_mut(), StrTendril::new()),
+ });
+ self.process_token(token)
+ }
+
+ fn consume_char_ref(&self, addnl_allowed: Option<char>) {
+ // NB: The char ref tokenizer assumes we have an additional allowed
+ // character iff we're tokenizing in an attribute value.
+ *self.char_ref_tokenizer.borrow_mut() =
+ Some(Box::new(CharRefTokenizer::new(addnl_allowed)));
+ }
+
+ fn emit_eof(&self) {
+ self.process_token(Token::EndOfFile);
+ }
+
+ fn emit_error(&self, error: Cow<'static, str>) {
+ self.process_token(Token::ParseError(error));
+ }
+
+ fn emit_current_comment(&self) {
+ let comment = self.current_comment.take();
+ self.process_token(Token::Comment(comment));
+ }
+
+ fn emit_current_cdata(&self) -> ProcessResult<<Sink as TokenSink>::Handle> {
+ let cdata = self.current_cdata.take();
+ self.process_token(Token::Cdata(cdata))
+ }
+
+ fn emit_incomplete_cdata_as_characters(&self, trailing_brackets: usize) {
+ let mut cdata = self.current_cdata.take();
+ for _ in 0..trailing_brackets {
+ cdata.push_char(']');
+ }
+ self.process_token(Token::Characters(cdata));
+ }
+
+ fn emit_current_doctype(&self) {
+ let doctype = self.current_doctype.take();
+ self.process_token(Token::Doctype(doctype));
+ }
+
+ fn doctype_id(&self, kind: DoctypeKind) -> RefMut<'_, Option<StrTendril>> {
+ let current_doctype = self.current_doctype.borrow_mut();
+ match kind {
+ DoctypeKind::Public => RefMut::map(current_doctype, |d| &mut d.public_id),
+ DoctypeKind::System => RefMut::map(current_doctype, |d| &mut d.system_id),
+ }
+ }
+
+ fn clear_doctype_id(&self, kind: DoctypeKind) {
+ let mut id = self.doctype_id(kind);
+ match *id {
+ Some(ref mut s) => s.clear(),
+ None => *id = Some(StrTendril::new()),
+ }
+ }
+
+ fn peek(&self, input: &BufferQueue) -> Option<char> {
+ if self.reconsume.get() {
+ Some(self.current_char.get())
+ } else {
+ input.peek()
+ }
+ }
+
+ fn discard_char(&self, input: &BufferQueue) {
+ let c = self.get_char(input);
+ assert!(c.is_some());
+ }
+
+ fn unconsume(&self, input: &BufferQueue, buf: StrTendril) {
+ input.push_front(buf);
+ }
+}
+
+// Shorthand for common state machine behaviors.
+macro_rules! shorthand (
+ ( $me:ident : emit $c:expr ) => ( $me.emit_char($c) );
+ ( $me:ident : create_tag $kind:ident $c:expr ) => ( $me.create_tag($kind, $c) );
+ ( $me:ident : push_tag $c:expr ) => ( $me.current_tag_name.borrow_mut().push_char($c) );
+ ( $me:ident : discard_tag $input:expr ) => ( $me.discard_tag($input) );
+ ( $me:ident : discard_char ) => ( $me.discard_char() );
+ ( $me:ident : push_temp $c:expr ) => ( $me.temp_buf.borrow_mut().push_char($c) );
+ ( $me:ident : emit_temp ) => ( $me.emit_temp_buf() );
+ ( $me:ident : clear_temp ) => ( $me.clear_temp_buf() );
+ ( $me:ident : create_attr $c:expr ) => ( $me.create_attribute($c) );
+ ( $me:ident : push_name $c:expr ) => ( $me.current_attr_name.borrow_mut().push_char($c) );
+ ( $me:ident : push_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_char($c) );
+ ( $me:ident : append_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_tendril($c));
+ ( $me:ident : push_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_char($c) );
+ ( $me:ident : append_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_slice($c) );
+ ( $me:ident : emit_comment ) => ( $me.emit_current_comment() );
+ ( $me:ident : clear_comment ) => ( $me.current_comment.borrow_mut().clear() );
+ ( $me:ident : push_cdata $c:expr ) => ( $me.current_cdata.borrow_mut().push_char($c) );
+ ( $me:ident : clear_cdata ) => ( $me.current_cdata.borrow_mut().clear() );
+ ( $me:ident : emit_incomplete_cdata $n:expr ) => ( $me.emit_incomplete_cdata_as_characters($n) );
+ ( $me:ident : create_doctype ) => ( *$me.current_doctype.borrow_mut() = Doctype::default() );
+ ( $me:ident : push_doctype_name $c:expr ) => ( option_push(&mut $me.current_doctype.borrow_mut().name, $c) );
+ ( $me:ident : push_doctype_id $k:ident $c:expr ) => ( option_push(&mut $me.doctype_id($k), $c) );
+ ( $me:ident : clear_doctype_id $k:ident ) => ( $me.clear_doctype_id($k) );
+ ( $me:ident : emit_doctype ) => ( $me.emit_current_doctype() );
+ ( $me:ident : error ) => ( $me.bad_char_error() );
+ ( $me:ident : error_eof ) => ( $me.bad_eof_error() );
+ ( $me:ident : create_pi $c:expr ) => ( $me.create_pi($c) );
+ ( $me:ident : push_pi_target $c:expr ) => ( $me.current_pi_target.borrow_mut().push_char($c) );
+ ( $me:ident : push_pi_data $c:expr ) => ( $me.current_pi_data.borrow_mut().push_char($c) );
+ ( $me:ident : set_empty_tag ) => ( $me.set_empty_tag() );
+);
+
+// Tracing of tokenizer actions. This adds significant bloat and compile time,
+// so it's behind a cfg flag.
+#[cfg(feature = "trace_tokenizer")]
+macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ({
+ debug!(" {:?}", stringify!($($cmds)*));
+ shorthand!($me : $($cmds)*);
+}));
+
+#[cfg(not(feature = "trace_tokenizer"))]
+macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ( shorthand!($me: $($cmds)*) ) );
+
+// A little DSL for sequencing shorthand actions.
+macro_rules! go (
+ // A pattern like $($cmd:tt)* ; $($rest:tt)* causes parse ambiguity.
+ // We have to tell the parser how much lookahead we need.
+
+ ( $me:ident : $a:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a); go!($me: $($rest)*); });
+ ( $me:ident : $a:tt $b:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b); go!($me: $($rest)*); });
+ ( $me:ident : $a:tt $b:tt $c:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c); go!($me: $($rest)*); });
+ ( $me:ident : $a:tt $b:tt $c:tt $d:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c $d); go!($me: $($rest)*); });
+
+ // These can only come at the end.
+
+ ( $me:ident : to $s:ident ) => ({ $me.state.set(XmlState::$s); return ProcessResult::Continue; });
+ ( $me:ident : to $s:ident $k1:expr ) => ({ $me.state.set(XmlState::$s($k1)); return ProcessResult::Continue; });
+ ( $me:ident : to $s:ident $k1:ident $k2:expr ) => ({ $me.state.set(XmlState::$s($k1($k2))); return ProcessResult::Continue; });
+
+ ( $me:ident : reconsume $s:ident ) => ({ $me.reconsume.set(true); go!($me: to $s); });
+ ( $me:ident : reconsume $s:ident $k1:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1); });
+ ( $me:ident : reconsume $s:ident $k1:ident $k2:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1 $k2); });
+
+ ( $me:ident : consume_char_ref ) => ({ $me.consume_char_ref(None); return ProcessResult::Continue; });
+ ( $me:ident : consume_char_ref $addnl:expr ) => ({ $me.consume_char_ref(Some($addnl)); return ProcessResult::Continue; });
+
+ // We have a default next state after emitting a tag, but the sink can override.
+ ( $me:ident : emit_tag $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_current_tag();
+ });
+
+ // We have a special when dealing with empty and short tags in Xml
+ ( $me:ident : emit_short_tag $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_short_tag();
+ });
+
+ ( $me:ident : emit_empty_tag $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_empty_tag();
+ });
+
+ ( $me:ident : emit_start_tag $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_start_tag();
+ });
+
+ ( $me:ident : emit_pi $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_pi();
+ });
+
+ ( $me:ident : emit_cdata $s:ident ) => ({
+ $me.state.set(XmlState::$s);
+ return $me.emit_current_cdata();
+ });
+
+ ( $me:ident : eof ) => ({ $me.emit_eof(); return ProcessResult::Done; });
+
+ // If nothing else matched, it's a single command
+ ( $me:ident : $($cmd:tt)+ ) => ( sh_trace!($me: $($cmd)+) );
+
+ // or nothing.
+ ( $me:ident : ) => (());
+);
+
+// This is a macro because it can cause early return
+// from the function where it is used.
+macro_rules! get_char ( ($me:expr, $input:expr) => {{
+ let Some(character) = $me.get_char($input) else {
+ return ProcessResult::Done;
+ };
+ character
+}});
+
+macro_rules! pop_except_from ( ($me:expr, $input:expr, $set:expr) => {{
+ let Some(popped_element) = $me.pop_except_from($input, $set) else {
+ return ProcessResult::Done;
+ };
+ popped_element
+}});
+
+macro_rules! eat ( ($me:expr, $input:expr, $pat:expr) => {{
+ let Some(value) = $me.eat($input, $pat) else {
+ return ProcessResult::Done;
+ };
+ value
+}});
+
+/// The result of a single tokenization operation
+pub enum ProcessResult<Handle> {
+ /// The tokenizer needs more input before it can continue
+ Done,
+ /// The tokenizer can be invoked again immediately
+ Continue,
+ /// The tokenizer encountered a script element that must be executed
+ /// before tokenization can continue
+ Script(Handle),
+}
+
+impl<Sink: TokenSink> XmlTokenizer<Sink> {
+ // Run the state machine for a while.
+ #[allow(clippy::never_loop)]
+ fn step(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle> {
+ if self.char_ref_tokenizer.borrow().is_some() {
+ return self.step_char_ref_tokenizer(input);
+ }
+
+ debug!("processing in state {:?}", self.state);
+ match self.state.get() {
+ //§ data-state
+ XmlState::Data => loop {
+ match pop_except_from!(self, input, small_char_set!('\r' '&' '<')) {
+ FromSet('&') => go!(self: consume_char_ref),
+ FromSet('<') => go!(self: to TagState),
+ FromSet(c) => go!(self: emit c),
+ NotFromSet(b) => self.emit_chars(b),
+ }
+ },
+ //§ tag-state
+ XmlState::TagState => loop {
+ match get_char!(self, input) {
+ '!' => go!(self: to MarkupDecl),
+ '/' => go!(self: to EndTagState),
+ '?' => go!(self: to Pi),
+ '\t' | '\n' | ' ' | ':' | '<' | '>' => {
+ go!(self: error; emit '<'; reconsume Data)
+ },
+ cl => go!(self: create_tag StartTag cl; to TagName),
+ }
+ },
+ //§ end-tag-state
+ XmlState::EndTagState => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_short_tag Data),
+ '\t' | '\n' | ' ' | '<' | ':' => {
+ go!(self: error; emit '<'; emit '/'; reconsume Data)
+ },
+ cl => go!(self: create_tag EndTag cl; to EndTagName),
+ }
+ },
+ //§ end-tag-name-state
+ XmlState::EndTagName => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => go!(self: to EndTagNameAfter),
+ '/' => go!(self: error; to EndTagNameAfter),
+ '>' => go!(self: emit_tag Data),
+ cl => go!(self: push_tag cl),
+ }
+ },
+ //§ end-tag-name-after-state
+ XmlState::EndTagNameAfter => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_tag Data),
+ '\t' | '\n' | ' ' => (),
+ _ => self.emit_error(Borrowed("Unexpected element in tag name")),
+ }
+ },
+ //§ pi-state
+ XmlState::Pi => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => go!(self: error; reconsume BogusComment),
+ cl => go!(self: create_pi cl; to PiTarget),
+ }
+ },
+ //§ pi-target-state
+ XmlState::PiTarget => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => go!(self: to PiTargetAfter),
+ '?' => go!(self: to PiAfter),
+ cl => go!(self: push_pi_target cl),
+ }
+ },
+ //§ pi-target-after-state
+ XmlState::PiTargetAfter => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => (),
+ _ => go!(self: reconsume PiData),
+ }
+ },
+ //§ pi-data-state
+ XmlState::PiData => loop {
+ match get_char!(self, input) {
+ '?' => go!(self: to PiAfter),
+ cl => go!(self: push_pi_data cl),
+ }
+ },
+ //§ pi-after-state
+ XmlState::PiAfter => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_pi Data),
+ '?' => go!(self: to PiAfter),
+ cl => go!(self: push_pi_data cl),
+ }
+ },
+ //§ markup-declaration-state
+ XmlState::MarkupDecl => loop {
+ if eat!(self, input, "--") {
+ go!(self: clear_comment; to CommentStart);
+ } else if eat!(self, input, "[CDATA[") {
+ go!(self: clear_cdata; to Cdata);
+ } else if eat!(self, input, "DOCTYPE") {
+ go!(self: to Doctype);
+ } else {
+ // FIXME: 'error' gives wrong message
+ go!(self: error; to BogusComment);
+ }
+ },
+ //§ comment-start-state
+ XmlState::CommentStart => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: to CommentStartDash),
+ '>' => go!(self: error; emit_comment; to Data),
+ _ => go!(self: reconsume Comment),
+ }
+ },
+ //§ comment-start-dash-state
+ XmlState::CommentStartDash => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: to CommentEnd),
+ '>' => go!(self: error; emit_comment; to Data),
+ _ => go!(self: push_comment '-'; reconsume Comment),
+ }
+ },
+ //§ comment-state
+ XmlState::Comment => loop {
+ match get_char!(self, input) {
+ '<' => go!(self: push_comment '<'; to CommentLessThan),
+ '-' => go!(self: to CommentEndDash),
+ c => go!(self: push_comment c),
+ }
+ },
+ //§ comment-less-than-sign-state
+ XmlState::CommentLessThan => loop {
+ match get_char!(self, input) {
+ '!' => go!(self: push_comment '!';to CommentLessThanBang),
+ '<' => go!(self: push_comment '<'),
+ _ => go!(self: reconsume Comment),
+ }
+ },
+ //§ comment-less-than-sign-bang-state
+ XmlState::CommentLessThanBang => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: to CommentLessThanBangDash),
+ _ => go!(self: reconsume Comment),
+ }
+ },
+ //§ comment-less-than-sign-bang-dash-state
+ XmlState::CommentLessThanBangDash => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: to CommentLessThanBangDashDash),
+ _ => go!(self: reconsume CommentEndDash),
+ }
+ },
+ //§ comment-less-than-sign-bang-dash-dash-state
+ XmlState::CommentLessThanBangDashDash => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: reconsume CommentEnd),
+ _ => go!(self: error; reconsume CommentEnd),
+ }
+ },
+ //§ comment-end-dash-state
+ XmlState::CommentEndDash => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: to CommentEnd),
+ _ => go!(self: push_comment '-'; reconsume Comment),
+ }
+ },
+ //§ comment-end-state
+ XmlState::CommentEnd => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_comment; to Data),
+ '!' => go!(self: to CommentEndBang),
+ '-' => go!(self: push_comment '-'),
+ _ => go!(self: append_comment "--"; reconsume Comment),
+ }
+ },
+ //§ comment-end-bang-state
+ XmlState::CommentEndBang => loop {
+ match get_char!(self, input) {
+ '-' => go!(self: append_comment "--!"; to CommentEndDash),
+ '>' => go!(self: error; emit_comment; to Data),
+ _ => go!(self: append_comment "--!"; reconsume Comment),
+ }
+ },
+ //§ bogus-comment-state
+ XmlState::BogusComment => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_comment; to Data),
+ c => go!(self: push_comment c),
+ }
+ },
+ //§ cdata-state
+ XmlState::Cdata => loop {
+ match get_char!(self, input) {
+ ']' => go!(self: to CdataBracket),
+ cl => go!(self: push_cdata cl),
+ }
+ },
+ //§ cdata-bracket-state
+ XmlState::CdataBracket => loop {
+ match get_char!(self, input) {
+ ']' => go!(self: to CdataEnd),
+ cl => go!(self: push_cdata ']'; push_cdata cl; to Cdata),
+ }
+ },
+ //§ cdata-end-state
+ XmlState::CdataEnd => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_cdata Data),
+ ']' => go!(self: push_cdata ']'),
+ cl => go!(self: push_cdata ']'; push_cdata ']'; push_cdata cl; to Cdata),
+ }
+ },
+ //§ tag-name-state
+ XmlState::TagName => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => go!(self: to TagAttrNameBefore),
+ '>' => go!(self: emit_tag Data),
+ '/' => go!(self: set_empty_tag; to TagEmpty),
+ cl => go!(self: push_tag cl),
+ }
+ },
+ //§ empty-tag-state
+ XmlState::TagEmpty => loop {
+ match get_char!(self, input) {
+ '>' => go!(self: emit_empty_tag Data),
+ _ => go!(self: reconsume TagAttrValueBefore),
+ }
+ },
+ //§ tag-attribute-name-before-state
+ XmlState::TagAttrNameBefore => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => (),
+ '>' => go!(self: emit_tag Data),
+ '/' => go!(self: set_empty_tag; to TagEmpty),
+ ':' => go!(self: error),
+ cl => go!(self: create_attr cl; to TagAttrName),
+ }
+ },
+ //§ tag-attribute-name-state
+ XmlState::TagAttrName => loop {
+ match get_char!(self, input) {
+ '=' => go!(self: to TagAttrValueBefore),
+ '>' => go!(self: emit_tag Data),
+ '\t' | '\n' | ' ' => go!(self: to TagAttrNameAfter),
+ '/' => go!(self: set_empty_tag; to TagEmpty),
+ cl => go!(self: push_name cl),
+ }
+ },
+ //§ tag-attribute-name-after-state
+ XmlState::TagAttrNameAfter => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => (),
+ '=' => go!(self: to TagAttrValueBefore),
+ '>' => go!(self: emit_tag Data),
+ '/' => go!(self: set_empty_tag; to TagEmpty),
+ cl => go!(self: create_attr cl; to TagAttrName),
+ }
+ },
+ //§ tag-attribute-value-before-state
+ XmlState::TagAttrValueBefore => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | ' ' => (),
+ '"' => go!(self: to TagAttrValue DoubleQuoted),
+ '\'' => go!(self: to TagAttrValue SingleQuoted),
+ '&' => go!(self: reconsume TagAttrValue(Unquoted)),
+ '>' => go!(self: emit_tag Data),
+ cl => go!(self: push_value cl; to TagAttrValue(Unquoted)),
+ }
+ },
+ //§ tag-attribute-value-double-quoted-state
+ XmlState::TagAttrValue(DoubleQuoted) => loop {
+ match pop_except_from!(self, input, small_char_set!('\n' '"' '&')) {
+ FromSet('"') => go!(self: to TagAttrNameBefore),
+ FromSet('&') => go!(self: consume_char_ref '"' ),
+ FromSet(c) => go!(self: push_value c),
+ NotFromSet(ref b) => go!(self: append_value b),
+ }
+ },
+ //§ tag-attribute-value-single-quoted-state
+ XmlState::TagAttrValue(SingleQuoted) => loop {
+ match pop_except_from!(self, input, small_char_set!('\n' '\'' '&')) {
+ FromSet('\'') => go!(self: to TagAttrNameBefore),
+ FromSet('&') => go!(self: consume_char_ref '\''),
+ FromSet(c) => go!(self: push_value c),
+ NotFromSet(ref b) => go!(self: append_value b),
+ }
+ },
+ //§ tag-attribute-value-double-quoted-state
+ XmlState::TagAttrValue(Unquoted) => loop {
+ match pop_except_from!(self, input, small_char_set!('\n' '\t' ' ' '&' '>')) {
+ FromSet('\t') | FromSet('\n') | FromSet(' ') => go!(self: to TagAttrNameBefore),
+ FromSet('&') => go!(self: consume_char_ref),
+ FromSet('>') => go!(self: emit_tag Data),
+ FromSet(c) => go!(self: push_value c),
+ NotFromSet(ref b) => go!(self: append_value b),
+ }
+ },
+
+ //§ doctype-state
+ XmlState::Doctype => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeName),
+ _ => go!(self: error; reconsume BeforeDoctypeName),
+ }
+ },
+ //§ before-doctype-name-state
+ XmlState::BeforeDoctypeName => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => (),
+ '>' => go!(self: error; emit_doctype; to Data),
+ c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase());
+ to DoctypeName),
+ }
+ },
+ //§ doctype-name-state
+ XmlState::DoctypeName => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to AfterDoctypeName),
+ '>' => go!(self: emit_doctype; to Data),
+ c => go!(self: push_doctype_name (c.to_ascii_lowercase());
+ to DoctypeName),
+ }
+ },
+ //§ after-doctype-name-state
+ XmlState::AfterDoctypeName => loop {
+ if eat!(self, input, "public") {
+ go!(self: to AfterDoctypeKeyword Public);
+ } else if eat!(self, input, "system") {
+ go!(self: to AfterDoctypeKeyword System);
+ } else {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => (),
+ '>' => go!(self: emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ }
+ },
+ //§ after-doctype-public-keyword-state
+ XmlState::AfterDoctypeKeyword(Public) => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeIdentifier Public),
+ '"' => {
+ go!(self: error; clear_doctype_id Public; to DoctypeIdentifierDoubleQuoted Public)
+ },
+ '\'' => {
+ go!(self: error; clear_doctype_id Public; to DoctypeIdentifierSingleQuoted Public)
+ },
+ '>' => go!(self: error; emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ after-doctype-system-keyword-state
+ XmlState::AfterDoctypeKeyword(System) => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeIdentifier System),
+ '"' => {
+ go!(self: error; clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System)
+ },
+ '\'' => {
+ go!(self: error; clear_doctype_id System; to DoctypeIdentifierSingleQuoted System)
+ },
+ '>' => go!(self: error; emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ before_doctype_public_identifier_state before_doctype_system_identifier_state
+ XmlState::BeforeDoctypeIdentifier(kind) => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => (),
+ '"' => go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind),
+ '\'' => go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind),
+ '>' => go!(self: error; emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ doctype_public_identifier_double_quoted_state doctype_system_identifier_double_quoted_state
+ XmlState::DoctypeIdentifierDoubleQuoted(kind) => loop {
+ match get_char!(self, input) {
+ '"' => go!(self: to AfterDoctypeIdentifier kind),
+ '>' => go!(self: error; emit_doctype; to Data),
+ c => go!(self: push_doctype_id kind c),
+ }
+ },
+ //§ doctype_public_identifier_single_quoted_state doctype_system_identifier_single_quoted_state
+ XmlState::DoctypeIdentifierSingleQuoted(kind) => loop {
+ match get_char!(self, input) {
+ '\'' => go!(self: to AfterDoctypeIdentifier kind),
+ '>' => go!(self: error; emit_doctype; to Data),
+ c => go!(self: push_doctype_id kind c),
+ }
+ },
+ //§ doctype_public_identifier_single_quoted_state
+ XmlState::AfterDoctypeIdentifier(Public) => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => {
+ go!(self: to BetweenDoctypePublicAndSystemIdentifiers)
+ },
+ '\'' => {
+ go!(self: error; clear_doctype_id System; to DoctypeIdentifierSingleQuoted(System))
+ },
+ '"' => {
+ go!(self: error; clear_doctype_id System; to DoctypeIdentifierDoubleQuoted(System))
+ },
+ '>' => go!(self: emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ doctype_system_identifier_single_quoted_state
+ XmlState::AfterDoctypeIdentifier(System) => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => (),
+ '>' => go!(self: emit_doctype; to Data),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ between_doctype_public_and_system_identifier_state
+ XmlState::BetweenDoctypePublicAndSystemIdentifiers => loop {
+ match get_char!(self, input) {
+ '\t' | '\n' | '\x0C' | ' ' => (),
+ '>' => go!(self: emit_doctype; to Data),
+ '\'' => go!(self: to DoctypeIdentifierSingleQuoted System),
+ '"' => go!(self: to DoctypeIdentifierDoubleQuoted System),
+ _ => go!(self: error; to BogusDoctype),
+ }
+ },
+ //§ bogus_doctype_state
+ XmlState::BogusDoctype => loop {
+ if get_char!(self, input) == '>' {
+ go!(self: emit_doctype; to Data);
+ }
+ },
+ }
+ }
+
+ /// Indicate that we have reached the end of the input.
+ pub fn end(&self) {
+ // Handle EOF in the char ref sub-tokenizer, if there is one.
+ // Do this first because it might un-consume stuff.
+ let input = BufferQueue::default();
+ match self.char_ref_tokenizer.take() {
+ None => (),
+ Some(mut tok) => {
+ tok.end_of_file(self, &input);
+ self.process_char_ref(tok.get_result());
+ },
+ }
+
+ // Process all remaining buffered input.
+ // If we're waiting for lookahead, we're not gonna get it.
+ self.at_eof.set(true);
+ let _ = self.run(&input);
+
+ loop {
+ if !matches!(self.eof_step(), ProcessResult::Continue) {
+ break;
+ }
+ }
+
+ self.sink.end();
+
+ if self.opts.profile {
+ self.dump_profile();
+ }
+ }
+
+ #[cfg(for_c)]
+ fn dump_profile(&self) {
+ unreachable!();
+ }
+
+ #[cfg(not(for_c))]
+ fn dump_profile(&self) {
+ let mut results: Vec<(XmlState, u64)> = self
+ .state_profile
+ .borrow()
+ .iter()
+ .map(|(s, t)| (*s, *t))
+ .collect();
+ results.sort_by(|&(_, x), &(_, y)| y.cmp(&x));
+
+ let total: u64 = results
+ .iter()
+ .map(|&(_, t)| t)
+ .fold(0, ::std::ops::Add::add);
+ debug!("\nTokenizer profile, in nanoseconds");
+ debug!(
+ "\n{:12} total in token sink",
+ self.time_in_sink.get()
+ );
+ debug!("\n{total:12} total in tokenizer");
+
+ for (k, v) in results.into_iter() {
+ let pct = 100.0 * (v as f64) / (total as f64);
+ debug!("{v:12} {pct:4.1}% {k:?}");
+ }
+ }
+
+ fn eof_step(&self) -> ProcessResult<Sink::Handle> {
+ debug!("processing EOF in state {:?}", self.state.get());
+ match self.state.get() {
+ XmlState::Data => go!(self: eof),
+ XmlState::CommentStart | XmlState::CommentLessThan | XmlState::CommentLessThanBang => {
+ go!(self: reconsume Comment)
+ },
+ XmlState::CommentLessThanBangDash => go!(self: reconsume CommentEndDash),
+ XmlState::CommentLessThanBangDashDash => go!(self: reconsume CommentEnd),
+ XmlState::CommentStartDash
+ | XmlState::Comment
+ | XmlState::CommentEndDash
+ | XmlState::CommentEnd
+ | XmlState::CommentEndBang => go!(self: error_eof; emit_comment; eof),
+ XmlState::TagState => go!(self: error_eof; emit '<'; to Data),
+ XmlState::EndTagState => go!(self: error_eof; emit '<'; emit '/'; to Data),
+ XmlState::TagEmpty => go!(self: error_eof; to TagAttrNameBefore),
+ XmlState::Cdata => go!(self: error_eof; emit_incomplete_cdata 0; to Data),
+ XmlState::CdataBracket => {
+ go!(self: error_eof; emit_incomplete_cdata 1; to Data)
+ },
+ XmlState::CdataEnd => go!(self: error_eof; emit_incomplete_cdata 2; to Data),
+ XmlState::Pi => go!(self: error_eof; to BogusComment),
+ XmlState::PiTargetAfter | XmlState::PiAfter => go!(self: reconsume PiData),
+ XmlState::MarkupDecl => go!(self: error_eof; to BogusComment),
+ XmlState::TagName
+ | XmlState::TagAttrNameBefore
+ | XmlState::EndTagName
+ | XmlState::TagAttrNameAfter
+ | XmlState::EndTagNameAfter
+ | XmlState::TagAttrValueBefore
+ | XmlState::TagAttrValue(_) => go!(self: error_eof; emit_tag Data),
+ XmlState::PiData | XmlState::PiTarget => go!(self: error_eof; emit_pi Data),
+ XmlState::TagAttrName => go!(self: error_eof; emit_start_tag Data),
+ XmlState::BeforeDoctypeName
+ | XmlState::Doctype
+ | XmlState::DoctypeName
+ | XmlState::AfterDoctypeName
+ | XmlState::AfterDoctypeKeyword(_)
+ | XmlState::BeforeDoctypeIdentifier(_)
+ | XmlState::AfterDoctypeIdentifier(_)
+ | XmlState::DoctypeIdentifierSingleQuoted(_)
+ | XmlState::DoctypeIdentifierDoubleQuoted(_)
+ | XmlState::BetweenDoctypePublicAndSystemIdentifiers => {
+ go!(self: error_eof; emit_doctype; to Data)
+ },
+ XmlState::BogusDoctype => go!(self: emit_doctype; to Data),
+ XmlState::BogusComment => go!(self: emit_comment; to Data),
+ }
+ }
+
+ fn process_char_ref(&self, char_ref: CharRef) {
+ let CharRef {
+ mut chars,
+ mut num_chars,
+ } = char_ref;
+
+ if num_chars == 0 {
+ chars[0] = '&';
+ num_chars = 1;
+ }
+
+ for i in 0..num_chars {
+ let c = chars[i as usize];
+ match self.state.get() {
+ XmlState::Data | XmlState::Cdata => go!(self: emit c),
+
+ XmlState::TagAttrValue(_) => go!(self: push_value c),
+
+ _ => panic!(
+ "state {:?} should not be reachable in process_char_ref",
+ self.state.get()
+ ),
+ }
+ }
+ }
+
+ fn step_char_ref_tokenizer(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle> {
+ let mut tok = self.char_ref_tokenizer.take().unwrap();
+ let outcome = tok.step(self, input);
+
+ let progress = match outcome {
+ char_ref::Done => {
+ self.process_char_ref(tok.get_result());
+ return ProcessResult::Continue;
+ },
+
+ char_ref::Stuck => ProcessResult::Done,
+ char_ref::Progress => ProcessResult::Continue,
+ };
+
+ *self.char_ref_tokenizer.borrow_mut() = Some(tok);
+ progress
+ }
+
+ fn finish_attribute(&self) {
+ if self.current_attr_name.borrow().is_empty() {
+ return;
+ }
+
+ // Check for a duplicate attribute.
+ // FIXME: the spec says we should error as soon as the name is finished.
+ // FIXME: linear time search, do we care?
+ let dup = {
+ let current_attr_name = self.current_attr_name.borrow();
+ let name = &current_attr_name[..];
+ self.current_tag_attrs
+ .borrow()
+ .iter()
+ .any(|a| &*a.name.local == name)
+ };
+
+ if dup {
+ self.emit_error(Borrowed("Duplicate attribute"));
+ self.current_attr_name.borrow_mut().clear();
+ self.current_attr_value.borrow_mut().clear();
+ } else {
+ let qname = process_qname(replace(
+ &mut self.current_attr_name.borrow_mut(),
+ StrTendril::new(),
+ ));
+ let attr = Attribute {
+ name: qname.clone(),
+ value: replace(&mut self.current_attr_value.borrow_mut(), StrTendril::new()),
+ };
+
+ if qname.local == local_name!("xmlns")
+ || qname.prefix == Some(namespace_prefix!("xmlns"))
+ {
+ self.current_tag_attrs.borrow_mut().insert(0, attr);
+ } else {
+ self.current_tag_attrs.borrow_mut().push(attr);
+ }
+ }
+ }
+
+ fn create_attribute(&self, c: char) {
+ self.finish_attribute();
+
+ self.current_attr_name.borrow_mut().push_char(c);
+ }
+}
+
+#[cfg(test)]
+mod test {
+
+ use super::process_qname;
+ use crate::tendril::SliceExt;
+ use crate::{LocalName, Prefix};
+
+ #[test]
+ fn simple_namespace() {
+ let qname = process_qname("prefix:local".to_tendril());
+ assert_eq!(qname.prefix, Some(Prefix::from("prefix")));
+ assert_eq!(qname.local, LocalName::from("local"));
+
+ let qname = process_qname("a:b".to_tendril());
+ assert_eq!(qname.prefix, Some(Prefix::from("a")));
+ assert_eq!(qname.local, LocalName::from("b"));
+ }
+
+ #[test]
+ fn wrong_namespaces() {
+ let qname = process_qname(":local".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from(":local"));
+
+ let qname = process_qname("::local".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from("::local"));
+
+ let qname = process_qname("a::local".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from("a::local"));
+
+ let qname = process_qname("fake::".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from("fake::"));
+
+ let qname = process_qname(":::".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from(":::"));
+
+ let qname = process_qname(":a:b:".to_tendril());
+ assert_eq!(qname.prefix, None);
+ assert_eq!(qname.local, LocalName::from(":a:b:"));
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/tokenizer/qname.rs b/vendor/xml5ever-0.39.0/src/tokenizer/qname.rs
new file mode 100644
index 000000000..886dafc2d
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tokenizer/qname.rs
@@ -0,0 +1,84 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+enum QualNameState {
+ BeforeName,
+ InName,
+ AfterColon,
+}
+
+pub struct QualNameTokenizer<'a> {
+ state: QualNameState,
+ slice: &'a [u8],
+ valid_index: Option<u32>,
+ curr_ind: usize,
+}
+
+impl QualNameTokenizer<'_> {
+ pub fn new(tag: &[u8]) -> QualNameTokenizer<'_> {
+ QualNameTokenizer {
+ state: QualNameState::BeforeName,
+ slice: tag,
+ valid_index: None,
+ curr_ind: 0,
+ }
+ }
+
+ pub fn run(&mut self) -> Option<u32> {
+ if !self.slice.is_empty() {
+ loop {
+ if !self.step() {
+ break;
+ }
+ }
+ }
+ self.valid_index
+ }
+
+ fn incr(&mut self) -> bool {
+ if self.curr_ind + 1 < self.slice.len() {
+ self.curr_ind += 1;
+ return true;
+ }
+ false
+ }
+
+ fn step(&mut self) -> bool {
+ match self.state {
+ QualNameState::BeforeName => self.do_before_name(),
+ QualNameState::InName => self.do_in_name(),
+ QualNameState::AfterColon => self.do_after_colon(),
+ }
+ }
+
+ fn do_before_name(&mut self) -> bool {
+ if self.slice[self.curr_ind] == b':' {
+ false
+ } else {
+ self.state = QualNameState::InName;
+ self.incr()
+ }
+ }
+
+ fn do_in_name(&mut self) -> bool {
+ if self.slice[self.curr_ind] == b':' && self.curr_ind + 1 < self.slice.len() {
+ self.valid_index = Some(self.curr_ind as u32);
+ self.state = QualNameState::AfterColon;
+ }
+ self.incr()
+ }
+
+ fn do_after_colon(&mut self) -> bool {
+ if self.slice[self.curr_ind] == b':' {
+ self.valid_index = None;
+ return false;
+ }
+ self.incr()
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/tokenizer/states.rs b/vendor/xml5ever-0.39.0/src/tokenizer/states.rs
new file mode 100644
index 000000000..05dc9903f
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tokenizer/states.rs
@@ -0,0 +1,167 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Tokenizer states.
+
+/// Specifies either the public or system identifier from a [Document Type Declaration] (DTD).
+///
+/// [Document Type Declaration]: https://en.wikipedia.org/wiki/Document_type_declaration
+#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)]
+pub enum DoctypeKind {
+ /// The public identifier.
+ Public,
+ /// The system identifier.
+ System,
+}
+
+/// Specifies the different states a XML tokenizer will assume during parsing.
+#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)]
+pub enum XmlState {
+ /// The initial state of the parser.
+ ///
+ /// It is equivalent to the [`Data`](https://html.spec.whatwg.org/#data-state) state of the html parser,
+ /// except null codepoints do not cause errors.
+ Data,
+ /// Indicates that the parser has found a `<` character and will try to parse a tag.
+ TagState,
+ /// Indicates that the parser has consumed the `/` of a closing tag, like `</foo>`.
+ EndTagState,
+ /// Indicates that the parser is currently parsing the name of a closing tag, like the `foo` of `</foo>`.
+ EndTagName,
+ /// Indicates that the parser has finished parsing the name of a closing tag and expects a `>` to follow.
+ EndTagNameAfter,
+ /// Indicates that the parser has started parsing a [processing instruction] (PI).
+ ///
+ /// This state is reached after the initial `?` character has been consumed.
+ ///
+ /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction
+ Pi,
+ /// Indicates that the parser is currently parsing the target of a [processing instruction].
+ ///
+ /// For example, the target of `<?xml-stylesheet type="text/xsl" href="style.xsl"?>` is `xml-stylesheet`.
+ ///
+ /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction
+ PiTarget,
+ /// Indicates that the parser has finished parsing the target of a [processing instruction].
+ ///
+ /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction
+ PiTargetAfter,
+ /// Indicates that the parser is currently parsing the data of a [processing instruction].
+ ///
+ /// The "data" refers to everything between the target and the closing `?` character.
+ ///
+ /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction
+ PiData,
+ /// Indicates that the parser has parsed the closing `?` of a [processing instruction].
+ ///
+ /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction
+ PiAfter,
+ /// Indicates that the parser has parsed the initial `!` of a markup declaration.
+ ///
+ /// Examples of such declarations include `<!ENTITY chap1 SYSTEM "chap1.xml">` or `<!-- Comment -->`.
+ MarkupDecl,
+ /// Indicates that the parser has parsed the start of a comment (`<!--`).
+ CommentStart,
+ /// Indicates that the parser has parsed the start of a comment and a `-` directly after it.
+ CommentStartDash,
+ /// Indicates that the parser is currently parsing the data within a comment.
+ Comment,
+ /// Indicates that the parser has parsed a `<` character within a comment.
+ CommentLessThan,
+ /// Indicates that the parser has parsed `<!` within a comment.
+ CommentLessThanBang,
+ /// Indicates that the parser has parsed `<!-` within a comment.
+ CommentLessThanBangDash,
+ /// Indicates that the parser has parsed `<!--` within a comment.
+ CommentLessThanBangDashDash,
+ /// Indicates that the parser has parsed two `-` characters within a comment which may or may not
+ /// be the beginning of the comment end (`-->`).
+ CommentEnd,
+ /// Indicates that the parser has parsed a `-` character within a comment which may or may not
+ /// be the beginning of the comment end (`-->`).
+ CommentEndDash,
+ /// Indicates that the parser has parsed `--!` within a comment which may or may not be part of the
+ /// end of the comment. Comments in XML can be closed with `--!>`.
+ CommentEndBang,
+ /// Indicates that the parser has parsed the beginning of a CDATA section (`<![CDATA[`).
+ Cdata,
+ /// Indicates that the parser has parsed a `]` character within a CDATA section, which may be part of
+ /// the end of the section (`]]>`).
+ CdataBracket,
+ /// Indicates that the parser has parsed two `]` characters within a CDATA section, which may be part of
+ /// the end of the section (`]]>`).
+ CdataEnd,
+ /// Indicates that the parser is currently parsing the name of a tag, such as `foo` in `<foo>`.
+ TagName,
+ /// Indicates that the parser has parsed the `/` of a self-closing tag, such as `<foo/>`.
+ TagEmpty,
+ /// Indicates that the parser has finished parsing the name of a tag and is now expecting either attributes or
+ /// a `>`.
+ TagAttrNameBefore,
+ /// Indicates that the parser is currently parsing the name of an attribute within a tag, such as
+ /// `bar` in `<foo bar=baz>`.
+ TagAttrName,
+ /// Indicates that the parser has finished parsing the name of an attribute.
+ TagAttrNameAfter,
+ /// Indicates that the parser is about to parse the value of an attribute.
+ TagAttrValueBefore,
+ /// Indicates that the parser is currently parsing the value of an attribute, such as `baz` in
+ /// `<foo bar=baz>`.
+ ///
+ /// Includes information about how the value is quoted, because the quotes before and after the attribute
+ /// value need to match.
+ TagAttrValue(AttrValueKind),
+ /// Indicates that the parser has parsed the beginning of a document type definition (`<!DOCTYPE`).
+ Doctype,
+ /// Indicates that the parser expects to parse the name of the document type definition next.
+ BeforeDoctypeName,
+ /// Indicates that the parser is currently parsing the name of a document type definition, such as
+ /// `html` in `<!DOCTYPE html>`.
+ DoctypeName,
+ /// Indicates that the parser has finished parsing the name of the document type definition and now optionally
+ /// expects either a public or a system identifier.
+ AfterDoctypeName,
+ /// Indicates that the parser has parsed a keyword for either a public or system identifier (`PUBLIC` or `SYSTEM`).
+ AfterDoctypeKeyword(DoctypeKind),
+ /// Indicates that the parser is about to parse the value of a public or system identifier within
+ /// a document type definition, such as `foo` in
+ /// `<!DOCTYPE html PUBLIC "foo" "bar">`.
+ BeforeDoctypeIdentifier(DoctypeKind),
+ /// Indicates that the parser is currently parsing the value of a public or system identifier
+ /// that is surrounded by double quotes , such as `foo` in
+ /// `<!DOCTYPE html PUBLIC "foo" "bar">`.
+ DoctypeIdentifierDoubleQuoted(DoctypeKind),
+ /// Indicates that the parser is currently parsing the value of a public or system identifier
+ /// that is surrounded by single quotes , such as `foo` in
+ /// `<!DOCTYPE html PUBLIC 'foo' 'bar'>`.
+ DoctypeIdentifierSingleQuoted(DoctypeKind),
+ /// Indicates that the parser has finished parsing either a public or system identifier within a
+ /// document type definition.
+ AfterDoctypeIdentifier(DoctypeKind),
+ /// Indicates that the parser has finished parsing a public identifier and now expects
+ /// a system identifier.
+ BetweenDoctypePublicAndSystemIdentifiers,
+ /// Indicates that the parser is currently parsing an ill-formed document type defintion, such as
+ /// `<!DOCTYPE html what-is-this>`.
+ BogusDoctype,
+ /// Indicates that the parser is currently parsing an ill-formed comment, such as
+ /// `<? this is not what a comment should look like! >`.
+ BogusComment,
+}
+
+/// Specifies how an attribute value is quoted, if at all.
+#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)]
+pub enum AttrValueKind {
+ /// A attribute value that is not surrounded by quotes, like `bar` in `foo=bar`.
+ Unquoted,
+ /// A attribute value that is not surrounded by quotes, like `bar` in `foo='bar'`.
+ SingleQuoted,
+ /// A attribute value that is not surrounded by quotes, like `bar` in `foo="bar"`.
+ DoubleQuoted,
+}
diff --git a/vendor/xml5ever-0.39.0/src/tree_builder/mod.rs b/vendor/xml5ever-0.39.0/src/tree_builder/mod.rs
new file mode 100644
index 000000000..381ba8e0c
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tree_builder/mod.rs
@@ -0,0 +1,789 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+mod types;
+
+use log::{debug, warn};
+use markup5ever::{local_name, namespace_prefix, ns};
+use std::borrow::Cow;
+use std::borrow::Cow::Borrowed;
+use std::cell::{Cell, Ref, RefCell};
+use std::collections::btree_map::Iter;
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::fmt::{Debug, Error, Formatter};
+use std::mem;
+
+pub use self::interface::{ElemName, NodeOrText, Tracer, TreeSink};
+use self::types::*;
+use crate::interface::{self, create_element, AppendNode, Attribute, QualName};
+use crate::interface::{AppendText, ExpandedName};
+use crate::tokenizer::{self, EndTag, ProcessResult, StartTag, Tag, TokenSink};
+use crate::tokenizer::{Doctype, EmptyTag, Pi, ShortTag};
+use crate::{LocalName, Namespace, Prefix};
+
+use crate::tendril::{StrTendril, Tendril};
+
+/// XML-specific tree sink operations that are not part of markup5ever's
+/// HTML-oriented [`TreeSink`] interface.
+pub trait XmlTreeSink: TreeSink {
+ /// Create a CDATA section node.
+ fn create_cdata(&self, text: StrTendril) -> Self::Handle;
+}
+
+static XML_URI: &str = "http://www.w3.org/XML/1998/namespace";
+static XMLNS_URI: &str = "http://www.w3.org/2000/xmlns/";
+
+type InsResult = Result<(), Cow<'static, str>>;
+
+#[derive(Debug)]
+struct NamespaceMapStack(Vec<NamespaceMap>);
+
+impl NamespaceMapStack {
+ fn new() -> NamespaceMapStack {
+ NamespaceMapStack(vec![NamespaceMap::default()])
+ }
+
+ fn push(&mut self, map: NamespaceMap) {
+ self.0.push(map);
+ }
+
+ fn pop(&mut self) {
+ self.0.pop();
+ }
+}
+
+pub(crate) struct NamespaceMap {
+ // Map that maps prefixes to URI.
+ //
+ // Key denotes namespace prefix, and value denotes
+ // URI it maps to.
+ //
+ // If value of value is None, that means the namespace
+ // denoted by key has been undeclared.
+ scope: BTreeMap<Option<Prefix>, Option<Namespace>>,
+}
+
+impl Debug for NamespaceMap {
+ fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+ write!(f, "\nNamespaceMap[")?;
+ for (key, value) in &self.scope {
+ writeln!(f, " {key:?} : {value:?}")?;
+ }
+ write!(f, "]")
+ }
+}
+
+impl NamespaceMap {
+ // Returns an empty namespace.
+ pub(crate) fn empty() -> NamespaceMap {
+ NamespaceMap {
+ scope: BTreeMap::new(),
+ }
+ }
+
+ fn default() -> NamespaceMap {
+ NamespaceMap {
+ scope: {
+ let mut map = BTreeMap::new();
+ map.insert(None, None);
+ map.insert(Some(namespace_prefix!("xml")), Some(ns!(xml)));
+ map.insert(Some(namespace_prefix!("xmlns")), Some(ns!(xmlns)));
+ map
+ },
+ }
+ }
+
+ pub(crate) fn get(&self, prefix: &Option<Prefix>) -> Option<&Option<Namespace>> {
+ self.scope.get(prefix)
+ }
+
+ pub(crate) fn get_scope_iter(&self) -> Iter<'_, Option<Prefix>, Option<Namespace>> {
+ self.scope.iter()
+ }
+
+ pub(crate) fn insert(&mut self, name: &QualName) {
+ let prefix = name.prefix.as_ref().cloned();
+ let namespace = Some(Namespace::from(&*name.ns));
+ self.scope.insert(prefix, namespace);
+ }
+
+ fn insert_ns(&mut self, attr: &Attribute) -> InsResult {
+ if &*attr.value == XMLNS_URI {
+ return Err(Borrowed("Can't declare XMLNS URI"));
+ };
+
+ let opt_uri = if attr.value.is_empty() {
+ None
+ } else {
+ Some(Namespace::from(&*attr.value))
+ };
+
+ let result = match (&attr.name.prefix, &*attr.name.local) {
+ (&Some(namespace_prefix!("xmlns")), "xml") => {
+ if &*attr.value != XML_URI {
+ Err(Borrowed("XML namespace can't be redeclared"))
+ } else {
+ Ok(())
+ }
+ },
+
+ (&Some(namespace_prefix!("xmlns")), "xmlns") => {
+ Err(Borrowed("XMLNS namespaces can't be changed"))
+ },
+
+ (&Some(namespace_prefix!("xmlns")), _) | (&None, "xmlns") => {
+ // We can have two cases of properly defined xmlns
+ // First with default namespace e.g.
+ //
+ // <a xmlns = "www.uri.org" />
+ let ns_prefix = if &*attr.name.local == "xmlns" {
+ None
+
+ // Second is with named namespace e.g.
+ //
+ // <a xmlns:a = "www.uri.org" />
+ } else {
+ Some(Prefix::from(&*attr.name.local))
+ };
+
+ if opt_uri.is_some() && self.scope.contains_key(&ns_prefix) {
+ Err(Borrowed("Namespace already defined"))
+ } else {
+ self.scope.insert(ns_prefix, opt_uri);
+ Ok(())
+ }
+ },
+
+ (_, _) => Err(Borrowed("Invalid namespace declaration.")),
+ };
+ result
+ }
+}
+
+/// Tree builder options, with an impl for Default.
+#[derive(Copy, Clone, Default)]
+pub struct XmlTreeBuilderOpts {}
+
+/// The XML tree builder.
+pub struct XmlTreeBuilder<Handle, Sink> {
+ /// Configuration options for XmlTreeBuilder
+ _opts: XmlTreeBuilderOpts,
+
+ /// Consumer of tree modifications.
+ pub sink: Sink,
+
+ /// The document node, which is created by the sink.
+ doc_handle: Handle,
+
+ /// Stack of open elements, most recently added at end.
+ open_elems: RefCell<Vec<Handle>>,
+
+ /// Current element pointer.
+ curr_elem: RefCell<Option<Handle>>,
+
+ /// Stack of namespace identifiers and namespaces.
+ namespace_stack: RefCell<NamespaceMapStack>,
+
+ /// Current namespace identifier
+ current_namespace: RefCell<NamespaceMap>,
+
+ /// Current tree builder phase.
+ phase: Cell<XmlPhase>,
+}
+impl<Handle, Sink> XmlTreeBuilder<Handle, Sink>
+where
+ Handle: Clone,
+ Sink: XmlTreeSink<Handle = Handle>,
+{
+ /// Create a new tree builder which sends tree modifications to a particular `TreeSink`.
+ ///
+ /// The tree builder is also a `TokenSink`.
+ pub fn new(sink: Sink, opts: XmlTreeBuilderOpts) -> XmlTreeBuilder<Handle, Sink> {
+ let doc_handle = sink.get_document();
+ XmlTreeBuilder {
+ _opts: opts,
+ sink,
+ doc_handle,
+ open_elems: RefCell::new(vec![]),
+ curr_elem: RefCell::new(None),
+ namespace_stack: RefCell::new(NamespaceMapStack::new()),
+ current_namespace: RefCell::new(NamespaceMap::empty()),
+ phase: Cell::new(XmlPhase::Start),
+ }
+ }
+
+ /// Call the `Tracer`'s `trace_handle` method on every `Handle` in the tree builder's
+ /// internal state. This is intended to support garbage-collected DOMs.
+ pub fn trace_handles(&self, tracer: &dyn Tracer<Handle = Handle>) {
+ tracer.trace_handle(&self.doc_handle);
+ for e in self.open_elems.borrow().iter() {
+ tracer.trace_handle(e);
+ }
+ if let Some(h) = self.curr_elem.borrow().as_ref() {
+ tracer.trace_handle(h);
+ }
+ }
+
+ // Debug helper
+ #[cfg(not(for_c))]
+ #[allow(dead_code)]
+ fn dump_state(&self, label: String) {
+ debug!("dump_state on {label}");
+ debug!(" open_elems:");
+ for node in self.open_elems.borrow().iter() {
+ debug!(" {:?}", self.sink.elem_name(node));
+ }
+ debug!("");
+ }
+
+ #[cfg(for_c)]
+ fn debug_step(&self, _mode: XmlPhase, _token: &Token) {}
+
+ #[cfg(not(for_c))]
+ fn debug_step(&self, mode: XmlPhase, token: &Token) {
+ debug!(
+ "processing {:?} in insertion mode {:?}",
+ format!("{:?}", token),
+ mode
+ );
+ }
+
+ fn declare_ns(&self, attr: &mut Attribute) {
+ if let Err(msg) = self.current_namespace.borrow_mut().insert_ns(attr) {
+ self.sink.parse_error(msg);
+ } else {
+ attr.name.ns = ns!(xmlns);
+ }
+ }
+
+ fn find_uri(&self, prefix: &Option<Prefix>) -> Result<Option<Namespace>, Cow<'static, str>> {
+ let mut uri = Err(Borrowed("No appropriate namespace found"));
+
+ let current_namespace = self.current_namespace.borrow();
+ for ns in self
+ .namespace_stack
+ .borrow()
+ .0
+ .iter()
+ .chain(Some(&*current_namespace))
+ .rev()
+ {
+ if let Some(el) = ns.get(prefix) {
+ uri = Ok(el.clone());
+ break;
+ }
+ }
+ uri
+ }
+
+ fn bind_qname(&self, name: &mut QualName) {
+ match self.find_uri(&name.prefix) {
+ Ok(uri) => {
+ let ns_uri = match uri {
+ Some(e) => e,
+ None => ns!(),
+ };
+ name.ns = ns_uri;
+ },
+ Err(msg) => {
+ self.sink.parse_error(msg);
+ },
+ }
+ }
+
+ // This method takes in name qualified name and binds it to the
+ // existing namespace context.
+ //
+ // Returns false if the attribute is a duplicate, returns true otherwise.
+ fn bind_attr_qname(
+ &self,
+ present_attrs: &mut HashSet<(Namespace, LocalName)>,
+ name: &mut QualName,
+ ) -> bool {
+ // Attributes don't have default namespace
+ let mut not_duplicate = true;
+
+ if name.prefix.is_some() {
+ self.bind_qname(name);
+ not_duplicate = Self::check_duplicate_attr(present_attrs, name);
+ }
+ not_duplicate
+ }
+
+ fn check_duplicate_attr(
+ present_attrs: &mut HashSet<(Namespace, LocalName)>,
+ name: &QualName,
+ ) -> bool {
+ let pair = (name.ns.clone(), name.local.clone());
+
+ if present_attrs.contains(&pair) {
+ return false;
+ }
+ present_attrs.insert(pair);
+ true
+ }
+
+ fn process_namespaces(&self, tag: &mut Tag) {
+ // List of already present namespace local name attribute pairs.
+ let mut present_attrs: HashSet<(Namespace, LocalName)> = Default::default();
+
+ let mut new_attr = vec![];
+ // First we extract all namespace declarations
+ for attr in tag.attrs.iter_mut().filter(|attr| {
+ attr.name.prefix == Some(namespace_prefix!("xmlns"))
+ || attr.name.local == local_name!("xmlns")
+ }) {
+ self.declare_ns(attr);
+ }
+
+ // Then we bind those namespace declarations to attributes
+ for attr in tag.attrs.iter_mut().filter(|attr| {
+ attr.name.prefix != Some(namespace_prefix!("xmlns"))
+ && attr.name.local != local_name!("xmlns")
+ }) {
+ if self.bind_attr_qname(&mut present_attrs, &mut attr.name) {
+ new_attr.push(attr.clone());
+ }
+ }
+ tag.attrs = new_attr;
+
+ // Then we bind the tags namespace.
+ self.bind_qname(&mut tag.name);
+
+ // Finally, we dump current namespace if its unneeded.
+ let x = mem::replace(
+ &mut *self.current_namespace.borrow_mut(),
+ NamespaceMap::empty(),
+ );
+
+ // Only start tag doesn't dump current namespace. However, <script /> is treated
+ // differently than every other empty tag, so it needs to retain the current
+ // namespace as well.
+ if tag.kind == StartTag || (tag.kind == EmptyTag && tag.name.local == local_name!("script"))
+ {
+ self.namespace_stack.borrow_mut().push(x);
+ }
+ }
+
+ fn process_to_completion(
+ &self,
+ mut token: Token,
+ ) -> ProcessResult<<Self as TokenSink>::Handle> {
+ // Queue of additional tokens yet to be processed.
+ // This stays empty in the common case where we don't split whitespace.
+ let mut more_tokens = VecDeque::new();
+
+ loop {
+ let phase = self.phase.get();
+
+ #[allow(clippy::unused_unit)]
+ match self.step(phase, token) {
+ XmlProcessResult::Done => {
+ let Some(popped_token) = more_tokens.pop_front() else {
+ return ProcessResult::Continue;
+ };
+ token = popped_token;
+ },
+ XmlProcessResult::Reprocess(m, t) => {
+ self.phase.set(m);
+ token = t;
+ },
+ XmlProcessResult::Script(node) => {
+ assert!(more_tokens.is_empty());
+ return ProcessResult::Script(node);
+ },
+ }
+ }
+ }
+}
+
+impl<Handle, Sink> TokenSink for XmlTreeBuilder<Handle, Sink>
+where
+ Handle: Clone,
+ Sink: XmlTreeSink<Handle = Handle>,
+{
+ type Handle = Handle;
+
+ fn process_token(&self, token: tokenizer::Token) -> ProcessResult<Self::Handle> {
+ // Handle `ParseError` and `DoctypeToken`; convert everything else to the local `Token` type.
+ let token = match token {
+ tokenizer::Token::ParseError(e) => {
+ self.sink.parse_error(e);
+ return ProcessResult::Done;
+ },
+
+ tokenizer::Token::Doctype(d) => Token::Doctype(d),
+ tokenizer::Token::ProcessingInstruction(instruction) => Token::Pi(instruction),
+ tokenizer::Token::Tag(x) => Token::Tag(x),
+ tokenizer::Token::Comment(x) => Token::Comment(x),
+ tokenizer::Token::NullCharacter => Token::NullCharacter,
+ tokenizer::Token::EndOfFile => Token::Eof,
+ tokenizer::Token::Characters(x) => Token::Characters(x),
+ tokenizer::Token::Cdata(x) => Token::Cdata(x),
+ };
+
+ self.process_to_completion(token)
+ }
+
+ fn end(&self) {
+ for node in self.open_elems.borrow_mut().drain(..).rev() {
+ self.sink.pop(&node);
+ }
+ }
+}
+
+fn current_node<Handle>(open_elems: &[Handle]) -> &Handle {
+ open_elems.last().expect("no current element")
+}
+
+impl<Handle, Sink> XmlTreeBuilder<Handle, Sink>
+where
+ Handle: Clone,
+ Sink: XmlTreeSink<Handle = Handle>,
+{
+ fn current_node(&self) -> Ref<'_, Handle> {
+ Ref::map(self.open_elems.borrow(), |elems| {
+ elems.last().expect("no current element")
+ })
+ }
+
+ fn insert_appropriately(&self, child: NodeOrText<Handle>) {
+ let open_elems = self.open_elems.borrow();
+ let target = current_node(&open_elems);
+ self.sink.append(target, child);
+ }
+
+ fn insert_tag(&self, tag: Tag) -> XmlProcessResult<Handle> {
+ let child = create_element(&self.sink, tag.name, tag.attrs);
+ self.insert_appropriately(AppendNode(child.clone()));
+ self.add_to_open_elems(child)
+ }
+
+ fn append_tag(&self, tag: Tag) -> XmlProcessResult<Handle> {
+ let child = create_element(&self.sink, tag.name, tag.attrs);
+ self.insert_appropriately(AppendNode(child.clone()));
+ self.sink.pop(&child);
+ XmlProcessResult::Done
+ }
+
+ fn append_tag_to_doc(&self, tag: Tag) -> Handle {
+ let child = create_element(&self.sink, tag.name, tag.attrs);
+
+ self.sink
+ .append(&self.doc_handle, AppendNode(child.clone()));
+ child
+ }
+
+ fn add_to_open_elems(&self, el: Handle) -> XmlProcessResult<Handle> {
+ self.open_elems.borrow_mut().push(el);
+
+ XmlProcessResult::Done
+ }
+
+ fn append_comment_to_doc(&self, text: StrTendril) -> XmlProcessResult<Handle> {
+ let comment = self.sink.create_comment(text);
+ self.sink.append(&self.doc_handle, AppendNode(comment));
+ XmlProcessResult::Done
+ }
+
+ fn append_comment_to_tag(&self, text: StrTendril) -> XmlProcessResult<Handle> {
+ let open_elems = self.open_elems.borrow();
+ let target = current_node(&open_elems);
+ let comment = self.sink.create_comment(text);
+ self.sink.append(target, AppendNode(comment));
+ XmlProcessResult::Done
+ }
+
+ fn append_doctype_to_doc(&self, doctype: Doctype) -> XmlProcessResult<Handle> {
+ fn get_tendril(opt: Option<StrTendril>) -> StrTendril {
+ match opt {
+ Some(expr) => expr,
+ None => Tendril::new(),
+ }
+ }
+ self.sink.append_doctype_to_document(
+ get_tendril(doctype.name),
+ get_tendril(doctype.public_id),
+ get_tendril(doctype.system_id),
+ );
+ XmlProcessResult::Done
+ }
+
+ fn append_pi_to_doc(&self, pi: Pi) -> XmlProcessResult<Handle> {
+ let pi = self.sink.create_pi(pi.target, pi.data);
+ self.sink.append(&self.doc_handle, AppendNode(pi));
+ XmlProcessResult::Done
+ }
+
+ fn append_pi_to_tag(&self, pi: Pi) -> XmlProcessResult<Handle> {
+ let open_elems = self.open_elems.borrow();
+ let target = current_node(&open_elems);
+ let pi = self.sink.create_pi(pi.target, pi.data);
+ self.sink.append(target, AppendNode(pi));
+ XmlProcessResult::Done
+ }
+
+ fn append_text(&self, chars: StrTendril) -> XmlProcessResult<Handle> {
+ self.insert_appropriately(AppendText(chars));
+ XmlProcessResult::Done
+ }
+
+ fn append_cdata(&self, text: StrTendril) -> XmlProcessResult<Handle> {
+ let cdata = self.sink.create_cdata(text);
+ self.insert_appropriately(AppendNode(cdata));
+ XmlProcessResult::Done
+ }
+
+ fn tag_in_open_elems(&self, tag: &Tag) -> bool {
+ self.open_elems
+ .borrow()
+ .iter()
+ .any(|a| self.sink.elem_name(a).expanded() == tag.name.expanded())
+ }
+
+ // Pop elements until an element from the set has been popped.
+ fn pop_until<P>(&self, pred: P)
+ where
+ P: Fn(ExpandedName) -> bool,
+ {
+ loop {
+ if self.current_node_in(&pred) {
+ break;
+ }
+ self.pop();
+ }
+ }
+
+ fn current_node_in<TagSet>(&self, set: TagSet) -> bool
+ where
+ TagSet: Fn(ExpandedName) -> bool,
+ {
+ // FIXME: take namespace into consideration:
+ set(self.sink.elem_name(&self.current_node()).expanded())
+ }
+
+ fn close_tag(&self, tag: Tag) -> XmlProcessResult<Handle> {
+ debug!(
+ "Close tag: current_node.name {:?} \n Current tag {:?}",
+ self.sink.elem_name(&self.current_node()),
+ &tag.name
+ );
+
+ if *self.sink.elem_name(&self.current_node()).local_name() != tag.name.local {
+ self.sink
+ .parse_error(Borrowed("Current node doesn't match tag"));
+ }
+
+ let is_closed = self.tag_in_open_elems(&tag);
+
+ if is_closed {
+ self.pop_until(|p| p == tag.name.expanded());
+ self.pop();
+ }
+
+ XmlProcessResult::Done
+ }
+
+ fn no_open_elems(&self) -> bool {
+ self.open_elems.borrow().is_empty()
+ }
+
+ fn pop(&self) -> Handle {
+ self.namespace_stack.borrow_mut().pop();
+ let node = self
+ .open_elems
+ .borrow_mut()
+ .pop()
+ .expect("no current element");
+ self.sink.pop(&node);
+ node
+ }
+
+ fn stop_parsing(&self) -> XmlProcessResult<Handle> {
+ warn!("stop_parsing for XML5 not implemented, full speed ahead!");
+ XmlProcessResult::Done
+ }
+}
+
+fn any_not_whitespace(x: &StrTendril) -> bool {
+ !x.bytes()
+ .all(|b| matches!(b, b'\t' | b'\r' | b'\n' | b'\x0C' | b' '))
+}
+
+impl<Handle, Sink> XmlTreeBuilder<Handle, Sink>
+where
+ Handle: Clone,
+ Sink: XmlTreeSink<Handle = Handle>,
+{
+ fn step(&self, mode: XmlPhase, token: Token) -> XmlProcessResult<<Self as TokenSink>::Handle> {
+ self.debug_step(mode, &token);
+
+ match mode {
+ XmlPhase::Start => match token {
+ Token::Tag(Tag {
+ kind: StartTag,
+ name,
+ attrs,
+ }) => {
+ let tag = {
+ let mut tag = Tag {
+ kind: StartTag,
+ name,
+ attrs,
+ };
+ self.process_namespaces(&mut tag);
+ tag
+ };
+ self.phase.set(XmlPhase::Main);
+ let handle = self.append_tag_to_doc(tag);
+ self.add_to_open_elems(handle)
+ },
+ Token::Tag(Tag {
+ kind: EmptyTag,
+ name,
+ attrs,
+ }) => {
+ let tag = {
+ let mut tag = Tag {
+ kind: EmptyTag,
+ name,
+ attrs,
+ };
+ self.process_namespaces(&mut tag);
+ tag
+ };
+ self.phase.set(XmlPhase::End);
+ let handle = self.append_tag_to_doc(tag);
+ self.sink.pop(&handle);
+ XmlProcessResult::Done
+ },
+ Token::Comment(comment) => self.append_comment_to_doc(comment),
+ Token::Pi(pi) => self.append_pi_to_doc(pi),
+ Token::Characters(ref chars) if !any_not_whitespace(chars) => {
+ XmlProcessResult::Done
+ },
+ Token::Eof => {
+ self.sink
+ .parse_error(Borrowed("Unexpected EOF in start phase"));
+ XmlProcessResult::Reprocess(XmlPhase::End, Token::Eof)
+ },
+ Token::Doctype(d) => {
+ self.append_doctype_to_doc(d);
+ XmlProcessResult::Done
+ },
+ _ => {
+ self.sink
+ .parse_error(Borrowed("Unexpected element in start phase"));
+ XmlProcessResult::Done
+ },
+ },
+ XmlPhase::Main => match token {
+ Token::Characters(chs) => self.append_text(chs),
+ Token::Cdata(text) => self.append_cdata(text),
+ Token::Tag(Tag {
+ kind: StartTag,
+ name,
+ attrs,
+ }) => {
+ let tag = {
+ let mut tag = Tag {
+ kind: StartTag,
+ name,
+ attrs,
+ };
+ self.process_namespaces(&mut tag);
+ tag
+ };
+ self.insert_tag(tag)
+ },
+ Token::Tag(Tag {
+ kind: EmptyTag,
+ name,
+ attrs,
+ }) => {
+ let tag = {
+ let mut tag = Tag {
+ kind: EmptyTag,
+ name,
+ attrs,
+ };
+ self.process_namespaces(&mut tag);
+ tag
+ };
+ if tag.name.local == local_name!("script") {
+ self.insert_tag(tag.clone());
+ let script = current_node(&self.open_elems.borrow()).clone();
+ self.close_tag(tag);
+ XmlProcessResult::Script(script)
+ } else {
+ self.append_tag(tag)
+ }
+ },
+ Token::Tag(Tag {
+ kind: EndTag,
+ name,
+ attrs,
+ }) => {
+ let tag = {
+ let mut tag = Tag {
+ kind: EndTag,
+ name,
+ attrs,
+ };
+ self.process_namespaces(&mut tag);
+ tag
+ };
+ if tag.name.local == local_name!("script") {
+ let script = current_node(&self.open_elems.borrow()).clone();
+ self.close_tag(tag);
+ if self.no_open_elems() {
+ self.phase.set(XmlPhase::End);
+ }
+ return XmlProcessResult::Script(script);
+ }
+ let retval = self.close_tag(tag);
+ if self.no_open_elems() {
+ self.phase.set(XmlPhase::End);
+ }
+ retval
+ },
+ Token::Tag(Tag { kind: ShortTag, .. }) => {
+ self.pop();
+ if self.no_open_elems() {
+ self.phase.set(XmlPhase::End);
+ }
+ XmlProcessResult::Done
+ },
+ Token::Comment(comment) => self.append_comment_to_tag(comment),
+ Token::Pi(pi) => self.append_pi_to_tag(pi),
+ Token::Eof | Token::NullCharacter => {
+ XmlProcessResult::Reprocess(XmlPhase::End, Token::Eof)
+ },
+ Token::Doctype(_) => {
+ self.sink
+ .parse_error(Borrowed("Unexpected element in main phase"));
+ XmlProcessResult::Done
+ },
+ },
+ XmlPhase::End => match token {
+ Token::Comment(comment) => self.append_comment_to_doc(comment),
+ Token::Pi(pi) => self.append_pi_to_doc(pi),
+ Token::Characters(ref chars) if !any_not_whitespace(chars) => {
+ XmlProcessResult::Done
+ },
+ Token::Eof => self.stop_parsing(),
+ _ => {
+ self.sink
+ .parse_error(Borrowed("Unexpected element in end phase"));
+ XmlProcessResult::Done
+ },
+ },
+ }
+ }
+}
diff --git a/vendor/xml5ever-0.39.0/src/tree_builder/types.rs b/vendor/xml5ever-0.39.0/src/tree_builder/types.rs
new file mode 100644
index 000000000..dde7a96b9
--- /dev/null
+++ b/vendor/xml5ever-0.39.0/src/tree_builder/types.rs
@@ -0,0 +1,38 @@
+// Copyright 2014-2017 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use crate::tendril::StrTendril;
+use crate::tokenizer::{Doctype, Pi, Tag};
+
+#[derive(PartialEq, Eq, Copy, Clone, Debug)]
+pub enum XmlPhase {
+ Start,
+ Main,
+ End,
+}
+
+/// A subset/refinement of `tokenizer::Token`. Everything else is handled
+/// specially at the beginning of `process_token`.
+#[derive(PartialEq, Eq, Clone, Debug)]
+pub enum Token {
+ Tag(Tag),
+ Doctype(Doctype),
+ Comment(StrTendril),
+ Characters(StrTendril),
+ Cdata(StrTendril),
+ Pi(Pi),
+ NullCharacter,
+ Eof,
+}
+
+pub enum XmlProcessResult<Handle> {
+ Done,
+ Reprocess(XmlPhase, Token),
+ Script(Handle),
+}