Lightpanda: the headless browser designed for AI and automation
feature: Parse xml documents
Karl Seguinbcdd5cc2835e
diff --git a/src/TestHTTPServer.zig b/src/TestHTTPServer.zigindex f0d4481c9..1d920dbe8 100644--- a/src/TestHTTPServer.zig+++ b/src/TestHTTPServer.zig@@ -162,6 +162,10 @@ fn getContentType(file_path: []const u8) []const u8 {return "text/plain";}+ if (std.mem.endsWith(u8, file_path, ".xhtml")) {+ return "application/xhtml+xml";+ }+if (std.mem.endsWith(u8, file_path, ".xml")) {// some wpt tests do thisreturn "text/xml";diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex fc0c00aa6..df54a0845 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1933,6 +1933,19 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {.arena = try self.getArena(.large, "Frame.navigate"),} };},+ .text_xml, .other_xml => {+ // XML ignores <meta charset>. Priority is BOM, header and XML declaration+ // declaration (a BOM overrides both when decoding).+ const label = if (mime.is_default_charset) Mime.xmlDeclarationEncoding(data) else mime.charsetString();+ if (h5e.EncodingInfo.forLabel(label)) |info| {+ self.charset = info.name();+ }+ self._parse_state = .{ .html = .{+ .buffer = .empty,+ .arena = try self.getArena(.large, "Frame.navigate"),+ .xml = true,+ } };+ },.application_json, .text_javascript, .text_css, .text_plain, .text_markdown => {var arr: std.ArrayList(u8) = .empty;try arr.appendSlice(self.page_arena, "<html><head><meta charset=\"utf-8\"></head><body><pre>");@@ -2025,7 +2038,17 @@ fn frameDoneCallback(ctx: *anyopaque) !void {preload.prescan(self, raw_html);- if (std.mem.eql(u8, self.charset, "UTF-8")) {+ if (html.xml) {+ const encoding = h5e.EncodingInfo.forLabel(self.charset) orelse h5e.EncodingInfo.utf8();+ const xml = try h5e.validUtf8(html.arena.allocator(), encoding, raw_html);+ if (Parser.isWellFormedXML(xml)) {+ parser.parseXML(xml);+ } else {+ log.info(.frame, "malformed xml parsed as html", .{ .url = self.url });+ // try to pass it through the HTML parser+ parser.parse(xml);+ }+ } else if (std.mem.eql(u8, self.charset, "UTF-8")) {parser.parse(raw_html);} else {parser.parseWithEncoding(raw_html, self.charset);@@ -3708,6 +3731,7 @@ const ParseState = union(enum) {err: anyerror,html: struct {arena: *lp.Arena,+ xml: bool = false,buffer: std.ArrayList(u8),},text: std.ArrayList(u8),diff --git a/src/browser/Mime.zig b/src/browser/Mime.zigindex c9f6c8422..03367a4c7 100644--- a/src/browser/Mime.zig+++ b/src/browser/Mime.zig@@ -298,6 +298,42 @@ pub fn prescanCharset(html: []const u8) ?[]const u8 {return null;}+/// The encoding named by an XML declaration:+/// <?xml version="1.0" encoding="X"?>+/// optionally at the very start of `xml`+pub fn xmlDeclarationEncoding(xml: []const u8) ?[]const u8 {+ if (std.mem.startsWith(u8, xml, "<?xml") == false) {+ return null;+ }+ const end = std.mem.findPos(u8, xml, 5, "?>") orelse return null;+ const decl = xml[5..end];++ var pos: usize = 0;+ while (std.mem.findPos(u8, decl, pos, "encoding")) |index| {+ pos = index + "encoding".len;+ // Only as an attribute name: "1.0" or "standalone" values can't+ // contain it, but a pseudo-attribute like "xencoding" could.+ if (index == 0 or std.ascii.isWhitespace(decl[index - 1]) == false) {+ continue;+ }+ var rest = std.mem.trimStart(u8, decl[pos..], " \t\r\n");+ if (rest.len == 0 or rest[0] != '=') {+ continue;+ }+ rest = std.mem.trimStart(u8, rest[1..], " \t\r\n");+ if (rest.len == 0 or (rest[0] != '"' and rest[0] != '\'')) {+ return null;+ }+ const close = std.mem.findScalarPos(u8, rest, 1, rest[0]) orelse return null;+ const encoding = rest[1..close];+ if (encoding.len == 0 or encoding.len > 40) {+ return null;+ }+ return encoding;+ }+ return null;+}+fn findAttrValue(attrs: []const u8, name: []const u8) ?[]const u8 {var pos: usize = 0;while (pos < attrs.len) {@@ -1068,6 +1104,29 @@ test "Mime: isXML" {try assert(false, "text/xm");}+test "Mime: xmlDeclarationEncoding" {+ const assert = struct {+ fn assert(expected: ?[]const u8, input: []const u8) !void {+ const actual = Mime.xmlDeclarationEncoding(input);+ if (expected) |e| {+ try testing.expectEqual(e, actual.?);+ } else {+ try testing.expectEqual(null, actual);+ }+ }+ }.assert;+ try assert("ISO-8859-1", "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><html/>");+ try assert("windows-1252", "<?xml version='1.0' encoding = 'windows-1252' standalone='yes'?>");+ try assert("utf-8", "<?xml version=\"1.0\"\n\tencoding=\"utf-8\"?>");+ try assert(null, "<?xml version=\"1.0\"?><html/>");+ try assert(null, "<?xml version=\"1.0\" xencoding=\"utf-8\"?>");+ try assert(null, "<?xml version=\"1.0\" encoding=utf-8?>");+ try assert(null, "<?xml version=\"1.0\" encoding=\"\"?>");+ try assert(null, " <?xml version=\"1.0\" encoding=\"utf-8\"?>");+ try assert(null, "<html><?xml version=\"1.0\" encoding=\"utf-8\"?>");+ try assert(null, "<?xml version=\"1.0\" encoding=\"utf-8\"");+}+test "Mime: sniff" {try testing.expectEqual(null, Mime.sniff(""));try testing.expectEqual(null, Mime.sniff("<htm"));diff --git a/src/browser/parser/Parser.zig b/src/browser/parser/Parser.zigindex 3eb7b84e2..5faf7c536 100644--- a/src/browser/parser/Parser.zig+++ b/src/browser/parser/Parser.zig@@ -313,6 +313,12 @@ pub fn parseXML(self: *Parser, xml: []const u8) void {};}+pub fn isWellFormedXML(xml: []const u8) bool {+ var has_error = false;+ h5e.xml5ever_validate_document(xml.ptr, xml.len, &has_error, validateErrorCallback);+ return has_error == false;+}+pub fn parseFragment(self: *Parser, html: []const u8) void {self.fragment = true;const context_el = self.context orelse self.container.node.is(Element);@@ -478,16 +484,28 @@ fn parseErrorCallback(ctx: *anyopaque, err: h5e.StringSlice) callconv(.c) void {// xml5ever xml error callback indicating a well-formedness violating.fn xmlParseErrorCallback(ctx: *anyopaque, err: h5e.StringSlice) callconv(.c) void {const self: *Parser = @ptrCast(@alignCast(ctx));- const msg = err.slice();+ if (isWellFormednessError(err.slice())) {+ self.xml_error = true;+ }+}++fn validateErrorCallback(ctx: *anyopaque, err: h5e.StringSlice) callconv(.c) void {+ const has_error: *bool = @ptrCast(@alignCast(ctx));+ if (isWellFormednessError(err.slice())) {+ has_error.* = true;+ }+}++fn isWellFormednessError(msg: []const u8) bool {if (std.mem.startsWith(u8, msg, "Bad character") and msg.len > "Bad character".len) {// discouraged by legal code point- return;+ return false;}if (std.mem.startsWith(u8, msg, "Invalid character reference")) {// xml5ever doesn't know about this entity, but it could be valid- return;+ return false;}- self.xml_error = true;+ return true;}fn popCallback(ctx: *anyopaque, node_ref: *anyopaque) callconv(.c) void {@@ -1052,3 +1070,13 @@ fn pollTerminate(self: *Parser) bool {self.terminated = terminated;return terminated;}++const testing = @import("../../testing.zig");+test "Parser: isWellFormedXML" {+ try testing.expectEqual(true, isWellFormedXML("<?xml version=\"1.0\"?><a><b/> &</a>"));+ try testing.expectEqual(true, isWellFormedXML("<!DOCTYPE svg [ <!ENTITY x \"y\"> ]><svg/>"));+ try testing.expectEqual(false, isWellFormedXML("<a><b></a>"));+ try testing.expectEqual(false, isWellFormedXML("<a>unclosed"));+ try testing.expectEqual(false, isWellFormedXML("<a x=1/>"));+ try testing.expectEqual(false, isWellFormedXML("<?xml version=\"2.0\"?><a/>"));+}diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex 421dc79f5..e3e538e87 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -233,6 +233,13 @@ pub extern "c" fn xml5ever_parse_document(allow_declarative_shadow: bool,) void;+pub extern "c" fn xml5ever_validate_document(+ xml: [*c]const u8,+ len: usize,+ ctx: *anyopaque,+ parseErrorCallback: *const fn (ctx: *anyopaque, StringSlice) callconv(.c) void,+) void;+// 0: no reference starts with `name`, 1: `name` is a prefix of one,// 2: `name` is one, its codepoints are in `out` (out[1] == 0 for a single one).pub extern "c" fn html5ever_named_character_reference(name: [*]const u8, len: usize, out: *[2]u32) u8;diff --git a/src/browser/tests/frames/support/feed.xml b/src/browser/tests/frames/support/feed.xmlnew file mode 100644index 000000000..2f85bb78b--- /dev/null+++ b/src/browser/tests/frames/support/feed.xml@@ -0,0 +1,2 @@+<?xml version="1.0" encoding="UTF-8"?>+<rss version="2.0"><channel><title>Feed</title><item><title>First</title></item></channel></rss>diff --git a/src/browser/tests/frames/support/xhtml_latin1.xhtml b/src/browser/tests/frames/support/xhtml_latin1.xhtmlnew file mode 100644index 000000000..e16c8328d--- /dev/null+++ b/src/browser/tests/frames/support/xhtml_latin1.xhtml@@ -0,0 +1,2 @@+<?xml version="1.0" encoding="ISO-8859-1"?>+<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Latin1</title></head><body><p id="p">caf�</p></body></html>diff --git a/src/browser/tests/frames/support/xhtml_malformed.xhtml b/src/browser/tests/frames/support/xhtml_malformed.xhtmlnew file mode 100644index 000000000..b7c9f2686--- /dev/null+++ b/src/browser/tests/frames/support/xhtml_malformed.xhtml@@ -0,0 +1,9 @@+<?xml version="1.0" encoding="UTF-8"?>+<html xmlns="http://www.w3.org/1999/xhtml">+<head><title>Malformed</title></head>+<body>+<p id="p">unclosed <b>bold</p>+<div id="d"/><span id="s"></span>+<script>window.ran = (window.ran || 0) + 1;</script>+</body>+</html>diff --git a/src/browser/tests/frames/support/xhtml_page.xhtml b/src/browser/tests/frames/support/xhtml_page.xhtmlnew file mode 100644index 000000000..59bc1a5c4--- /dev/null+++ b/src/browser/tests/frames/support/xhtml_page.xhtml@@ -0,0 +1,14 @@+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">+<head><title>XHTML page</title></head>+<body>+<div id="root">+ <script>window.sawAfter = document.getElementById('after') !== null;</script>+ <script/>+ <div id="empty"/>+ <p id="after">a b&c</p>+ <svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"><rect id="rect"/></svg>+</div>+</body>+</html>diff --git a/src/browser/tests/frames/xml_documents.html b/src/browser/tests/frames/xml_documents.htmlnew file mode 100644index 000000000..b10fbd180--- /dev/null+++ b/src/browser/tests/frames/xml_documents.html@@ -0,0 +1,84 @@+<!DOCTYPE html>+<script src="../testing.js"></script>++<script>+ function loadFrame(src) {+ return new Promise((resolve) => {+ const f = document.createElement('iframe');+ f.onload = () => resolve(f);+ f.src = src;+ document.documentElement.appendChild(f);+ });+ }+</script>++<script id=xhtml type=module>+ {+ const state = await testing.async();+ const f = await loadFrame('support/xhtml_page.xhtml');+ state.resolve(f);+ await state.done((f) => {+ const doc = f.contentDocument;+ testing.expectEqual('application/xhtml+xml', doc.contentType);+ testing.expectEqual('XHTML page', doc.title);+ testing.expectEqual('http://www.w3.org/1999/xhtml', doc.documentElement.namespaceURI);+ testing.expectEqual(true, doc.body instanceof f.contentWindow.HTMLBodyElement);++ // Scripts run as they're parsed.+ testing.expectEqual(false, f.contentWindow.sawAfter);++ // Self-closing non-void elements are empty, not left open.+ testing.expectEqual(0, doc.getElementById('empty').childNodes.length);+ testing.expectEqual('root', doc.getElementById('after').parentNode.id);++ testing.expectEqual('a\u00a0b&c', doc.getElementById('after').textContent);+ testing.expectEqual('http://www.w3.org/2000/svg', doc.getElementById('rect').namespaceURI);+ });+ }+</script>++<script id=malformed type=module>+ {+ // Not well-formed: falls back to the HTML parser, scripts run once.+ // Chrome fails this on purpose: it stops at the first error and shows+ // the partial tree.+ const state = await testing.async();+ const f = await loadFrame('support/xhtml_malformed.xhtml');+ state.resolve(f);+ await state.done((f) => {+ const doc = f.contentDocument;+ testing.expectEqual('application/xhtml+xml', doc.contentType);+ testing.expectEqual('unclosed bold', doc.getElementById('p').textContent);+ // HTML parsing: <div/> stays open.+ testing.expectEqual('d', doc.getElementById('s').parentNode.id);+ testing.expectEqual(1, f.contentWindow.ran);+ });+ }+</script>++<script id=xml_declaration_encoding type=module>+ {+ const state = await testing.async();+ const f = await loadFrame('support/xhtml_latin1.xhtml');+ state.resolve(f);+ await state.done((f) => {+ const doc = f.contentDocument;+ testing.expectEqual('windows-1252', doc.characterSet);+ testing.expectEqual('caf\u00e9', doc.getElementById('p').textContent);+ });+ }+</script>++<script id=text_xml type=module>+ {+ const state = await testing.async();+ const f = await loadFrame('support/feed.xml');+ state.resolve(f);+ await state.done((f) => {+ const doc = f.contentDocument;+ testing.expectEqual('rss', doc.documentElement.localName);+ testing.expectEqual(null, doc.documentElement.namespaceURI);+ testing.expectEqual('First', doc.querySelector('item > title').textContent);+ });+ }+</script>diff --git a/src/rust/html5ever/lib.rs b/src/rust/html5ever/lib.rsindex 826dd6df6..a42d8edf2 100644--- a/src/rust/html5ever/lib.rs+++ b/src/rust/html5ever/lib.rs@@ -823,6 +823,140 @@ pub extern "C" fn xml5ever_parse_document(.one(&*bytes);}+// Runs xml5ever over `xml` without building a tree, reporting every error+// through `parse_error_callback`, so the caller can decide between the XML+// and HTML parsers before any node is created (or script run).+#[no_mangle]+pub extern "C" fn xml5ever_validate_document(+ xml: *mut c_uchar,+ len: usize,+ ctx: Ref,+ parse_error_callback: ParseErrorCallback,+) -> () {+ if xml.is_null() || len == 0 {+ return ();+ }++ let sink = ValidateSink {+ ctx: ctx,+ parse_error_callback: parse_error_callback,+ names: std::cell::RefCell::new(Vec::new()),+ };++ let bytes = unsafe { std::slice::from_raw_parts(xml, len) };+ let bytes = strip_doctype_internal_subset(bytes);+ let tb = xml5ever::tree_builder::XmlTreeBuilder::new(sink, Default::default());+ let tokenizer = xml5ever::tokenizer::XmlTokenizer::new(+ UnclosedTagSink {+ tb,+ depth: Cell::new(0),+ },+ Default::default(),+ );+ html5ever::tendril::stream::Utf8LossyDecoder::new(XmlDocumentParser {+ tokenizer,+ input_buffer: Default::default(),+ })+ .one(&*bytes);+}++// A TreeSink that only keeps what the tree builder queries back: element+// names, so end tags can be matched. A handle is an index into `names`; the+// document and non-element nodes share index 0.+struct ValidateSink {+ ctx: Ref,+ parse_error_callback: ParseErrorCallback,+ names: std::cell::RefCell<Vec<QualName>>,+}++impl xml5ever::tree_builder::TreeSink for ValidateSink {+ type Handle = usize;+ type Output = ();+ type ElemName<'a> = std::cell::Ref<'a, QualName>;++ fn finish(self) -> () {}++ fn parse_error(&self, err: std::borrow::Cow<'static, str>) {+ unsafe {+ (self.parse_error_callback)(+ self.ctx,+ StringSlice {+ ptr: err.as_ptr(),+ len: err.len(),+ },+ );+ }+ }++ fn get_document(&self) -> usize {+ 0+ }++ fn elem_name<'a>(&'a self, target: &'a usize) -> Self::ElemName<'a> {+ std::cell::Ref::map(self.names.borrow(), |names| &names[*target - 1])+ }++ fn create_element(+ &self,+ name: QualName,+ _attrs: Vec<html5ever::Attribute>,+ _flags: html5ever::interface::ElementFlags,+ ) -> usize {+ let mut names = self.names.borrow_mut();+ names.push(name);+ names.len()+ }++ fn create_comment(&self, _text: StrTendril) -> usize {+ 0+ }++ fn create_pi(&self, _target: StrTendril, _data: StrTendril) -> usize {+ 0+ }++ fn append(&self, _parent: &usize, _child: html5ever::interface::NodeOrText<usize>) {}++ fn append_based_on_parent_node(+ &self,+ _element: &usize,+ _prev_element: &usize,+ _child: html5ever::interface::NodeOrText<usize>,+ ) {+ }++ fn append_doctype_to_document(+ &self,+ _name: StrTendril,+ _public_id: StrTendril,+ _system_id: StrTendril,+ ) {+ }++ fn get_template_contents(&self, target: &usize) -> usize {+ *target+ }++ fn same_node(&self, x: &usize, y: &usize) -> bool {+ x == y+ }++ fn set_quirks_mode(&self, _mode: QuirksMode) {}++ fn append_before_sibling(+ &self,+ _sibling: &usize,+ _new_node: html5ever::interface::NodeOrText<usize>,+ ) {+ }++ fn add_attrs_if_missing(&self, _target: &usize, _attrs: Vec<html5ever::Attribute>) {}++ fn remove_from_parent(&self, _target: &usize) {}++ fn reparent_children(&self, _node: &usize, _new_parent: &usize) {}+}+// xml5ever's tokenizer has no notion of a DOCTYPE internal subset// (`<!DOCTYPE svg [ <!ENTITY ns_svg "..."> ]>`, as old Illustrator exports// emit): the `[` ends the doctype as bogus and the declarations tokenize as@@ -867,18 +1001,20 @@ fn strip_doctype_internal_subset(bytes: &[u8]) -> std::borrow::Cow<'_, [u8]> {// (`<root><a>text`) parses "cleanly". Browsers reject it. This sits between// the tokenizer and the tree builder, tracks tag nesting from the raw token// stream, and reports an error when EOF arrives with tags still open.-struct UnclosedTagSink<'arena> {- tb: xml5ever::tree_builder::XmlTreeBuilder<Ref, sink::Sink<'arena>>,+struct UnclosedTagSink<H: Clone, S: xml5ever::tree_builder::TreeSink<Handle = H>> {+ tb: xml5ever::tree_builder::XmlTreeBuilder<H, S>,depth: Cell<u32>,}-impl<'arena> xml5ever::tokenizer::TokenSink for UnclosedTagSink<'arena> {- type Handle = Ref;+impl<H: Clone, S: xml5ever::tree_builder::TreeSink<Handle = H>> xml5ever::tokenizer::TokenSink+ for UnclosedTagSink<H, S>+{+ type Handle = H;fn process_token(&self,token: xml5ever::tokenizer::Token,- ) -> xml5ever::tokenizer::ProcessResult<Ref> {+ ) -> xml5ever::tokenizer::ProcessResult<H> {use xml5ever::tokenizer::{TagKind, Token};match &token {Token::Tag(tag) => match tag.kind {@@ -890,7 +1026,6 @@ impl<'arena> xml5ever::tokenizer::TokenSink for UnclosedTagSink<'arena> {},Token::EndOfFile => {if self.depth.get() > 0 {- use xml5ever::tree_builder::TreeSink;self.tb.sink.parse_error(std::borrow::Cow::Borrowed("Unclosed element at EOF"));@@ -900,7 +1035,6 @@ impl<'arena> xml5ever::tokenizer::TokenSink for UnclosedTagSink<'arena> {// is created for it. xml5ever doesn't validate it.Token::ProcessingInstruction(pi) if &*pi.target == "xml" => {if !is_valid_xml_declaration(&pi.data) {- use xml5ever::tree_builder::TreeSink;self.tb.sink.parse_error(std::borrow::Cow::Borrowed("Invalid XML declaration"));@@ -943,12 +1077,14 @@ fn is_valid_xml_declaration(data: &str) -> bool {// xml5ever::driver::XmlParser, minus the tree-builder-typed tokenizer so the// UnclosedTagSink can sit in between.-struct XmlDocumentParser<'arena> {- tokenizer: xml5ever::tokenizer::XmlTokenizer<UnclosedTagSink<'arena>>,+struct XmlDocumentParser<H: Clone, S: xml5ever::tree_builder::TreeSink<Handle = H>> {+ tokenizer: xml5ever::tokenizer::XmlTokenizer<UnclosedTagSink<H, S>>,input_buffer: xml5ever::buffer_queue::BufferQueue,}-impl<'arena> TendrilSink<html5ever::tendril::fmt::UTF8> for XmlDocumentParser<'arena> {+impl<H: Clone, S: xml5ever::tree_builder::TreeSink<Handle = H>>+ TendrilSink<html5ever::tendril::fmt::UTF8> for XmlDocumentParser<H, S>+{type Output = ();fn process(&mut self, t: StrTendril) {@@ -957,7 +1093,6 @@ impl<'arena> TendrilSink<html5ever::tendril::fmt::UTF8> for XmlDocumentParser<'a}fn error(&mut self, desc: std::borrow::Cow<'static, str>) {- use xml5ever::tree_builder::TreeSink;self.tokenizer.sink.tb.sink.parse_error(desc)}