Lightpanda: the headless browser designed for AI and automation
webapi: FormData encoding
Karl Seguin318ba730b541
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 3fc143d52..9826092b3 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1608,16 +1608,18 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {switch (self._parse_state) {.html => |*html| try html.buffer.appendSlice(html.arena.allocator(), data),.text => |*buf| {- // we have to escape the data...+ // The payload is text, not markup: `&` has to be escaped too, or+ // the parser turns a literal "★" in the body into a star.var v = data;while (v.len > 0) {- const index = std.mem.indexOfAnyPos(u8, v, 0, &.{ '<', '>' }) orelse {+ const index = std.mem.indexOfAnyPos(u8, v, 0, &.{ '<', '>', '&' }) orelse {return buf.appendSlice(self.arena, v);};try buf.appendSlice(self.arena, v[0..index]);switch (v[index]) {'<' => try buf.appendSlice(self.arena, "<"),'>' => try buf.appendSlice(self.arena, ">"),+ '&' => try buf.appendSlice(self.arena, "&"),else => unreachable,}v = v[index + 1 ..];@@ -3400,9 +3402,11 @@ pub fn resolveTargetFrame(self: *Frame, target_name: []const u8) ?*Frame {fn findFrameByName(frame: *Frame, name: []const u8) ?*Frame {for (frame.child_frames.items) |f| {if (f.iframe) |iframe| {- const frame_name = iframe.asElement().getAttributeSafe(comptime .wrap("name")) orelse "";- if (std.mem.eql(u8, frame_name, name)) {- return f;+ if (iframe.asNode().isConnected()) {+ const frame_name = iframe.asElement().getAttributeSafe(comptime .wrap("name")) orelse "";+ if (std.mem.eql(u8, frame_name, name)) {+ return f;+ }}}// Recursively search child frames@@ -3515,7 +3519,20 @@ pub fn submitForm(self: *Frame, submitter_: ?*Element, form_: ?*Element.Html.For// The submitter can be an input box (if enter was entered on the box)// I don't think this is technically correct, but FormData handles it ok- const form_data = try FormData.init(form, submitter_, &self.js.execution);+ // Resolved before the entry list is built: a hidden `_charset_` field+ // takes the submission encoding's name as its value.+ const charset: []const u8 = blk: {+ if (form_element.getAttributeSafe(.wrap("accept-charset"))) |ac| {+ // Normalize to canonical encoding name+ const info = h5e.encoding_for_label(ac.ptr, ac.len);+ if (info.isValid()) {+ break :blk info.name();+ }+ }+ break :blk self.charset;+ };++ const form_data = try FormData.initWithCharset(form, submitter_, charset, &self.js.execution);form_data.acquireRef();defer form_data.releaseRef(self._page);@@ -3567,18 +3584,6 @@ pub fn submitForm(self: *Frame, submitter_: ?*Element, form_: ?*Element.Html.Forconst arena = try self._session.getArena(.medium, "submitForm");errdefer arena.release();- // Get charset from accept-charset attribute or fall back to document charset- const charset: []const u8 = blk: {- if (form_element.getAttributeSafe(.wrap("accept-charset"))) |ac| {- // Normalize to canonical encoding name- const info = h5e.encoding_for_label(ac.ptr, ac.len);- if (info.isValid()) {- break :blk info.name();- }- }- break :blk self.charset;- };-var boundary_buf: [36]u8 = undefined;// GET ignores enctype per HTML spec; only resolve the union for POST.const encoding: FormData.EncType = blk: {@@ -3595,7 +3600,7 @@ pub fn submitForm(self: *Frame, submitter_: ?*Element, form_: ?*Element.Html.For};var buf = std.Io.Writer.Allocating.init(arena.allocator());- try form_data.write(.{ .encoding = encoding, .charset = charset, .allocator = arena.allocator() }, &buf.writer);+ try form_data.write(arena.allocator(), .{ .encoding = encoding, .charset = charset }, &buf.writer);var action = blk: {if (submit_button) |s| {diff --git a/src/browser/webapi/element/html/Input.zig b/src/browser/webapi/element/html/Input.zigindex c349ebbf4..0664b134c 100644--- a/src/browser/webapi/element/html/Input.zig+++ b/src/browser/webapi/element/html/Input.zig@@ -262,13 +262,23 @@ pub fn getFiles(self: *Input, frame: *Frame) !?*FileList {return try self.ensureFileList(frame);}-/// Replaces the selected file list and fires `input` + `change` events.-/// Used by CDP `DOM.setFileInputFiles` and any future DataTransfer-style setter.-///+/// Simulates the user selecting files: replaces the file list and fires+/// `input` + `change`. Used by CDP `DOM.setFileInputFiles`.+pub fn selectFiles(self: *Input, files: []const *File, frame: *Frame) !void {+ try self.replaceFiles(files, frame);++ // A file input fires `input` then `change`, both as plain bubbling Events+ // (not InputEvents — `inputType`/`data` only apply to editable text inputs).+ const input_evt = try Event.initTrusted(comptime .wrap("input"), .{ .bubbles = true }, frame._page);+ try frame._event_manager.dispatch(self.asElement().asEventTarget(), input_evt);+ const change_evt = try Event.initTrusted(comptime .wrap("change"), .{ .bubbles = true }, frame._page);+ try frame._event_manager.dispatch(self.asElement().asEventTarget(), change_evt);+}+/// The FileList holds a reference on each File (whose backing arena is reference/// counted via its Blob proto), so we acquire on the incoming files and release/// the outgoing ones; the frame releases whatever remains at teardown.-pub fn setFiles(self: *Input, files: []const *File, frame: *Frame) !void {+fn replaceFiles(self: *Input, files: []const *File, frame: *Frame) !void {if (self._input_type != .file) {return error.InvalidStateError;}@@ -285,13 +295,19 @@ pub fn setFiles(self: *Input, files: []const *File, frame: *Frame) !void {}fl._files = dupe;+}- // A file input fires `input` then `change`, both as plain bubbling Events- // (not InputEvents — `inputType`/`data` only apply to editable text inputs).- const input_evt = try Event.initTrusted(comptime .wrap("input"), .{ .bubbles = true }, frame._page);- try frame._event_manager.dispatch(self.asElement().asEventTarget(), input_evt);- const change_evt = try Event.initTrusted(comptime .wrap("change"), .{ .bubbles = true }, frame._page);- try frame._event_manager.dispatch(self.asElement().asEventTarget(), change_evt);+/// The `files` IDL setter. Unlike a user picking files (selectFiles), an+/// assignment fires no input/change event.+pub fn setFiles(self: *Input, list_: ?*FileList, frame: *Frame) !void {+ if (self._input_type != .file) {+ return;+ }+ const list = list_ orelse return;+ if (self._files == list) {+ return;+ }+ return self.replaceFiles(list._files, frame);}/// JS-binding wrapper for the `value` getter: for type=file, return the spec@@ -1292,7 +1308,7 @@ pub const JsApi = struct {pub const onselectionchange = bridge.accessor(Input.getOnSelectionChange, Input.setOnSelectionChange, .{});pub const @"type" = bridge.accessor(Input.getType, Input.setType, .{ .ce_reactions = true });pub const value = bridge.accessor(Input.getValueForJS, setValueFromJS, .{ .ce_reactions = true });- pub const files = bridge.accessor(Input.getFiles, null, .{});+ pub const files = bridge.accessor(Input.getFiles, Input.setFiles, .{});pub const defaultValue = bridge.accessor(Input.getDefaultValue, Input.setDefaultValue, .{ .ce_reactions = true });pub const checked = bridge.accessor(Input.getChecked, Input.setChecked, .{});pub const defaultChecked = bridge.accessor(Input.getDefaultChecked, Input.setDefaultChecked, .{ .ce_reactions = true });diff --git a/src/browser/webapi/net/FormData.zig b/src/browser/webapi/net/FormData.zigindex 4b4c4ccdd..682a51378 100644--- a/src/browser/webapi/net/FormData.zig+++ b/src/browser/webapi/net/FormData.zig@@ -35,6 +35,7 @@ const log = lp.log;const String = lp.String;const Execution = js.Execution;const Allocator = std.mem.Allocator;+const h5e = @import("../../parser/html5ever.zig");const FormData = @This();@@ -76,6 +77,10 @@ pub const Entry = struct {};pub fn init(form_: ?*Form, submitter: ?*Element, exec: *const Execution) !*FormData {+ return initWithCharset(form_, submitter, "UTF-8", exec);+}++pub fn initWithCharset(form_: ?*Form, submitter: ?*Element, charset: []const u8, exec: *const Execution) !*FormData {const arena = try exec.getArena(.small, "FormData");errdefer arena.release();@@ -101,7 +106,7 @@ pub fn init(form_: ?*Form, submitter: ?*Element, exec: *const Execution) !*FormDform._constructing_entry_list = true;defer form._constructing_entry_list = false;- form_data._entries = try collectForm(arena.allocator(), form, submitter, frame);+ form_data._entries = try collectForm(arena.allocator(), form, submitter, charset, frame);// Hold a reference on each entry's File for the FormData's lifetime; released// in deinit.@@ -329,32 +334,31 @@ pub const EncType = union(enum) {pub const WriteOpts = struct {encoding: EncType = .urlencode,charset: []const u8 = "UTF-8",- allocator: ?std.mem.Allocator = null,};-pub fn write(self: *const FormData, opts: WriteOpts, writer: *std.Io.Writer) !void {+pub fn write(self: *const FormData, allocator: Allocator, opts: WriteOpts, writer: *std.Io.Writer) !void {switch (opts.encoding) {- .urlencode => return self.urlEncode(opts, writer),- .formdata => |boundary| return self.multipartEncode(boundary, writer),+ .urlencode => return self.urlEncode(allocator, opts.charset, writer),+ .formdata => |boundary| return self.multipartEncode(allocator, boundary, opts.charset, writer),.plaintext => return self.plaintextEncode(writer),}}-fn urlEncode(self: *const FormData, opts: WriteOpts, writer: *std.Io.Writer) !void {+fn urlEncode(self: *const FormData, allocator: Allocator, charset: []const u8, writer: *std.Io.Writer) !void {const items = self._entries.items;if (items.len == 0) return;- try urlEncodeEntry(&items[0], opts, writer);+ try urlEncodeEntry(&items[0], allocator, charset, writer);for (items[1..]) |*entry| {try writer.writeByte('&');- try urlEncodeEntry(entry, opts, writer);+ try urlEncodeEntry(entry, allocator, charset, writer);}}-fn urlEncodeEntry(entry: *const Entry, opts: WriteOpts, writer: *std.Io.Writer) !void {- try KeyValueList.urlEncodeFormValue(entry.name.str(), opts.allocator, opts.charset, writer);+fn urlEncodeEntry(entry: *const Entry, allocator: Allocator, charset: []const u8, writer: *std.Io.Writer) !void {+ try KeyValueList.urlEncodeFormValue(entry.name.str(), allocator, charset, writer);try writer.writeByte('=');- try KeyValueList.urlEncodeFormValue(entry.value.asString(), opts.allocator, opts.charset, writer);+ try KeyValueList.urlEncodeFormValue(entry.value.asString(), allocator, charset, writer);}/// https://html.spec.whatwg.org/multipage/form-control-infrastructure.html#text/plain-encoding-algorithm@@ -371,22 +375,23 @@ fn plaintextEncode(self: *const FormData, writer: *std.Io.Writer) !void {}}-fn multipartEncode(self: *const FormData, boundary: []const u8, writer: *std.Io.Writer) !void {+fn multipartEncode(self: *const FormData, allocator: Allocator, boundary: []const u8, charset: []const u8, writer: *std.Io.Writer) !void {for (self._entries.items) |*entry| {- try multipartEncodeEntry(entry, boundary, writer);+ try multipartEncodeEntry(entry, allocator, boundary, charset, writer);}try writer.print("--{s}--\r\n", .{boundary});}-fn multipartEncodeEntry(entry: *const Entry, boundary: []const u8, writer: *std.Io.Writer) !void {+fn multipartEncodeEntry(entry: *const Entry, allocator: Allocator, boundary: []const u8, charset: []const u8, writer: *std.Io.Writer) !void {try writer.print("--{s}\r\n", .{boundary});+ const name = try encodeForCharset(allocator, entry.name.str(), charset);const value_ptr = &entry.value;switch (value_ptr.*) {.string => |*s| {try writer.writeAll("Content-Disposition: form-data; name=\"");- try writeMultipartName(writer, entry.name.str());+ try writeMultipartName(writer, name, .normalize_newlines);try writer.writeAll("\"\r\n\r\n");- try writer.writeAll(s.str());+ try writeNormalizedNewlines(writer, try encodeForCharset(allocator, s.str(), charset));try writer.writeAll("\r\n");},// Per RFC 7578 + WHATWG FormData: file parts carry a filename in@@ -394,9 +399,9 @@ fn multipartEncodeEntry(entry: *const Entry, boundary: []const u8, writer: *std.// application/octet-stream when the Blob has no MIME), and the raw bytes..file => |file| {try writer.writeAll("Content-Disposition: form-data; name=\"");- try writeMultipartName(writer, entry.name.str());+ try writeMultipartName(writer, name, .normalize_newlines);try writer.writeAll("\"; filename=\"");- try writeMultipartName(writer, file.getName());+ try writeMultipartName(writer, try encodeForCharset(allocator, file.getName(), charset), .verbatim_newlines);try writer.writeAll("\"\r\n");const mime = file._proto._mime;@@ -413,7 +418,7 @@ fn multipartEncodeEntry(entry: *const Entry, boundary: []const u8, writer: *std.// algorithm creates, and Chrome's wire output..no_file => {try writer.writeAll("Content-Disposition: form-data; name=\"");- try writeMultipartName(writer, entry.name.str());+ try writeMultipartName(writer, name, .normalize_newlines);try writer.writeAll("\"; filename=\"\"\r\n");try writer.writeAll("Content-Type: application/octet-stream\r\n\r\n");try writer.writeAll("\r\n");@@ -421,15 +426,68 @@ fn multipartEncodeEntry(entry: *const Entry, boundary: []const u8, writer: *std.}}-// Per RFC 7578 §4.2, Content-Disposition names are quoted-string form;-// CR/LF/" must be escaped.-fn writeMultipartName(writer: *std.Io.Writer, name: []const u8) !void {- for (name) |c| {- switch (c) {+/// Re-encodes UTF-8 text into the form's submission encoding+fn encodeForCharset(allocator: Allocator, input: []const u8, charset: []const u8) ![]const u8 {+ if (input.len == 0 or std.mem.eql(u8, charset, "UTF-8")) {+ return input;+ }++ const info = h5e.encoding_for_label(charset.ptr, charset.len);+ if (info.isValid() == false) {+ return input;+ }++ const base_len = h5e.encoding_max_encode_buffer_length(info.handle.?, input.len);+ if (base_len == 0) {+ return input;+ }++ // Because of numeric character refernece (NCR) a single character can+ // become "&#NNNNNNN;", so give it room to grow.+ const buf = try allocator.alloc(u8, base_len * 4);+ const result = h5e.encoding_encode_with_ncr(info.handle.?, input.ptr, input.len, buf.ptr, buf.len);+ if (result.isSuccess() == false) {+ return input;+ }+ return buf[0..result.bytes_written];+}++// multipart/form-data normalizes lone CR and lone LF into CRLF+fn writeNormalizedNewlines(writer: *std.Io.Writer, text: []const u8) !void {+ var i: usize = 0;+ while (i < text.len) : (i += 1) {+ switch (text[i]) {+ '\r' => {+ try writer.writeAll("\r\n");+ if (i + 1 < text.len and text[i + 1] == '\n') {+ i += 1;+ }+ },+ '\n' => try writer.writeAll("\r\n"),+ else => |c| try writer.writeByte(c),+ }+ }+}++const Newlines = enum { normalize_newlines, verbatim_newlines };+// Content-Disposition names are quoted-string form, CR and LF must be escaped+fn writeMultipartName(writer: *std.Io.Writer, name: []const u8, comptime newlines: Newlines) !void {+ var i: usize = 0;+ while (i < name.len) : (i += 1) {+ switch (name[i]) {'"' => try writer.writeAll("%22"),- '\r' => try writer.writeAll("%0D"),- '\n' => try writer.writeAll("%0A"),- else => try writer.writeByte(c),+ '\r' => {+ if (comptime newlines == .normalize_newlines) {+ try writer.writeAll("%0D%0A");+ if (i + 1 < name.len and name[i + 1] == '\n') {+ i += 1;+ }+ } else {+ try writer.writeAll("%0D");+ }+ },+ '\n' => try writer.writeAll(if (comptime newlines == .normalize_newlines) "%0D%0A" else "%0A"),+ else => |c| try writer.writeByte(c),}}}@@ -734,7 +792,7 @@ pub fn registerTypes() []const type {};}-fn collectForm(arena: Allocator, form_: ?*Form, submitter_: ?*Element, frame: *Frame) !std.ArrayList(Entry) {+fn collectForm(arena: Allocator, form_: ?*Form, submitter_: ?*Element, charset: []const u8, frame: *Frame) !std.ArrayList(Entry) {var list: std.ArrayList(Entry) = .empty;const form = form_ orelse return list;@@ -777,6 +835,11 @@ fn collectForm(arena: Allocator, form_: ?*Form, submitter_: ?*Element, frame: *Fcontinue;}}+ if (input_type == .hidden and std.ascii.eqlIgnoreCase(name, "_charset_")) {+ // The FormData's charset is used as the value for hidden+ // field name "_charset"+ break :blk charset;+ }if (input_type == .file) {// WHATWG: a file input with zero selected files contributes a// single entry whose value is an empty File of MIME@@ -888,10 +951,7 @@ test "FormData: multipart write" {try fd.appendText("note", "two\r\nlines");var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "BOUNDARY" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "BOUNDARY" } }, &buf.writer);try testing.expectString("--BOUNDARY\r\n" ++@@ -918,10 +978,7 @@ test "FormData: multipart escapes name CR/LF/quote" {try fd.appendText("a\"b\r\nc", "v");var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "B" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "B" } }, &buf.writer);try testing.expectString("--B\r\n" ++@@ -944,10 +1001,7 @@ test "FormData: multipart empty body" {};var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "B" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "B" } }, &buf.writer);try testing.expectString("--B--\r\n", buf.written());}@@ -989,10 +1043,7 @@ test "FormData: multipart with file" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "BOUNDARY" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "BOUNDARY" } }, &buf.writer);try testing.expectString("--BOUNDARY\r\n" ++@@ -1028,10 +1079,7 @@ test "FormData: multipart with empty file defaults to octet-stream" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "B" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "B" } }, &buf.writer);try testing.expectString("--B\r\n" ++@@ -1064,10 +1112,7 @@ test "FormData: multipart escapes file name and filename" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "B" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "B" } }, &buf.writer);try testing.expectString("--B\r\n" ++@@ -1100,7 +1145,7 @@ test "FormData: file entry collapses to filename in urlencode" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{ .encoding = .urlencode, .allocator = allocator }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .urlencode }, &buf.writer);try testing.expectString("upload=hello.txt", buf.written());}@@ -1120,10 +1165,7 @@ test "FormData: multipart no_file (unselected file input)" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{- .encoding = .{ .formdata = "B" },- .allocator = allocator,- }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .{ .formdata = "B" } }, &buf.writer);try testing.expectString("--B\r\n" ++@@ -1151,7 +1193,7 @@ test "FormData: no_file entry collapses to empty in urlencode" {});var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{ .encoding = .urlencode, .allocator = allocator }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .urlencode }, &buf.writer);try testing.expectString("upload=", buf.written());}@@ -1170,7 +1212,7 @@ test "FormData: plaintext write" {try fd.appendText("equals", "a=b");var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{ .encoding = .plaintext, .allocator = allocator }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .plaintext }, &buf.writer);// Per WHATWG HTML text/plain encoding algorithm: name=value CRLF per entry.// Values containing "=" or CRLF are written verbatim — the spec accepts that@@ -1195,7 +1237,7 @@ test "FormData: plaintext empty body" {};var buf = std.Io.Writer.Allocating.init(allocator);- try fd.write(.{ .encoding = .plaintext, .allocator = allocator }, &buf.writer);+ try fd.write(allocator, .{ .encoding = .plaintext }, &buf.writer);try testing.expectString("", buf.written());}@@ -1355,10 +1397,7 @@ test "FormData: multipart round-trip" {try src.appendText("a\tb", "tabbed");var buf = std.Io.Writer.Allocating.init(allocator);- try src.write(.{- .encoding = .{ .formdata = "BOUNDARY" },- .allocator = allocator,- }, &buf.writer);+ try src.write(allocator, .{ .encoding = .{ .formdata = "BOUNDARY" } }, &buf.writer);var fd = FormData{._rc = .{},diff --git a/src/browser/webapi/net/body_init.zig b/src/browser/webapi/net/body_init.zigindex 1b04fbe56..4e6b85ea7 100644--- a/src/browser/webapi/net/body_init.zig+++ b/src/browser/webapi/net/body_init.zig@@ -90,10 +90,7 @@ pub const BodyInit = union(enum) {@memcpy(boundary[4..], &hex);var buf = std.Io.Writer.Allocating.init(arena);- try fd.write(.{- .allocator = arena,- .encoding = .{ .formdata = &boundary },- }, &buf.writer);+ try fd.write(arena, .{ .encoding = .{ .formdata = &boundary } }, &buf.writer);const ct = try std.fmt.allocPrint(arena, "multipart/form-data; boundary={s}", .{boundary});return .{diff --git a/src/cdp/domains/dom.zig b/src/cdp/domains/dom.zigindex 517390084..349901957 100644--- a/src/cdp/domains/dom.zig+++ b/src/cdp/domains/dom.zig@@ -638,7 +638,7 @@ fn setFileInputFiles(cmd: *CDP.Command) !void {var files = try cmd.arena.alloc(*File, params.files.len);{- // Files are created at refcount 0; setFiles takes ownership. If a later+ // Files are created at refcount 0; selectFiles takes ownership. If a later// path fails to load, release the arenas of the ones already created.var created: usize = 0;errdefer for (files[0..created]) |f| f._proto.deinit(frame._page);@@ -647,7 +647,7 @@ fn setFileInputFiles(cmd: *CDP.Command) !void {created = i + 1;}}- try input.setFiles(files, frame);+ try input.selectFiles(files, frame);return cmd.sendResult(null, .{});}