Lightpanda: the headless browser designed for AI and automation
webapi: make FileReader asynchronously + proper text decoding
Karl Seguin0d28a1c11957
diff --git a/src/browser/js/bridge.zig b/src/browser/js/bridge.zigindex 6d11b2ece..e2857fb3a 100644--- a/src/browser/js/bridge.zig+++ b/src/browser/js/bridge.zig@@ -1285,6 +1285,7 @@ const worker_common_apis = [_]type{@import("../webapi/net/WebSocket.zig"),@import("../webapi/net/EventSource.zig"),@import("../webapi/FileReader.zig"),+ @import("../webapi/FileReaderSync.zig"),@import("../webapi/ImageData.zig"),@import("../webapi/Performance.zig"),@import("../webapi/PerformanceObserver.zig"),@@ -1310,6 +1311,8 @@ pub const SharedWorkerJsApis = flattenTypes(&([_]type{@import("../webapi/SharedW// subsets (PageJsApis, WorkerSnapshot.JsApis).pub const JsApis = blk: {const base = PageJsApis ++ [_]type{+ // Worker-only, so it isn't in PageJsApis.+ @import("../webapi/FileReaderSync.zig").JsApi,@import("../webapi/DedicatedWorkerGlobalScope.zig").JsApi,@import("../webapi/SharedWorkerGlobalScope.zig").JsApi,@import("../webapi/WorkerGlobalScope.zig").JsApi,diff --git a/src/browser/webapi/FileReader.zig b/src/browser/webapi/FileReader.zigindex e7e8f82ef..a42ab53e8 100644--- a/src/browser/webapi/FileReader.zig+++ b/src/browser/webapi/FileReader.zig@@ -22,9 +22,12 @@ const lp = @import("lightpanda");const js = @import("../js/js.zig");const Page = @import("../Page.zig");+const Mime = @import("../Mime.zig");+const html5ever = @import("../parser/html5ever.zig");++const Blob = @import("Blob.zig");const EventTarget = @import("EventTarget.zig");const ProgressEvent = @import("event/ProgressEvent.zig");-const Blob = @import("Blob.zig");const Execution = js.Execution;const Allocator = std.mem.Allocator;@@ -40,6 +43,7 @@ _exec: *Execution,_proto: *EventTarget,_arena: *lp.Arena,+_generation: u32 = 0,_ready_state: ReadyState = .empty,_result: ?Result = null,_error: ?[]const u8 = null,@@ -51,8 +55,6 @@ _on_load_end: ?js.Function.Global = null,_on_load_start: ?js.Function.Global = null,_on_progress: ?js.Function.Global = null,-_aborted: bool = false,-const ReadyState = enum(u8) {empty = 0,loading = 1,@@ -160,20 +162,19 @@ pub fn getError(self: *const FileReader) ?[]const u8 {}pub fn readAsArrayBuffer(self: *FileReader, blob: *Blob) !void {- try self.readInternal(blob, .arraybuffer);+ try self.readInternal(blob, .arraybuffer, null);}pub fn readAsBinaryString(self: *FileReader, blob: *Blob) !void {- try self.readInternal(blob, .binary_string);+ try self.readInternal(blob, .binary_string, null);}pub fn readAsText(self: *FileReader, blob: *Blob, encoding_: ?[]const u8) !void {- _ = encoding_; // TODO: Handle encoding properly- try self.readInternal(blob, .text);+ try self.readInternal(blob, .text, encoding_);}pub fn readAsDataURL(self: *FileReader, blob: *Blob) !void {- try self.readInternal(blob, .data_url);+ try self.readInternal(blob, .data_url, null);}const ReadType = enum {@@ -183,61 +184,198 @@ const ReadType = enum {data_url,};-fn readInternal(self: *FileReader, blob: *Blob, read_type: ReadType) !void {+fn readInternal(self: *FileReader, blob: *Blob, read_type: ReadType, encoding_: ?[]const u8) !void {if (self._ready_state == .loading) {return error.InvalidStateError;}- // Reset state- self._ready_state = .loading;- self._result = null;self._error = null;- self._aborted = false;+ self._result = null;+ self._generation +%= 1;+ self._ready_state = .loading;const exec = self._exec;- try self.dispatch(.load_start, .{ .loaded = 0, .total = blob.getSize() }, exec);- if (self._aborted) {- return;- }+ // The task holds the blob rather than a copy of its bytes: the events are+ // dispatched from later tasks, by which point JS may have dropped its last+ // reference. Only `encoding` is copied, because it comes off the call arena.+ const task = try exec._factory.create(ReadTask{+ .blob = blob,+ .reader = self,+ .step = .load_start,+ .read_type = read_type,+ .generation = self._generation,+ .encoding = if (encoding_) |e| try self._arena.dupe(u8, e) else null,+ });+ errdefer exec._factory.destroy(task);- // Perform the read (synchronous since data is in memory). _result- // outlives this call and the blob can be GC'd before JS reads it,- // so the result must not borrow the blob's memory.- const data = try self._arena.dupe(u8, blob._slice);- const size = data.len;- try self.dispatch(.progress, .{ .loaded = size, .total = size }, exec);- if (self._aborted) {- return;- }+ blob.acquireRef();+ errdefer blob.releaseRef(exec.page);++ self.acquireRef();+ errdefer self.releaseRef(exec.page);++ try exec._scheduler.add(task, ReadTask.run, 0, .{+ .name = "FileReader.read",+ .finalizer = ReadTask.cancelled,+ });+}- // Process the data based on read type- self._result = switch (read_type) {- .arraybuffer => .{ .arraybuffer = .{ .values = data } },- .binary_string => .{ .binary_string = .{ .bytes = data } },- .text => .{ .string = data },+fn complete(self: *FileReader, task: *const ReadTask) !void {+ const arena = self._arena.allocator();+ const data = task.blob._slice;++ self._result = switch (task.read_type) {+ .arraybuffer => .{ .arraybuffer = .{ .values = try arena.dupe(u8, data) } },+ .binary_string => .{ .binary_string = .{ .bytes = try arena.dupe(u8, data) } },+ .text => .{ .string = try decodeText(arena, data, task.encoding, task.blob._mime) },.data_url => blk: {- // Create data URL with base64 encoding- const mime = if (blob._mime.len > 0) blob._mime else "application/octet-stream";- const data_url = try encodeDataURL(self._arena.allocator(), mime, data);- break :blk .{ .string = data_url };+ const mime = if (task.blob._mime.len > 0) task.blob._mime else "application/octet-stream";+ break :blk .{ .string = try encodeDataURL(arena, mime, data) };},};self._ready_state = .done;+ const exec = self._exec;+ const size = data.len;try self.dispatch(.load, .{ .loaded = size, .total = size }, exec);try self.dispatch(.load_end, .{ .loaded = size, .total = size }, exec);}+const ReadTask = struct {+ step: Step,+ blob: *Blob,+ generation: u32,+ reader: *FileReader,+ read_type: ReadType,+ encoding: ?[]const u8,++ const Step = enum { load_start, progress, complete };++ fn deinit(self: *ReadTask) void {+ const reader = self.reader;+ const exec = reader._exec;++ self.blob.releaseRef(exec.page);+ reader.releaseRef(exec.page);+ exec._factory.destroy(self);+ }++ fn cancelled(ctx: *anyopaque) void {+ const self: *ReadTask = @ptrCast(@alignCast(ctx));+ self.deinit();+ }++ fn run(ctx: *anyopaque) !?u32 {+ const self: *ReadTask = @ptrCast(@alignCast(ctx));+ const reader = self.reader;+ const exec = reader._exec;++ if (self.generation != reader._generation) {+ self.deinit();+ return null;+ }+ errdefer self.deinit();++ const size = self.blob._slice.len;+ switch (self.step) {+ .load_start => {+ self.step = if (size == 0) .complete else .progress;+ try reader.dispatch(.load_start, .{ .loaded = 0, .total = size }, exec);+ },+ .progress => {+ self.step = .complete;+ try reader.dispatch(.progress, .{ .loaded = size, .total = size }, exec);+ },+ .complete => {+ try reader.complete(self);+ self.deinit();+ return null;+ },+ }++ if (self.generation != reader._generation) {+ // a handler aborted or started another thread+ self.deinit();+ return null;+ }++ // enqueue for the next step+ try exec._scheduler.add(self, run, 0, .{+ .name = "FileReader.read",+ .finalizer = cancelled,+ });+ return null;+ }+};++fn encodingHandle(label: []const u8) ?*anyopaque {+ if (label.len == 0) {+ return null;+ }+ const info = html5ever.encoding_for_label(label.ptr, label.len);+ if (info.isValid() == false or std.mem.eql(u8, info.name(), "replacement")) {+ return null;+ }+ return info.handle;+}++pub fn decodeText(arena: Allocator, input: []const u8, label_: ?[]const u8, mime_type: []const u8) ![]const u8 {+ var data = input;+ var handle: ?*anyopaque = null;++ // BOM wins over an explicit encoding+ if (std.mem.startsWith(u8, data, "\xEF\xBB\xBF")) {+ data = data[3..];+ handle = encodingHandle("utf-8");+ } else if (std.mem.startsWith(u8, data, "\xFE\xFF")) {+ data = data[2..];+ handle = encodingHandle("utf-16be");+ } else if (std.mem.startsWith(u8, data, "\xFF\xFE")) {+ data = data[2..];+ handle = encodingHandle("utf-16le");+ } else if (label_) |label| {+ handle = encodingHandle(label);+ }++ if (handle == null and mime_type.len > 0) {+ if (Mime.parse(mime_type)) |mime| {+ if (mime.is_default_charset == false) {+ handle = encodingHandle(mime.charsetString());+ }+ } else |_| {}+ }++ const encoding = handle orelse encodingHandle("utf-8") orelse return arena.dupe(u8, data);++ const max_out = html5ever.encoding_max_utf8_buffer_length(encoding, if (data.len == 0) 4 else data.len);+ if (max_out == 0) {+ return "";+ }++ const out = try arena.alloc(u8, max_out);+ const result = html5ever.encoding_decode(+ encoding,+ if (data.len == 0) null else data.ptr,+ data.len,+ out.ptr,+ out.len,+ 1, // is_last+ );+ return out[0..result.bytes_written];+}+pub fn abort(self: *FileReader) !void {if (self._ready_state != .loading) {+ self._result = null;return;}- self._aborted = true;self._ready_state = .done;self._result = null;+ // Retires the queued read chain.+ self._generation +%= 1;const exec = self._exec;try self.dispatch(.abort, null, exec);@@ -287,7 +425,7 @@ const Progress = struct {/// Encodes binary data as a data URL with base64 encoding./// Format: data:[<mediatype>][;base64],<data>-fn encodeDataURL(arena: Allocator, mime: []const u8, data: []const u8) ![]const u8 {+pub fn encodeDataURL(arena: Allocator, mime: []const u8, data: []const u8) ![]const u8 {const base64 = std.base64.standard.Encoder;// Calculate size needed for base64 encodingdiff --git a/src/browser/webapi/FileReaderSync.zig b/src/browser/webapi/FileReaderSync.zignew file mode 100644index 000000000..7c1d74119--- /dev/null+++ b/src/browser/webapi/FileReaderSync.zig@@ -0,0 +1,70 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const js = @import("../js/js.zig");++const Blob = @import("Blob.zig");+const FileReader = @import("FileReader.zig");++const Execution = js.Execution;++/// https://w3c.github.io/FileAPI/#FileReaderSync+/// A worker-only, stateless reader: the blob's bytes are already in memory, so+/// every read is just a conversion. None of them re-enters JavaScript, so the+/// output goes on the local_arena.+const FileReaderSync = @This();++_pad: bool = false,++pub fn init() FileReaderSync {+ return .{};+}++pub fn readAsArrayBuffer(_: *const FileReaderSync, blob: *Blob) !js.ArrayBuffer {+ return .{ .values = blob._slice };+}++pub fn readAsBinaryString(_: *const FileReaderSync, blob: *Blob) !js.String.OneByte {+ return .{ .bytes = blob._slice };+}++pub fn readAsText(_: *const FileReaderSync, blob: *Blob, encoding_: ?[]const u8, exec: *const Execution) ![]const u8 {+ return FileReader.decodeText(exec.local_arena, blob._slice, encoding_, blob._mime);+}++pub fn readAsDataURL(_: *const FileReaderSync, blob: *Blob, exec: *const Execution) ![]const u8 {+ const mime = if (blob._mime.len > 0) blob._mime else "application/octet-stream";+ return FileReader.encodeDataURL(exec.local_arena, mime, blob._slice);+}++pub const JsApi = struct {+ pub const bridge = js.Bridge(FileReaderSync);++ pub const Meta = struct {+ pub const name = "FileReaderSync";+ pub const prototype_chain = bridge.prototypeChain();+ pub var class_id: bridge.ClassId = undefined;+ pub const empty_with_no_proto = true;+ };++ pub const constructor = bridge.constructor(FileReaderSync.init, .{});+ pub const readAsArrayBuffer = bridge.function(FileReaderSync.readAsArrayBuffer, .{});+ pub const readAsBinaryString = bridge.function(FileReaderSync.readAsBinaryString, .{});+ pub const readAsText = bridge.function(FileReaderSync.readAsText, .{});+ pub const readAsDataURL = bridge.function(FileReaderSync.readAsDataURL, .{});+};