Lightpanda: the headless browser designed for AI and automation
webapi: fetch resolves after header received
Karl Seguinfc7935e51a8d
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.htmlindex 61bfda7a5..a0d7990ef 100644--- a/src/browser/tests/net/fetch.html+++ b/src/browser/tests/net/fetch.html@@ -591,3 +591,36 @@});}</script>++<script id=fetch_bad_content_encoding type=module>+ {+ const state = await testing.async();+ const url = 'http://127.0.0.1:9582/xhr/bad_gzip';+ const response = await fetch(url);+ const errors = [];+ for (const method of ['arrayBuffer', 'blob', 'bytes', 'formData', 'json', 'text']) {+ const res = await fetch(url);+ try {+ await res[method]();+ errors.push(null);+ } catch (e) {+ errors.push([e instanceof TypeError, res.bodyUsed]);+ }+ }+ let stream_error = null;+ try {+ await response.body.getReader().read();+ } catch (e) {+ stream_error = e;+ }+ state.resolve();++ await state.done(() => {+ testing.expectEqual(200, response.status);+ for (const err of errors) {+ testing.expectEqual([true, true], err);+ }+ testing.expectEqual(true, stream_error !== null);+ });+ }+</script>diff --git a/src/browser/webapi/cache/Cache.zig b/src/browser/webapi/cache/Cache.zigindex 7180f02bd..35bd605fc 100644--- a/src/browser/webapi/cache/Cache.zig+++ b/src/browser/webapi/cache/Cache.zig@@ -299,6 +299,9 @@ fn unstorable(response: *const Response) ?[]const u8 {if (response._body == .stream) {return "Response with a ReadableStream body is unsupported";}+ if (response._body == .errored) {+ return "Response body is errored";+ }for (response._headers._list._entries.items) |*header| {if (std.ascii.eqlIgnoreCase(header.name.str(), "vary") == false) {continue;diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex 685adac56..734acbd5d 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -46,6 +46,7 @@ _signal: ?*AbortSignal,_manual_redirect: bool,_no_cors: bool,_null_body: bool,+_headers_done: bool = false,_sink: Sink,pub const Input = Request.Input;@@ -160,6 +161,7 @@ fn submit(request: *Request, body: ?[]const u8, sink: Sink, exec: *const Executi.manual => .manual,.@"error" => .@"error",},+ .header_before_body_error = true,.header_callback = httpHeaderDoneCallback,.data_callback = httpDataCallback,.done_callback = httpDoneCallback,@@ -249,6 +251,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {res._url = "";res._type = .@"opaque";res._is_redirected = false;+ self._headers_done = true;return .proceed;}@@ -260,6 +263,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {res._url = "";res._type = .opaqueredirect;res._is_redirected = false;+ self._headers_done = true;return .proceed;}@@ -286,6 +290,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {try res._headers.append(hdr.name, hdr.value, exec);}+ self._headers_done = true;return .proceed;}@@ -354,7 +359,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {// Capture this before we reject. Rejection could trigger httpShutdownCallback// (via a microtask callback). But if we're here, then we'll take care of// cleaning up when we're done.- const owns_response = self._owns_response;+ var owns_response = self._owns_response;self._owns_response = false;// the response is only passed on v8 on success, if we're here, it's safe to@@ -373,10 +378,29 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {self._exec.js.localScope(&ls);defer ls.deinit();+ if (owns_response and self._headers_done and self.isAborted() == false) {+ // Once we have the headers, the promise should resolve. A later error,+ // like libcurl returning a BadContentEncoding errors at the body, not+ // the fetch.+ if (self._null_body == false) {+ response._body = .errored;+ }+ if (ls.local.zigValueToJs(response, .{})) |js_val| {+ owns_response = false;+ response._arena.report();+ return ls.toLocal(resolver).resolve("fetch done", js_val);+ } else |_| {}+ }+// fetch() must reject with a TypeError on network errors per specls.toLocal(resolver).rejectError("fetch error", .{ .type_error = "fetch error" });}+fn isAborted(self: *const Fetch) bool {+ const signal = self._signal orelse return false;+ return signal._aborted;+}+fn httpShutdownCallback(ctx: *anyopaque) void {const self: *Fetch = @ptrCast(@alignCast(ctx));diff --git a/src/browser/webapi/net/Response.zig b/src/browser/webapi/net/Response.zigindex be6f4602c..85cf63008 100644--- a/src/browser/webapi/net/Response.zig+++ b/src/browser/webapi/net/Response.zig@@ -62,6 +62,7 @@ const Body = union(enum) {empty,bytes: []const u8,stream: *ReadableStream,+ errored,};const InitOpts = struct {@@ -316,6 +317,18 @@ pub fn getBody(self: *Response, exec: *const Execution) !?*ReadableStream {}return stream;},+ .errored => {+ if (self._body_stream) |stream| {+ return stream;+ }+ const stream = try ReadableStream.init(null, null, exec);+ try stream._controller.doError("Failed to read response body");+ self._body_stream = stream;+ if (self._body_used) {+ try lockStream(stream, exec);+ }+ return stream;+ },};}@@ -334,7 +347,7 @@ fn getBodyUsed(self: *const Response) bool {return switch (self._body) {.empty => false,.stream => |stream| stream._disturbed,- .bytes => self._body_used,+ .bytes, .errored => self._body_used,};}@@ -343,7 +356,7 @@ fn checkUnusable(self: *const Response, local: *const js.Local) !void {const stream = switch (self._body) {.empty => return,.stream => |stream| stream,- .bytes => self._body_stream orelse {+ .bytes, .errored => self._body_stream orelse {if (self._body_used) {return local.typeError("Body has already been read");}@@ -360,10 +373,11 @@ fn consume(self: *Response, exec: *const Execution) !void {const local = exec.js.local.?;try self.checkUnusable(local);self._body_used = true;- if (self._body == .bytes) {- if (self._body_stream) |stream| {+ switch (self._body) {+ .bytes, .errored => if (self._body_stream) |stream| {try lockStream(stream, exec);- }+ },+ .empty, .stream => {},}}@@ -373,6 +387,7 @@ pub fn consumeBytes(self: *Response, exec: *const Execution) ![]const u8 {.empty => "",.bytes => |b| b,.stream => exec.js.local.?.typeError("Response with a ReadableStream body is unsupported"),+ .errored => exec.js.local.?.typeError("Failed to read response body"),};}@@ -410,6 +425,11 @@ fn consumeAs(self: *Response, kind: Package, exec: *const Execution) !js.Promiseconst content_type = try self._headers.get("content-type", exec);switch (self._body) {.stream => |stream| return StreamConsumer.start(stream, kind, content_type, exec),+ .errored => {+ var resolver = local.createPromiseResolver();+ resolver.rejectError("response body", .{ .type_error = "Failed to read response body" });+ return resolver.promise();+ },.bytes, .empty => {},}@@ -551,8 +571,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {const session = exec.session;const body_len = switch (self._body) {.bytes => |b| b.len,- .empty => 0,- .stream => 0,+ .empty, .stream, .errored => 0,};const arena = try session.getPinnedArena(body_len + self._url.len + 256, "Response.clone");errdefer arena.release();@@ -561,6 +580,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {.bytes => |b| .{ .bytes = try arena.dupe(u8, b) },.empty => .empty,.stream => .empty, // TODO: implement stream tee for proper cloning+ .errored => .errored,};const status_text = try arena.dupe(u8, self._status_text);const url = try arena.dupeZ(u8, self._url);diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 760f482cc..7ea212456 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1900,9 +1900,20 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T// callbacks run later, from dispatch(), never from here.if (effective_err != null and !is_conn_close_recv) {+ const err = transfer.res.callback_error orelse effective_err.?;+ if (try transfer.deliversHeaderBeforeError(msg.conn)) {+ try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });+ if (self.enforceCorsResponse(msg, transfer)) {+ return true;+ }+ self.removeConn(msg.conn);+ transfer._conn = null;+ try transfer.bufferHeaderThenError(err);+ return true;+ }self.removeConn(msg.conn);transfer._conn = null;- transfer.failAsync(transfer.res.callback_error orelse effective_err.?);+ transfer.failAsync(err);return true;}@@ -2098,6 +2109,12 @@ pub const Request = struct {// transfer, and thus we'll need to dupe it.body_outlives_request: bool = false,+ // Should an error after the header arives still call the header_callback.+ // Most cases just want the error. But fetch() is considered resolved once+ // we get the header, so where the error happens (before or after headers+ // is important)+ header_before_body_error: bool = false,+// arbitrary data that can be associated with this requestctx: *anyopaque = undefined,@@ -3265,6 +3282,33 @@ pub const Transfer = struct {};}+ fn deliversHeaderBeforeError(self: *const Transfer, conn: *const http.Connection) !bool {+ if (self.req.header_before_body_error == false or self.res.headers_complete == false) {+ return false;+ }+ if (self.res.stream.started) {+ // The header was already delivered.+ return false;+ }+ const status = try conn.getResponseCode();+ if (isRedirectStatus(status) and self.req.redirect != .manual and conn.getResponseHeader("location", 0) != null) {+ // this isn't the final response.+ return false;+ }+ return true;+ }++ // The header made it, the body didn't. Some cases (header_before_body_error = true)+ // want the header delivered first. (By default though, we just deliver the+ // error)+ fn bufferHeaderThenError(self: *Transfer, err: anyerror) !void {+ try self._events.ensureUnusedCapacity(self.arena.allocator(), 3);+ self._events.appendAssumeCapacity(.start);+ self._events.appendAssumeCapacity(.header);+ self._events.appendAssumeCapacity(.{ .err = err });+ self.scheduleDispatch();+ }+// Buffer the standard success event sequence. `body` is either owned by// transfer.arena OR, through some other mechanism, outlives the transfer.fn bufferEvents(self: *Transfer, body: []const u8) !void {@@ -3881,6 +3925,16 @@ pub const Transfer = struct {return chunk_len;}+ if (std.mem.trim(u8, line, "\r\n").len == 0) {+ // End of a header block. A 1xx is followed by another one.+ const conn: *http.Connection = @ptrCast(@alignCast(data));+ const status = conn.getResponseCode() catch return chunk_len;+ if (status >= 200) {+ conn.transport.http.res.headers_complete = true;+ }+ return chunk_len;+ }+if (announcesBody(line) == false) {return chunk_len;}@@ -4350,6 +4404,9 @@ const Response = struct {skip_body: bool = false,first_data_received: bool = false,+ // The final (non-1xx) response's header block has been fully received.+ headers_complete: bool = false,+// Set when dataCallback deliberately killed the transfer to satisfy// `Request.partial`. processOneMessage uses it to tell our own abort// apart from a real CURLE_WRITE_ERROR and deliver the response (headers,diff --git a/src/testing.zig b/src/testing.zigindex 949a42e2a..46c008bba 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -928,6 +928,14 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ if (std.mem.eql(u8, path, "/xhr/bad_gzip")) {+ return req.respond("not actually gzip", .{+ .extra_headers = &.{+ .{ .name = "Content-Encoding", .value = "gzip" },+ },+ });+ }+if (std.mem.eql(u8, path, "/xhr/no_content_type")) {return req.respond("untyped", .{});}