Skip to content

Lightpanda: the headless browser designed for AI and automation

webapi: fetch resolves after header received

Karl Seguinfc7935e51a8d
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.html
index 61bfda7a5..a0d7990ef 100644
--- a/src/browser/tests/net/fetch.html
+++ b/src/browser/tests/net/fetch.html
@@ -591,3 +591,36 @@
});
}
</script>
+
+<script id=fetch_bad_content_encoding type=module>
+ {
+ const state = await testing.async();
+ const url = 'http://127.0.0.1:9582/xhr/bad_gzip';
+ const response = await fetch(url);
+ const errors = [];
+ for (const method of ['arrayBuffer', 'blob', 'bytes', 'formData', 'json', 'text']) {
+ const res = await fetch(url);
+ try {
+ await res[method]();
+ errors.push(null);
+ } catch (e) {
+ errors.push([e instanceof TypeError, res.bodyUsed]);
+ }
+ }
+ let stream_error = null;
+ try {
+ await response.body.getReader().read();
+ } catch (e) {
+ stream_error = e;
+ }
+ state.resolve();
+
+ await state.done(() => {
+ testing.expectEqual(200, response.status);
+ for (const err of errors) {
+ testing.expectEqual([true, true], err);
+ }
+ testing.expectEqual(true, stream_error !== null);
+ });
+ }
+</script>
diff --git a/src/browser/webapi/cache/Cache.zig b/src/browser/webapi/cache/Cache.zig
index 7180f02bd..35bd605fc 100644
--- a/src/browser/webapi/cache/Cache.zig
+++ b/src/browser/webapi/cache/Cache.zig
@@ -299,6 +299,9 @@ fn unstorable(response: *const Response) ?[]const u8 {
if (response._body == .stream) {
return "Response with a ReadableStream body is unsupported";
}
+ if (response._body == .errored) {
+ return "Response body is errored";
+ }
for (response._headers._list._entries.items) |*header| {
if (std.ascii.eqlIgnoreCase(header.name.str(), "vary") == false) {
continue;
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 685adac56..734acbd5d 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -46,6 +46,7 @@ _signal: ?*AbortSignal,
_manual_redirect: bool,
_no_cors: bool,
_null_body: bool,
+_headers_done: bool = false,
_sink: Sink,
pub const Input = Request.Input;
@@ -160,6 +161,7 @@ fn submit(request: *Request, body: ?[]const u8, sink: Sink, exec: *const Executi
.manual => .manual,
.@"error" => .@"error",
},
+ .header_before_body_error = true,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,
@@ -249,6 +251,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
res._url = "";
res._type = .@"opaque";
res._is_redirected = false;
+ self._headers_done = true;
return .proceed;
}
@@ -260,6 +263,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
res._url = "";
res._type = .opaqueredirect;
res._is_redirected = false;
+ self._headers_done = true;
return .proceed;
}
@@ -286,6 +290,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
try res._headers.append(hdr.name, hdr.value, exec);
}
+ self._headers_done = true;
return .proceed;
}
@@ -354,7 +359,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
// Capture this before we reject. Rejection could trigger httpShutdownCallback
// (via a microtask callback). But if we're here, then we'll take care of
// cleaning up when we're done.
- const owns_response = self._owns_response;
+ var owns_response = self._owns_response;
self._owns_response = false;
// the response is only passed on v8 on success, if we're here, it's safe to
@@ -373,10 +378,29 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
self._exec.js.localScope(&ls);
defer ls.deinit();
+ if (owns_response and self._headers_done and self.isAborted() == false) {
+ // Once we have the headers, the promise should resolve. A later error,
+ // like libcurl returning a BadContentEncoding errors at the body, not
+ // the fetch.
+ if (self._null_body == false) {
+ response._body = .errored;
+ }
+ if (ls.local.zigValueToJs(response, .{})) |js_val| {
+ owns_response = false;
+ response._arena.report();
+ return ls.toLocal(resolver).resolve("fetch done", js_val);
+ } else |_| {}
+ }
+
// fetch() must reject with a TypeError on network errors per spec
ls.toLocal(resolver).rejectError("fetch error", .{ .type_error = "fetch error" });
}
+fn isAborted(self: *const Fetch) bool {
+ const signal = self._signal orelse return false;
+ return signal._aborted;
+}
+
fn httpShutdownCallback(ctx: *anyopaque) void {
const self: *Fetch = @ptrCast(@alignCast(ctx));
diff --git a/src/browser/webapi/net/Response.zig b/src/browser/webapi/net/Response.zig
index be6f4602c..85cf63008 100644
--- a/src/browser/webapi/net/Response.zig
+++ b/src/browser/webapi/net/Response.zig
@@ -62,6 +62,7 @@ const Body = union(enum) {
empty,
bytes: []const u8,
stream: *ReadableStream,
+ errored,
};
const InitOpts = struct {
@@ -316,6 +317,18 @@ pub fn getBody(self: *Response, exec: *const Execution) !?*ReadableStream {
}
return stream;
},
+ .errored => {
+ if (self._body_stream) |stream| {
+ return stream;
+ }
+ const stream = try ReadableStream.init(null, null, exec);
+ try stream._controller.doError("Failed to read response body");
+ self._body_stream = stream;
+ if (self._body_used) {
+ try lockStream(stream, exec);
+ }
+ return stream;
+ },
};
}
@@ -334,7 +347,7 @@ fn getBodyUsed(self: *const Response) bool {
return switch (self._body) {
.empty => false,
.stream => |stream| stream._disturbed,
- .bytes => self._body_used,
+ .bytes, .errored => self._body_used,
};
}
@@ -343,7 +356,7 @@ fn checkUnusable(self: *const Response, local: *const js.Local) !void {
const stream = switch (self._body) {
.empty => return,
.stream => |stream| stream,
- .bytes => self._body_stream orelse {
+ .bytes, .errored => self._body_stream orelse {
if (self._body_used) {
return local.typeError("Body has already been read");
}
@@ -360,10 +373,11 @@ fn consume(self: *Response, exec: *const Execution) !void {
const local = exec.js.local.?;
try self.checkUnusable(local);
self._body_used = true;
- if (self._body == .bytes) {
- if (self._body_stream) |stream| {
+ switch (self._body) {
+ .bytes, .errored => if (self._body_stream) |stream| {
try lockStream(stream, exec);
- }
+ },
+ .empty, .stream => {},
}
}
@@ -373,6 +387,7 @@ pub fn consumeBytes(self: *Response, exec: *const Execution) ![]const u8 {
.empty => "",
.bytes => |b| b,
.stream => exec.js.local.?.typeError("Response with a ReadableStream body is unsupported"),
+ .errored => exec.js.local.?.typeError("Failed to read response body"),
};
}
@@ -410,6 +425,11 @@ fn consumeAs(self: *Response, kind: Package, exec: *const Execution) !js.Promise
const content_type = try self._headers.get("content-type", exec);
switch (self._body) {
.stream => |stream| return StreamConsumer.start(stream, kind, content_type, exec),
+ .errored => {
+ var resolver = local.createPromiseResolver();
+ resolver.rejectError("response body", .{ .type_error = "Failed to read response body" });
+ return resolver.promise();
+ },
.bytes, .empty => {},
}
@@ -551,8 +571,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {
const session = exec.session;
const body_len = switch (self._body) {
.bytes => |b| b.len,
- .empty => 0,
- .stream => 0,
+ .empty, .stream, .errored => 0,
};
const arena = try session.getPinnedArena(body_len + self._url.len + 256, "Response.clone");
errdefer arena.release();
@@ -561,6 +580,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {
.bytes => |b| .{ .bytes = try arena.dupe(u8, b) },
.empty => .empty,
.stream => .empty, // TODO: implement stream tee for proper cloning
+ .errored => .errored,
};
const status_text = try arena.dupe(u8, self._status_text);
const url = try arena.dupeZ(u8, self._url);
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 760f482cc..7ea212456 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1900,9 +1900,20 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
// callbacks run later, from dispatch(), never from here.
if (effective_err != null and !is_conn_close_recv) {
+ const err = transfer.res.callback_error orelse effective_err.?;
+ if (try transfer.deliversHeaderBeforeError(msg.conn)) {
+ try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });
+ if (self.enforceCorsResponse(msg, transfer)) {
+ return true;
+ }
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ try transfer.bufferHeaderThenError(err);
+ return true;
+ }
self.removeConn(msg.conn);
transfer._conn = null;
- transfer.failAsync(transfer.res.callback_error orelse effective_err.?);
+ transfer.failAsync(err);
return true;
}
@@ -2098,6 +2109,12 @@ pub const Request = struct {
// transfer, and thus we'll need to dupe it.
body_outlives_request: bool = false,
+ // Should an error after the header arives still call the header_callback.
+ // Most cases just want the error. But fetch() is considered resolved once
+ // we get the header, so where the error happens (before or after headers
+ // is important)
+ header_before_body_error: bool = false,
+
// arbitrary data that can be associated with this request
ctx: *anyopaque = undefined,
@@ -3265,6 +3282,33 @@ pub const Transfer = struct {
};
}
+ fn deliversHeaderBeforeError(self: *const Transfer, conn: *const http.Connection) !bool {
+ if (self.req.header_before_body_error == false or self.res.headers_complete == false) {
+ return false;
+ }
+ if (self.res.stream.started) {
+ // The header was already delivered.
+ return false;
+ }
+ const status = try conn.getResponseCode();
+ if (isRedirectStatus(status) and self.req.redirect != .manual and conn.getResponseHeader("location", 0) != null) {
+ // this isn't the final response.
+ return false;
+ }
+ return true;
+ }
+
+ // The header made it, the body didn't. Some cases (header_before_body_error = true)
+ // want the header delivered first. (By default though, we just deliver the
+ // error)
+ fn bufferHeaderThenError(self: *Transfer, err: anyerror) !void {
+ try self._events.ensureUnusedCapacity(self.arena.allocator(), 3);
+ self._events.appendAssumeCapacity(.start);
+ self._events.appendAssumeCapacity(.header);
+ self._events.appendAssumeCapacity(.{ .err = err });
+ self.scheduleDispatch();
+ }
+
// Buffer the standard success event sequence. `body` is either owned by
// transfer.arena OR, through some other mechanism, outlives the transfer.
fn bufferEvents(self: *Transfer, body: []const u8) !void {
@@ -3881,6 +3925,16 @@ pub const Transfer = struct {
return chunk_len;
}
+ if (std.mem.trim(u8, line, "\r\n").len == 0) {
+ // End of a header block. A 1xx is followed by another one.
+ const conn: *http.Connection = @ptrCast(@alignCast(data));
+ const status = conn.getResponseCode() catch return chunk_len;
+ if (status >= 200) {
+ conn.transport.http.res.headers_complete = true;
+ }
+ return chunk_len;
+ }
+
if (announcesBody(line) == false) {
return chunk_len;
}
@@ -4350,6 +4404,9 @@ const Response = struct {
skip_body: bool = false,
first_data_received: bool = false,
+ // The final (non-1xx) response's header block has been fully received.
+ headers_complete: bool = false,
+
// Set when dataCallback deliberately killed the transfer to satisfy
// `Request.partial`. processOneMessage uses it to tell our own abort
// apart from a real CURLE_WRITE_ERROR and deliver the response (headers,
diff --git a/src/testing.zig b/src/testing.zig
index 949a42e2a..46c008bba 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -928,6 +928,14 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/xhr/bad_gzip")) {
+ return req.respond("not actually gzip", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Encoding", .value = "gzip" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/xhr/no_content_type")) {
return req.respond("untyped", .{});
}