Skip to content

Lightpanda: the headless browser designed for AI and automation

Merge headers_only and headers_only_prefix_bytes into a single optional

Karl Seguind6de5fee8ec7
diff --git a/src/browser/frame/resource_load.zig b/src/browser/frame/resource_load.zig
index ebd74e8a2..68a84d0bf 100644
--- a/src/browser/frame/resource_load.zig
+++ b/src/browser/frame/resource_load.zig
@@ -84,8 +84,7 @@ pub fn image(frame: *Frame, img: *Element.Html.Image, src: []const u8) !void {
.request_mode = .no_cors,
.credentials_mode = .include,
.resource_type = .image,
- .headers_only = true,
- .headers_only_prefix_bytes = 16 * 1024,
+ .partial = 16 * 1024,
.header_callback = ImageLoad.headerCallback,
.data_callback = ImageLoad.dataCallback,
.done_callback = ImageLoad.doneCallback,
diff --git a/src/browser/image_dimensions.zig b/src/browser/image_dimensions.zig
index 5b6da626c..d8ec40c7c 100644
--- a/src/browser/image_dimensions.zig
+++ b/src/browser/image_dimensions.zig
@@ -32,25 +32,7 @@ pub fn parse(bytes: []const u8) ?Dimensions {
}
if (bytes.len >= 4 and bytes[0] == 0xff and bytes[1] == 0xd8) {
- var pos: usize = 2;
- while (pos < bytes.len) {
- if (bytes[pos] != 0xff) return null;
- while (pos < bytes.len and bytes[pos] == 0xff) : (pos += 1) {}
- if (pos >= bytes.len) return null;
- const marker = bytes[pos];
- pos += 1;
- if (marker == 0xd9 or marker == 0xda) return null; // EOI or scan data
- if (marker == 0x01 or (marker >= 0xd0 and marker <= 0xd7)) continue;
- if (bytes.len - pos < 2) return null;
- const len = std.mem.readInt(u16, bytes[pos..][0..2], .big);
- if (len < 2 or len > bytes.len - pos) return null;
- // SOF markers (except the non-frame DHT, JPG and DAC markers).
- if ((marker >= 0xc0 and marker <= 0xcf) and marker != 0xc4 and marker != 0xc8 and marker != 0xcc) {
- if (len < 7) return null;
- return valid(std.mem.readInt(u16, bytes[pos + 5 ..][0..2], .big), std.mem.readInt(u16, bytes[pos + 3 ..][0..2], .big));
- }
- pos += len;
- }
+ return jpeg(bytes);
}
if (bytes.len >= 25 and std.mem.eql(u8, bytes[0..4], "RIFF") and std.mem.eql(u8, bytes[8..12], "WEBP")) {
@@ -71,6 +53,100 @@ pub fn parse(bytes: []const u8) ?Dimensions {
return null;
}
+fn jpeg(bytes: []const u8) ?Dimensions {
+ var orientation: u16 = 1;
+ var pos: usize = 2;
+ while (pos < bytes.len) {
+ if (bytes[pos] != 0xff) {
+ return null;
+ }
+ while (pos < bytes.len and bytes[pos] == 0xff) : (pos += 1) {}
+ if (pos >= bytes.len) {
+ return null;
+ }
+
+ const marker = bytes[pos];
+ pos += 1;
+ if (marker == 0xd9 or marker == 0xda) {
+ return null; // EOI or scan data
+ }
+ if (marker == 0x01 or (marker >= 0xd0 and marker <= 0xd7)) {
+ continue;
+ }
+
+ if (bytes.len - pos < 2) {
+ return null;
+ }
+ const len = std.mem.readInt(u16, bytes[pos..][0..2], .big);
+ if (len < 2 or len > bytes.len - pos) {
+ return null;
+ }
+
+ const segment = bytes[pos + 2 .. pos + len];
+ if (marker == 0xe1 and orientation == 1) {
+ orientation = exifOrientation(segment);
+ }
+ // SOF markers (except the non-frame DHT, JPG and DAC markers).
+ if ((marker >= 0xc0 and marker <= 0xcf) and marker != 0xc4 and marker != 0xc8 and marker != 0xcc) {
+ if (segment.len < 5) {
+ return null;
+ }
+ const height = std.mem.readInt(u16, segment[1..3], .big);
+ const width = std.mem.readInt(u16, segment[3..5], .big);
+ // Browsers apply EXIF orientation by default (image-orientation:
+ // from-image); 5-8 are the orientations that rotate by 90°.
+ if (orientation >= 5 and orientation <= 8) {
+ return valid(height, width);
+ }
+ return valid(width, height);
+ }
+ pos += len;
+ }
+ return null;
+}
+
+// The Orientation tag (0x0112) from IFD0 of an APP1 Exif segment, or 1
+// (upright) when there isn't one.
+fn exifOrientation(segment: []const u8) u16 {
+ if (segment.len < 14 or std.mem.eql(u8, segment[0..6], "Exif\x00\x00") == false) {
+ return 1;
+ }
+
+ const tiff = segment[6..];
+ const endian: std.builtin.Endian = if (std.mem.eql(u8, tiff[0..2], "II"))
+ .little
+ else if (std.mem.eql(u8, tiff[0..2], "MM"))
+ .big
+ else
+ return 1;
+
+ if (std.mem.readInt(u16, tiff[2..4], endian) != 42) {
+ return 1;
+ }
+
+ const ifd = std.mem.readInt(u32, tiff[4..8], endian);
+ if (ifd > tiff.len - 2) {
+ return 1;
+ }
+ const count = std.mem.readInt(u16, tiff[ifd..][0..2], endian);
+ var entry: usize = ifd + 2;
+ for (0..count) |_| {
+ if (tiff.len - entry < 12) {
+ return 1;
+ }
+
+ if (std.mem.readInt(u16, tiff[entry..][0..2], endian) == 0x0112) {
+ // SHORT: the value sits in the first two bytes of the value field.
+ if (std.mem.readInt(u16, tiff[entry + 2 ..][0..2], endian) != 3) {
+ return 1;
+ }
+ return std.mem.readInt(u16, tiff[entry + 8 ..][0..2], endian);
+ }
+ entry += 12;
+ }
+ return 1;
+}
+
fn valid(width: u32, height: u32) ?Dimensions {
if (width == 0 or height == 0) return null;
return .{ .width = width, .height = height };
@@ -85,6 +161,23 @@ test "image dimensions: PNG, GIF, JPEG, WebP" {
try testing.expectEqual(Dimensions{ .width = 1, .height = 1 }, parse("RIFF\x00\x00\x00\x00WEBPVP8L\x05\x00\x00\x00\x2f\x00\x00\x00\x00\x00").?);
}
+test "image dimensions: JPEG EXIF orientation" {
+ // APP1 Exif, big-endian, one IFD0 entry: Orientation (SHORT) = 6.
+ const rotated = "\xff\xd8\xff\xe1\x00\x22Exif\x00\x00MM\x00\x2a\x00\x00\x00\x08\x00\x01\x01\x12\x00\x03\x00\x00\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00" ++
+ "\xff\xc0\x00\x0b\x08\x02\xee\x03\xe8\x01\x01\x11\x00";
+ try testing.expectEqual(Dimensions{ .width = 750, .height = 1000 }, parse(rotated).?);
+
+ // Same, little-endian, Orientation = 3 (180°): no swap.
+ const flipped = "\xff\xd8\xff\xe1\x00\x22Exif\x00\x00II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00" ++
+ "\xff\xc0\x00\x0b\x08\x02\xee\x03\xe8\x01\x01\x11\x00";
+ try testing.expectEqual(Dimensions{ .width = 1000, .height = 750 }, parse(flipped).?);
+
+ // An IFD offset past the segment is ignored, not trusted.
+ const bogus = "\xff\xd8\xff\xe1\x00\x10Exif\x00\x00MM\x00\x2a\xff\xff\xff\xff" ++
+ "\xff\xc0\x00\x0b\x08\x02\xee\x03\xe8\x01\x01\x11\x00";
+ try testing.expectEqual(Dimensions{ .width = 1000, .height = 750 }, parse(bogus).?);
+}
+
test "image dimensions: truncated or invalid headers" {
try testing.expectEqual(null, parse("\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00"));
try testing.expectEqual(null, parse("\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\x00\x00\x00\x00\x01"));
diff --git a/src/browser/tests/element/html/image_fetch.html b/src/browser/tests/element/html/image_fetch.html
index ca8b83bef..74fc63050 100644
--- a/src/browser/tests/element/html/image_fetch.html
+++ b/src/browser/tests/element/html/image_fetch.html
@@ -9,7 +9,7 @@
complete successfully — that's what makes these a test of the fetch path
and not just of the event plumbing.
- /images/ok.png is over the headers-only drain threshold (abort branch),
+ /images/ok.png is over the partial drain threshold (abort branch),
/images/small.png is under it (drain branch).
-->
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index bff53232e..ef1fb4744 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1296,8 +1296,8 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
// Redirects rewrite req.url; the entry must be stored/renewed under the
// URL this lookup ran against, not the final hop. req.url is arena-owned,
// so the captured slice outlives any redirect rewrite.
- const key: [:0]const u8 = if (req.headers_only)
- try std.fmt.allocPrintSentinel(arena.allocator(), "headers-only:{s}", .{req.url}, 0)
+ const key: [:0]const u8 = if (req.partial != null)
+ try std.fmt.allocPrintSentinel(arena.allocator(), "partial:{s}", .{req.url}, 0)
else
req.url;
transfer._cache_key = key;
@@ -1782,9 +1782,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
// we match that behavior: when CURLE_WRITE_ERROR arrives but our callback
// never errored and bytes were received, treat it as success.
const effective_err: ?anyerror = if (msg.err) |err| blk: {
- // Our own headers_only abort, not a failure: fall through so the
- // response is materialized and delivered with an empty body.
- if (err == error.WriteError and transfer.res.headers_only_abort) {
+ // Our own partial abort, not a failure: fall through so the
+ // response is materialized and delivered with the kept prefix.
+ if (err == error.WriteError and transfer.res.partial_abort) {
break :blk null;
}
if (err == error.WriteError and transfer.res.callback_error == null and transfer.res.bytes_received > 0) {
@@ -2017,11 +2017,11 @@ pub const Request = struct {
// internal requests transparently following redirects.
const RedirectMode = enum { follow, manual, @"error" };
- // How much of a headers_only body we'll read rather than abort. Draining
+ // How much of a partial body we'll read rather than abort. Draining
// costs bandwidth but keeps the connection poolable; aborting saves
// bandwidth but forces a reconnect. 16 KiB is the rough break-even: about
// ten segments, versus a TCP handshake plus a TLS one.
- const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024;
+ const PARTIAL_DRAIN_MAX: usize = 16 * 1024;
pub const CredentialsMode = enum {
// Never send credentials, even same-origin.
@@ -2051,20 +2051,15 @@ pub const Request = struct {
timeout_ms: u32 = 0,
skip_cache: bool = false,
- // The caller wants the status and the response headers, not the body.
- // Unlike a HEAD, the request itself is byte-for-byte a normal GET, so
- // origins and CDNs see (and answer) exactly what a real browser sends;
- // the body is then discarded, and torn off the wire if it doesn't fit in
- // HEADERS_ONLY_DRAIN_MAX. The consumer still gets the usual
- // start/header/done sequence, with an empty body unless the optional
- // headers_only_prefix_bytes is set.
- headers_only: bool = false,
-
- // Keep at most this many leading bytes while discarding the rest of a
- // headers-only response. Used to identify image dimensions without
- // downloading or retaining the entire image. The prefix is delivered to
- // data_callback before done_callback. Capped at HEADERS_ONLY_DRAIN_MAX.
- headers_only_prefix_bytes: usize = 0,
+ // The caller wants the status, the response headers and at most this
+ // many leading body bytes (0 for none), not the whole body. Unlike a
+ // HEAD, the request itself is byte-for-byte a normal GET, so origins and
+ // CDNs see (and answer) exactly what a real browser sends; the rest of
+ // the body is discarded, and torn off the wire if it doesn't fit in
+ // PARTIAL_DRAIN_MAX, which also caps the prefix. The consumer gets the
+ // usual start/header/data/done sequence with the prefix as the body. CDP
+ // never sees the prefix: it isn't the response body.
+ partial: ?u32 = null,
// Should only be set when they need to differ from the owner's.
frame_id: u32 = 0,
@@ -3201,12 +3196,12 @@ pub const Transfer = struct {
}
// A cached body is stored decoded; its wire size is long gone. A
- // headers_only fetch (images) tears the body off the wire: the
+ // partial fetch (images) tears the body off the wire: the
// Content-Length, else whatever arrived before the abort, is the
// best size we have for both.
var decoded_body_size = t.decoded_body_size;
var encoded_body_size = if (t.cache == .none) t.encoded_body_size else decoded_body_size;
- if (self.req.headers_only) {
+ if (self.req.partial != null) {
const known = if (self._content_length > 0) self._content_length else t.encoded_body_size;
decoded_body_size = known;
encoded_body_size = known;
@@ -3400,11 +3395,11 @@ pub const Transfer = struct {
}
}
- // headers_only is exempt: the cap exists to bound how much body we
- // buffer, and this transfer buffers none of it. Failing a 4 MB image
+ // A partial fetch is exempt: the cap exists to bound how much body we
+ // buffer, and this transfer buffers at most PARTIAL_DRAIN_MAX of it. Failing a 4 MB image
// we were never going to read would turn the size limit into a
// spurious `error` event on a perfectly good response.
- if (opts.check_content_length and !self.req.headers_only) {
+ if (opts.check_content_length and self.req.partial == null) {
if (self.getContentLength()) |cl| {
if (cl > self.client.max_response_size) {
return error.ResponseTooLarge;
@@ -3940,7 +3935,7 @@ pub const Transfer = struct {
return @intCast(chunk_len);
}
- if (transfer.req.headers_only == false) {
+ if (transfer.req.partial == null) {
if (transfer.getContentLength()) |cl| {
if (cl > transfer.client.max_response_size) {
res.callback_error = error.ResponseTooLarge;
@@ -3964,8 +3959,8 @@ pub const Transfer = struct {
res.bytes_received += chunk_len;
- if (transfer.req.headers_only) {
- const limit = @min(transfer.req.headers_only_prefix_bytes, Request.HEADERS_ONLY_DRAIN_MAX);
+ if (transfer.req.partial) |partial| {
+ const limit = @min(partial, Request.PARTIAL_DRAIN_MAX);
if (res.buffer.items.len < limit) {
const count = @min(chunk_len, limit - res.buffer.items.len);
res.buffer.appendSlice(transfer.arena.allocator(), buffer[0..count]) catch |err| {
@@ -3975,15 +3970,15 @@ pub const Transfer = struct {
}
// Plenty of images have no Content-Length to decide this up front, so
// decide it as the body arrives.
- if (res.bytes_received <= Request.HEADERS_ONLY_DRAIN_MAX) {
+ if (res.bytes_received <= Request.PARTIAL_DRAIN_MAX) {
return @intCast(chunk_len);
}
// Returning writefunc_error is the only way to end a transfer
// early from a write callback; processOneMessage recognises the
// flag and treats the resulting CURLE_WRITE_ERROR as a completed
- // response with an empty body.
- res.headers_only_abort = true;
+ // response with the kept prefix as its body.
+ res.partial_abort = true;
return http.writefunc_error;
}
@@ -4172,7 +4167,8 @@ pub const Transfer = struct {
}
},
.data => |chunk| {
- if (transfer._notify_cdp) {
+ // A partial body is a prefix, not the response body.
+ if (transfer._notify_cdp and req.partial == null) {
transfer.notify(.http_response_data, &.{
.data = chunk,
.transfer = transfer,
@@ -4321,10 +4317,10 @@ const Response = struct {
first_data_received: bool = false,
// Set when dataCallback deliberately killed the transfer to satisfy
- // `Request.headers_only`. processOneMessage uses it to tell our own
- // abort apart from a real CURLE_WRITE_ERROR and deliver the response
- // (headers, status, empty body) as a success.
- headers_only_abort: bool = false,
+ // `Request.partial`. processOneMessage uses it to tell our own abort
+ // apart from a real CURLE_WRITE_ERROR and deliver the response (headers,
+ // status, kept prefix) as a success.
+ partial_abort: bool = false,
// Response body. Filled by dataCallback, consumed in processMessages.
// See Stream.spare to see how this works in streaming mode
diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zig
index daaf12385..a848262c0 100644
--- a/src/server/cdp/domains/network.zig
+++ b/src/server/cdp/domains/network.zig
@@ -1294,7 +1294,7 @@ const EchoDriver = struct {
self.err = err;
}
- fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8) ![14]u8 {
+ fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8, partial: ?u32) ![14]u8 {
const client = &bc.cdp.browser.http_client;
var request_id: [14]u8 = undefined;
_ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
@@ -1306,6 +1306,7 @@ const EchoDriver = struct {
.method = .POST,
.url = "http://127.0.0.1:9582/echo_body",
.body = body,
+ .partial = partial,
.origin = bc.security_origin,
.request_mode = .no_cors,
.credentials_mode = .same_origin,
@@ -1341,8 +1342,8 @@ test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" {
});
try ctx.expectSentResult(null, .{ .id = 1 });
- const big = try EchoDriver.run(bc, page.frame_id, "12345678");
- const small = try EchoDriver.run(bc, page.frame_id, "123");
+ const big = try EchoDriver.run(bc, page.frame_id, "12345678", null);
+ const small = try EchoDriver.run(bc, page.frame_id, "123", null);
try ctx.processMessage(.{
.id = 2,
@@ -1375,6 +1376,26 @@ test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" {
try testing.expectEqual(0, bc.captured_responses_size);
}
+test "cdp.Network: getResponseBody omits a partial body" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-PRT", .session_id = "SID-PRT" });
+ const page = try bc.session.createPage();
+
+ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ const request_id = try EchoDriver.run(bc, page.frame_id, "12345678", 4);
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &request_id },
+ });
+ try ctx.expectSentResult(.{ .body = "", .base64Encoded = false }, .{ .id = 2 });
+ try testing.expectEqual(0, bc.captured_responses_size);
+}
+
test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {
var ctx = try testing.context();
defer ctx.deinit();
@@ -1389,8 +1410,8 @@ test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {
});
try ctx.expectSentResult(null, .{ .id = 1 });
- const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa");
- const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb");
+ const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa", null);
+ const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb", null);
try ctx.processMessage(.{
.id = 2,
@@ -1435,7 +1456,7 @@ test "cdp.Network: enable maxPostDataSize omits inline postData" {
try ctx.expectSentResult(null, .{ .id = 1 });
const body = "{\"source\":\"xhr\",\"pageSize\":100}";
- const request_id = try EchoDriver.run(bc, page.frame_id, body);
+ const request_id = try EchoDriver.run(bc, page.frame_id, body, null);
try ctx.expectSentEvent("Network.requestWillBeSent", .{
.requestId = &request_id,
diff --git a/src/testing.zig b/src/testing.zig
index 0baa33751..949a42e2a 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1105,11 +1105,11 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
}
// Bodies are non-empty so that libcurl always reaches the write callback,
- // which is where a headers_only request decides whether to drain or abort.
+ // which is where a partial request decides whether to drain or abort.
// ok.png takes the abort branch, small.png the drain branch; both must
// behave identically as far as the DOM is concerned.
if (std.mem.eql(u8, path, "/images/ok.png")) {
- // > HttpClient.Request.HEADERS_ONLY_DRAIN_MAX. The synthetic PNG
+ // > HttpClient.Request.PARTIAL_DRAIN_MAX. The synthetic PNG
// header advertises 1000 x 750 pixels; no bitmap is decoded.
const body = try arena_allocator.alloc(u8, 16 * 1024 + 1);
@memset(body, 'x');