Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: Network.loadingFinished reports the bytes received for the response

Lucien Coffed8fa225b8e43
diff --git a/src/Notification.zig b/src/Notification.zig
index 47fdb76c6..2df53d6c0 100644
--- a/src/Notification.zig
+++ b/src/Notification.zig
@@ -266,7 +266,7 @@ pub const ResponseHeaderDone = struct {
pub const RequestDone = struct {
transfer: *Transfer,
- content_length: usize,
+ encoded_data_length: usize,
};
pub const RequestFail = struct {
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 0acb6fd13..23fb2025d 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1969,6 +1969,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
}
try transfer.materializeResponse(msg.conn, .{});
+ transfer.captureEncodedDataLength(msg.conn);
if (self.enforceCorsResponse(msg, transfer)) return true;
// Latency is only meaningful for responses that hit the network (cache
@@ -2676,9 +2677,14 @@ pub const Transfer = struct {
// rewrote it). Only meaningful while _cache_intent != .none.
_cache_key: [:0]const u8 = "",
- // Content length reported on the CDP loadingFinished event.
+ // Content length reported on the CDP responseReceived event, and the
+ // resource-timing size of a partial fetch.
_content_length: usize = 0,
+ // encodedDataLength of the CDP loadingFinished event: the header and body
+ // bytes received, the body still compressed.
+ _encoded_data_length: usize = 0,
+
// Length of the body data_callback will receive. Can be different than
// Content-Length if the content was compressed
_body_len: usize = 0,
@@ -3099,6 +3105,7 @@ pub const Transfer = struct {
}
self.materializeResponse(conn, .{}) catch return false;
+ self.captureEncodedDataLength(conn);
self.client.removeConn(conn);
self._conn = null;
self._content_length = self.res.buffer.items.len;
@@ -3343,6 +3350,12 @@ pub const Transfer = struct {
self.client.dispatch_count += 1;
}
+ fn captureEncodedDataLength(self: *Transfer, conn: *const http.Connection) void {
+ const header_size = conn.getHeaderSize() catch 0;
+ const body_size = conn.getDownloadSize() catch 0;
+ self._encoded_data_length = @intCast(header_size + body_size + self.res.partial_refused);
+ }
+
fn captureTiming(self: *Transfer, conn: *const http.Connection) void {
const t = &self._timing;
const ct = conn.getTiming() catch return;
@@ -3619,6 +3632,10 @@ pub const Transfer = struct {
self._from_cache = true;
self._timing.cache = cache_state;
self._content_length = body.len;
+ // A revalidated entry keeps the size of the 304 that crossed the wire.
+ if (cache_state == .local) {
+ self._encoded_data_length = body.len;
+ }
return body;
}
@@ -3649,6 +3666,7 @@ pub const Transfer = struct {
// It may replace a cached response paused at the response stage.
self._from_cache = false;
self._content_length = owned_body.len;
+ self._encoded_data_length = owned_body.len;
try self.bufferEvents(owned_body);
}
@@ -4309,6 +4327,7 @@ pub const Transfer = struct {
// flag and treats the resulting CURLE_WRITE_ERROR as a completed
// response with the kept prefix as its body.
res.partial_abort = true;
+ res.partial_refused = chunk_len;
return http.writefunc_error;
}
@@ -4553,7 +4572,7 @@ pub const Transfer = struct {
if (transfer._notify_cdp) {
transfer.notify(.http_request_done, &.{
.transfer = transfer,
- .content_length = transfer._content_length,
+ .encoded_data_length = transfer._encoded_data_length,
});
}
// Before done_callback: a load handler can already see
@@ -4674,6 +4693,9 @@ const Response = struct {
// status, kept prefix) as a success.
partial_abort: bool = false,
+ // The read refused to end a partial fetch: libcurl doesn't count it.
+ partial_refused: usize = 0,
+
// Response body. Filled by dataCallback, consumed in processMessages.
// See Stream.spare to see how this works in streaming mode
buffer: std.ArrayList(u8) = .empty,
@@ -4768,6 +4790,7 @@ const Synthetic = struct {
transfer.setResponseHeaders(h);
}
transfer._content_length = body.len;
+ transfer._encoded_data_length = body.len;
try transfer.bufferEvents(body);
}
};
@@ -6477,3 +6500,68 @@ test "HttpClient: bodyless status announcing a body" {
try testing.expectEqual(false, Transfer.announcesBody("Content-Type: text/html\r\n"));
try testing.expectEqual(false, Transfer.announcesBody("\r\n"));
}
+
+// Fetches `url` with the test browser's client and returns the
+// encodedDataLength it reports in http_request_done (CDP's loadingFinished).
+const EncodedLength = struct {
+ reported: ?usize = null,
+ bytes_received: usize = 0,
+
+ fn onRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {
+ const self: *EncodedLength = @ptrCast(@alignCast(ctx));
+ self.reported = msg.encoded_data_length;
+ self.bytes_received = msg.transfer.res.bytes_received;
+ }
+
+ fn fetch(self: *EncodedLength, url: [:0]const u8, partial: ?u32) !usize {
+ const notification = try Notification.init(testing.allocator);
+ defer notification.deinit();
+ try notification.register(.http_request_done, self, onRequestDone);
+
+ const client = &testing.test_browser.http_client;
+ self.reported = null;
+ try client.request(.{
+ .method = .GET,
+ .url = url,
+ .origin = null,
+ .credentials_mode = .omit,
+ .request_mode = .no_cors,
+ .resource_type = .xhr,
+ .partial = partial,
+ .notification = notification,
+ .shutdown_callback = noopShutdown,
+ }, null);
+ for (0..250) |_| {
+ if (self.reported != null) break;
+ _ = try client.tick(20);
+ }
+ // The delivering tick retired the transfer; the next one releases it.
+ _ = try client.tick(0);
+ return self.reported orelse error.NotReported;
+ }
+};
+
+test "HttpClient: encodedDataLength of an early-stopped partial fetch is the bytes that arrived" {
+ const large = testing.raw_responses.large;
+ var e: EncodedLength = .{};
+ const reported = try e.fetch("http://127.0.0.1:9582/raw/large", 16 * 1024);
+ try testing.expect(e.bytes_received < large.body.len);
+ try testing.expectEqual(large.head.len + e.bytes_received, reported);
+}
+
+test "HttpClient: encodedDataLength of a cache hit is the stored body, of a revalidation the 304's bytes" {
+ const client = &testing.test_browser.http_client;
+ var cache: Cache = .{ .kind = .{ .sqlite = try @import("cache/SqliteCache.zig").init(testing.allocator, .memory, 0) } };
+ defer cache.deinit();
+ const previous = client.cache;
+ client.cache = &cache;
+ defer client.cache = previous;
+
+ const raw = testing.raw_responses;
+ var e: EncodedLength = .{};
+ _ = try e.fetch("http://127.0.0.1:9582/raw/cacheable", null);
+ try testing.expectEqual(raw.cacheable.body.len, try e.fetch("http://127.0.0.1:9582/raw/cacheable", null));
+
+ _ = try e.fetch("http://127.0.0.1:9582/raw/revalidated", null);
+ try testing.expectEqual(raw.not_modified.head.len, try e.fetch("http://127.0.0.1:9582/raw/revalidated", null));
+}
diff --git a/src/network/http.zig b/src/network/http.zig
index d6f840f84..01a03cec7 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -639,6 +639,13 @@ pub const Connection = struct {
return @intCast(@max(0, size));
}
+ // Bytes of every response header received, status lines included.
+ pub fn getHeaderSize(self: *const Connection) !u64 {
+ var size: c_long = undefined;
+ try libcurl.curl_easy_getinfo(self._easy, .header_size, &size);
+ return @intCast(@max(0, size));
+ }
+
pub fn getHttpVersion(self: *const Connection) !libcurl.CurlHttpVersion {
var version: c_long = undefined;
try libcurl.curl_easy_getinfo(self._easy, .http_version, &version);
diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zig
index 3ccb16250..2b5d2d885 100644
--- a/src/server/cdp/domains/network.zig
+++ b/src/server/cdp/domains/network.zig
@@ -559,7 +559,7 @@ pub fn httpRequestDone(bc: *CDP.BrowserContext, msg: *const Notification.Request
try bc.cdp.sendEvent("Network.loadingFinished", .{
.requestId = &id.toRequestId(msg.transfer),
.timestamp = lp.datetime.timestamp(.boot),
- .encodedDataLength = msg.content_length,
+ .encodedDataLength = msg.encoded_data_length,
}, .{ .session_id = session_id });
}
@@ -1386,6 +1386,37 @@ test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {
try testing.expectEqual(error.UrlBlocked, error_context.err.?);
}
+test "cdp.Network: loadingFinished reports the bytes received, not the decoded body" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-ED", .session_id = "SID-ED" });
+ const page = try bc.session.createPage();
+ const client = &bc.cdp.browser.http_client;
+
+ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ try client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 1,
+ .method = .GET,
+ .url = "http://127.0.0.1:9582/raw/gzip",
+ .origin = bc.security_origin,
+ .credentials_mode = .omit,
+ .request_mode = .no_cors,
+ .resource_type = .fetch,
+ .notification = bc.session.notification,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ // The header block and the 29 gzipped bytes, not the 1000 decoded ones.
+ const r = testing.base.raw_responses.gzip;
+ try ctx.expectSentEvent("Network.loadingFinished", .{
+ .encodedDataLength = r.head.len + r.body.len,
+ }, .{ .session_id = "SID-ED" });
+}
+
test "cdp.Network: POST body exposed as postData" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zig
index 29e74e40f..30d507d7b 100644
--- a/src/sys/libcurl.zig
+++ b/src/sys/libcurl.zig
@@ -264,6 +264,7 @@ pub const CurlInfo = enum(c.CURLINFO) {
pretransfer_time_t = c.CURLINFO_PRETRANSFER_TIME_T,
starttransfer_time_t = c.CURLINFO_STARTTRANSFER_TIME_T,
size_download_t = c.CURLINFO_SIZE_DOWNLOAD_T,
+ header_size = c.CURLINFO_HEADER_SIZE,
http_version = c.CURLINFO_HTTP_VERSION,
num_connects = c.CURLINFO_NUM_CONNECTS,
conn_id = c.CURLINFO_CONN_ID,
@@ -685,6 +686,7 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err
.redirect_count,
.num_connects,
.http_version,
+ .header_size,
=> blk: {
const p: *c_long = out;
break :blk c.curl_easy_getinfo(easy, inf, p);
diff --git a/src/test_routes.zig b/src/test_routes.zig
index 2eef54c43..d6882d637 100644
--- a/src/test_routes.zig
+++ b/src/test_routes.zig
@@ -43,6 +43,56 @@ var serve_counts = [_]struct { name: []const u8, count: u32 = 0 }{
.{ .name = "prescan_module" },
};
+// Written byte for byte by /raw/<name>, so the encodedDataLength tests know
+// how many header and body bytes crossed the wire. A request carrying
+// If-None-Match gets `not_modified`.
+const RawResponse = struct { head: []const u8, body: []const u8 };
+
+pub const raw_responses = struct {
+ // 1000 'a's, gzipped.
+ pub const gzip: RawResponse = .{
+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\nContent-Length: 29\r\n\r\n",
+ .body = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4b\x4c\x1c\x05\xa3\x60\x14\x0c\x77\x00\x00\x03\xda\x38\x9a\xe8\x03\x00\x00",
+ };
+ pub const cacheable: RawResponse = .{
+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: max-age=600\r\nContent-Length: 11\r\n\r\n",
+ .body = "hello world",
+ };
+ pub const revalidated: RawResponse = .{
+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: no-cache\r\nETag: \"v1\"\r\nContent-Length: 11\r\n\r\n",
+ .body = "hello world",
+ };
+ pub const not_modified: RawResponse = .{
+ .head = "HTTP/1.1 304 Not Modified\r\nCache-Control: no-cache\r\nETag: \"v1\"\r\n\r\n",
+ .body = "",
+ };
+ // Far more than a partial fetch reads before it stops.
+ pub const large: RawResponse = .{
+ .head = "HTTP/1.1 200 OK\r\nContent-Length: 262144\r\n\r\n",
+ .body = &@as([256 * 1024]u8, @splat('a')),
+ };
+};
+
+fn serveRaw(req: *std.http.Server.Request, name: []const u8) !void {
+ // Before writing: the server's read and write buffers are the same memory.
+ const conditional = std.mem.indexOf(u8, req.head_buffer, "If-None-Match:") != null;
+ const r: RawResponse = if (conditional)
+ raw_responses.not_modified
+ else if (std.mem.eql(u8, name, "gzip"))
+ raw_responses.gzip
+ else if (std.mem.eql(u8, name, "cacheable"))
+ raw_responses.cacheable
+ else if (std.mem.eql(u8, name, "revalidated"))
+ raw_responses.revalidated
+ else if (std.mem.eql(u8, name, "large"))
+ raw_responses.large
+ else
+ return req.respond("unknown raw response", .{ .status = .not_found });
+ try req.server.out.writeAll(r.head);
+ try req.server.out.writeAll(r.body);
+ return req.server.out.flush();
+}
+
// /beacon/store/<id> records the request it gets, /beacon/read/<id> answers
// "METHOD|content-type|body" for it, or "(missing)". The test HTTP server runs
// a thread per connection, so access goes through beacon_mutex.
@@ -983,6 +1033,10 @@ pub fn handle(req: *std.http.Server.Request) !void {
return TestHTTPServer.sendFile(req, path[1..]);
}
+ if (std.mem.startsWith(u8, path, "/raw/")) {
+ return serveRaw(req, path["/raw/".len..]);
+ }
+
if (std.mem.eql(u8, path, "/favicon.ico")) {
return req.respond("", .{ .status = .not_found });
}
diff --git a/src/testing.zig b/src/testing.zig
index 5c5dc7ecd..3cfeea0f2 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -522,6 +522,7 @@ const Server = @import("server/Server.zig");
const TestWSServer = @import("TestWSServer.zig");
const TestHTTPServer = @import("TestHTTPServer.zig");
const test_routes = @import("test_routes.zig");
+pub const raw_responses = test_routes.raw_responses;
pub var test_cdp_server: ?*Server = null;
pub var test_cdp_port: u16 = 0;