Lightpanda: the headless browser designed for AI and automation
cdp: Network.loadingFinished reports the bytes received for the response
Lucien Coffed8fa225b8e43
diff --git a/src/Notification.zig b/src/Notification.zigindex 47fdb76c6..2df53d6c0 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -266,7 +266,7 @@ pub const ResponseHeaderDone = struct {pub const RequestDone = struct {transfer: *Transfer,- content_length: usize,+ encoded_data_length: usize,};pub const RequestFail = struct {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 0acb6fd13..23fb2025d 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1969,6 +1969,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T}try transfer.materializeResponse(msg.conn, .{});+ transfer.captureEncodedDataLength(msg.conn);if (self.enforceCorsResponse(msg, transfer)) return true;// Latency is only meaningful for responses that hit the network (cache@@ -2676,9 +2677,14 @@ pub const Transfer = struct {// rewrote it). Only meaningful while _cache_intent != .none._cache_key: [:0]const u8 = "",- // Content length reported on the CDP loadingFinished event.+ // Content length reported on the CDP responseReceived event, and the+ // resource-timing size of a partial fetch._content_length: usize = 0,+ // encodedDataLength of the CDP loadingFinished event: the header and body+ // bytes received, the body still compressed.+ _encoded_data_length: usize = 0,+// Length of the body data_callback will receive. Can be different than// Content-Length if the content was compressed_body_len: usize = 0,@@ -3099,6 +3105,7 @@ pub const Transfer = struct {}self.materializeResponse(conn, .{}) catch return false;+ self.captureEncodedDataLength(conn);self.client.removeConn(conn);self._conn = null;self._content_length = self.res.buffer.items.len;@@ -3343,6 +3350,12 @@ pub const Transfer = struct {self.client.dispatch_count += 1;}+ fn captureEncodedDataLength(self: *Transfer, conn: *const http.Connection) void {+ const header_size = conn.getHeaderSize() catch 0;+ const body_size = conn.getDownloadSize() catch 0;+ self._encoded_data_length = @intCast(header_size + body_size + self.res.partial_refused);+ }+fn captureTiming(self: *Transfer, conn: *const http.Connection) void {const t = &self._timing;const ct = conn.getTiming() catch return;@@ -3619,6 +3632,10 @@ pub const Transfer = struct {self._from_cache = true;self._timing.cache = cache_state;self._content_length = body.len;+ // A revalidated entry keeps the size of the 304 that crossed the wire.+ if (cache_state == .local) {+ self._encoded_data_length = body.len;+ }return body;}@@ -3649,6 +3666,7 @@ pub const Transfer = struct {// It may replace a cached response paused at the response stage.self._from_cache = false;self._content_length = owned_body.len;+ self._encoded_data_length = owned_body.len;try self.bufferEvents(owned_body);}@@ -4309,6 +4327,7 @@ pub const Transfer = struct {// flag and treats the resulting CURLE_WRITE_ERROR as a completed// response with the kept prefix as its body.res.partial_abort = true;+ res.partial_refused = chunk_len;return http.writefunc_error;}@@ -4553,7 +4572,7 @@ pub const Transfer = struct {if (transfer._notify_cdp) {transfer.notify(.http_request_done, &.{.transfer = transfer,- .content_length = transfer._content_length,+ .encoded_data_length = transfer._encoded_data_length,});}// Before done_callback: a load handler can already see@@ -4674,6 +4693,9 @@ const Response = struct {// status, kept prefix) as a success.partial_abort: bool = false,+ // The read refused to end a partial fetch: libcurl doesn't count it.+ partial_refused: usize = 0,+// Response body. Filled by dataCallback, consumed in processMessages.// See Stream.spare to see how this works in streaming modebuffer: std.ArrayList(u8) = .empty,@@ -4768,6 +4790,7 @@ const Synthetic = struct {transfer.setResponseHeaders(h);}transfer._content_length = body.len;+ transfer._encoded_data_length = body.len;try transfer.bufferEvents(body);}};@@ -6477,3 +6500,68 @@ test "HttpClient: bodyless status announcing a body" {try testing.expectEqual(false, Transfer.announcesBody("Content-Type: text/html\r\n"));try testing.expectEqual(false, Transfer.announcesBody("\r\n"));}++// Fetches `url` with the test browser's client and returns the+// encodedDataLength it reports in http_request_done (CDP's loadingFinished).+const EncodedLength = struct {+ reported: ?usize = null,+ bytes_received: usize = 0,++ fn onRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {+ const self: *EncodedLength = @ptrCast(@alignCast(ctx));+ self.reported = msg.encoded_data_length;+ self.bytes_received = msg.transfer.res.bytes_received;+ }++ fn fetch(self: *EncodedLength, url: [:0]const u8, partial: ?u32) !usize {+ const notification = try Notification.init(testing.allocator);+ defer notification.deinit();+ try notification.register(.http_request_done, self, onRequestDone);++ const client = &testing.test_browser.http_client;+ self.reported = null;+ try client.request(.{+ .method = .GET,+ .url = url,+ .origin = null,+ .credentials_mode = .omit,+ .request_mode = .no_cors,+ .resource_type = .xhr,+ .partial = partial,+ .notification = notification,+ .shutdown_callback = noopShutdown,+ }, null);+ for (0..250) |_| {+ if (self.reported != null) break;+ _ = try client.tick(20);+ }+ // The delivering tick retired the transfer; the next one releases it.+ _ = try client.tick(0);+ return self.reported orelse error.NotReported;+ }+};++test "HttpClient: encodedDataLength of an early-stopped partial fetch is the bytes that arrived" {+ const large = testing.raw_responses.large;+ var e: EncodedLength = .{};+ const reported = try e.fetch("http://127.0.0.1:9582/raw/large", 16 * 1024);+ try testing.expect(e.bytes_received < large.body.len);+ try testing.expectEqual(large.head.len + e.bytes_received, reported);+}++test "HttpClient: encodedDataLength of a cache hit is the stored body, of a revalidation the 304's bytes" {+ const client = &testing.test_browser.http_client;+ var cache: Cache = .{ .kind = .{ .sqlite = try @import("cache/SqliteCache.zig").init(testing.allocator, .memory, 0) } };+ defer cache.deinit();+ const previous = client.cache;+ client.cache = &cache;+ defer client.cache = previous;++ const raw = testing.raw_responses;+ var e: EncodedLength = .{};+ _ = try e.fetch("http://127.0.0.1:9582/raw/cacheable", null);+ try testing.expectEqual(raw.cacheable.body.len, try e.fetch("http://127.0.0.1:9582/raw/cacheable", null));++ _ = try e.fetch("http://127.0.0.1:9582/raw/revalidated", null);+ try testing.expectEqual(raw.not_modified.head.len, try e.fetch("http://127.0.0.1:9582/raw/revalidated", null));+}diff --git a/src/network/http.zig b/src/network/http.zigindex d6f840f84..01a03cec7 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -639,6 +639,13 @@ pub const Connection = struct {return @intCast(@max(0, size));}+ // Bytes of every response header received, status lines included.+ pub fn getHeaderSize(self: *const Connection) !u64 {+ var size: c_long = undefined;+ try libcurl.curl_easy_getinfo(self._easy, .header_size, &size);+ return @intCast(@max(0, size));+ }+pub fn getHttpVersion(self: *const Connection) !libcurl.CurlHttpVersion {var version: c_long = undefined;try libcurl.curl_easy_getinfo(self._easy, .http_version, &version);diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex 3ccb16250..2b5d2d885 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -559,7 +559,7 @@ pub fn httpRequestDone(bc: *CDP.BrowserContext, msg: *const Notification.Requesttry bc.cdp.sendEvent("Network.loadingFinished", .{.requestId = &id.toRequestId(msg.transfer),.timestamp = lp.datetime.timestamp(.boot),- .encodedDataLength = msg.content_length,+ .encodedDataLength = msg.encoded_data_length,}, .{ .session_id = session_id });}@@ -1386,6 +1386,37 @@ test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {try testing.expectEqual(error.UrlBlocked, error_context.err.?);}+test "cdp.Network: loadingFinished reports the bytes received, not the decoded body" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-ED", .session_id = "SID-ED" });+ const page = try bc.session.createPage();+ const client = &bc.cdp.browser.http_client;++ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });+ try ctx.expectSentResult(null, .{ .id = 1 });++ try client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 1,+ .method = .GET,+ .url = "http://127.0.0.1:9582/raw/gzip",+ .origin = bc.security_origin,+ .credentials_mode = .omit,+ .request_mode = .no_cors,+ .resource_type = .fetch,+ .notification = bc.session.notification,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ // The header block and the 29 gzipped bytes, not the 1000 decoded ones.+ const r = testing.base.raw_responses.gzip;+ try ctx.expectSentEvent("Network.loadingFinished", .{+ .encodedDataLength = r.head.len + r.body.len,+ }, .{ .session_id = "SID-ED" });+}+test "cdp.Network: POST body exposed as postData" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zigindex 29e74e40f..30d507d7b 100644--- a/src/sys/libcurl.zig+++ b/src/sys/libcurl.zig@@ -264,6 +264,7 @@ pub const CurlInfo = enum(c.CURLINFO) {pretransfer_time_t = c.CURLINFO_PRETRANSFER_TIME_T,starttransfer_time_t = c.CURLINFO_STARTTRANSFER_TIME_T,size_download_t = c.CURLINFO_SIZE_DOWNLOAD_T,+ header_size = c.CURLINFO_HEADER_SIZE,http_version = c.CURLINFO_HTTP_VERSION,num_connects = c.CURLINFO_NUM_CONNECTS,conn_id = c.CURLINFO_CONN_ID,@@ -685,6 +686,7 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err.redirect_count,.num_connects,.http_version,+ .header_size,=> blk: {const p: *c_long = out;break :blk c.curl_easy_getinfo(easy, inf, p);diff --git a/src/test_routes.zig b/src/test_routes.zigindex 2eef54c43..d6882d637 100644--- a/src/test_routes.zig+++ b/src/test_routes.zig@@ -43,6 +43,56 @@ var serve_counts = [_]struct { name: []const u8, count: u32 = 0 }{.{ .name = "prescan_module" },};+// Written byte for byte by /raw/<name>, so the encodedDataLength tests know+// how many header and body bytes crossed the wire. A request carrying+// If-None-Match gets `not_modified`.+const RawResponse = struct { head: []const u8, body: []const u8 };++pub const raw_responses = struct {+ // 1000 'a's, gzipped.+ pub const gzip: RawResponse = .{+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\nContent-Length: 29\r\n\r\n",+ .body = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4b\x4c\x1c\x05\xa3\x60\x14\x0c\x77\x00\x00\x03\xda\x38\x9a\xe8\x03\x00\x00",+ };+ pub const cacheable: RawResponse = .{+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: max-age=600\r\nContent-Length: 11\r\n\r\n",+ .body = "hello world",+ };+ pub const revalidated: RawResponse = .{+ .head = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: no-cache\r\nETag: \"v1\"\r\nContent-Length: 11\r\n\r\n",+ .body = "hello world",+ };+ pub const not_modified: RawResponse = .{+ .head = "HTTP/1.1 304 Not Modified\r\nCache-Control: no-cache\r\nETag: \"v1\"\r\n\r\n",+ .body = "",+ };+ // Far more than a partial fetch reads before it stops.+ pub const large: RawResponse = .{+ .head = "HTTP/1.1 200 OK\r\nContent-Length: 262144\r\n\r\n",+ .body = &@as([256 * 1024]u8, @splat('a')),+ };+};++fn serveRaw(req: *std.http.Server.Request, name: []const u8) !void {+ // Before writing: the server's read and write buffers are the same memory.+ const conditional = std.mem.indexOf(u8, req.head_buffer, "If-None-Match:") != null;+ const r: RawResponse = if (conditional)+ raw_responses.not_modified+ else if (std.mem.eql(u8, name, "gzip"))+ raw_responses.gzip+ else if (std.mem.eql(u8, name, "cacheable"))+ raw_responses.cacheable+ else if (std.mem.eql(u8, name, "revalidated"))+ raw_responses.revalidated+ else if (std.mem.eql(u8, name, "large"))+ raw_responses.large+ else+ return req.respond("unknown raw response", .{ .status = .not_found });+ try req.server.out.writeAll(r.head);+ try req.server.out.writeAll(r.body);+ return req.server.out.flush();+}+// /beacon/store/<id> records the request it gets, /beacon/read/<id> answers// "METHOD|content-type|body" for it, or "(missing)". The test HTTP server runs// a thread per connection, so access goes through beacon_mutex.@@ -983,6 +1033,10 @@ pub fn handle(req: *std.http.Server.Request) !void {return TestHTTPServer.sendFile(req, path[1..]);}+ if (std.mem.startsWith(u8, path, "/raw/")) {+ return serveRaw(req, path["/raw/".len..]);+ }+if (std.mem.eql(u8, path, "/favicon.ico")) {return req.respond("", .{ .status = .not_found });}diff --git a/src/testing.zig b/src/testing.zigindex 5c5dc7ecd..3cfeea0f2 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -522,6 +522,7 @@ const Server = @import("server/Server.zig");const TestWSServer = @import("TestWSServer.zig");const TestHTTPServer = @import("TestHTTPServer.zig");const test_routes = @import("test_routes.zig");+pub const raw_responses = test_routes.raw_responses;pub var test_cdp_server: ?*Server = null;pub var test_cdp_port: u16 = 0;