Lightpanda: the headless browser designed for AI and automation
Improve CDP response event data
Karl Seguin5acfd3825053
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 7e13f3db3..27312ab43 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1024,7 +1024,9 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {// Two-phase variant; see HttpClient.newRequest for the ownership contract.pub fn newRequest(self: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {- return self._session.browser.http_client.newRequest(req, &self._http_owner);+ var r = req;+ r.document_frame_id = self._frame_id;+ return self._session.browser.http_client.newRequest(r, &self._http_owner);}// Synchronously abort every transfer and WebSocket owned by this framediff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zigindex f9282718c..f6d7d2f8b 100644--- a/src/browser/webapi/WorkerGlobalScope.zig+++ b/src/browser/webapi/WorkerGlobalScope.zig@@ -262,7 +262,9 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {// Two-phase variant; see HttpClient.newRequest for the ownership contract.pub fn newRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !*HttpClient.Transfer {- return self._session.browser.http_client.newRequest(req, &self._http_owner);+ var r = req;+ r.document_frame_id = self._frame._frame_id;+ return self._session.browser.http_client.newRequest(r, &self._http_owner);}pub fn getSelf(self: *WorkerGlobalScope) *WorkerGlobalScope {@@ -399,6 +401,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !.url = resolved_url,.method = .GET,.frame_id = self._frame_id,+ .document_frame_id = self._frame._frame_id,.loader_id = self._loader_id,.headers = headers,.cookie_jar = &session.cookie_jar,diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 744c8856a..508f29c80 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -344,17 +344,8 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Requesconst transfer = msg.transfer;const req = &transfer.req;- const frame_id = req.frame_id;- // Worker requests use their WorkerGlobalScope id rather than a document- // frame id. They still belong to this page session and must be visible to- // Network clients; otherwise responseReceived/loadingFinished are emitted- // without the corresponding requestWillBeSent event and CDP clients drop- // the whole request. A worker has no document URL, so use its request URL- // when the id does not resolve to a document frame.- const document_url = if (bc.session.findFrameByFrameId(frame_id)) |frame|- frame.url- else- req.url;+ const frame_id = req.document_frame_id orelse req.frame_id;+ const frame = bc.session.findFrameByFrameId(frame_id) orelse return;// Modify request with extra CDP headers. Use set (replace by name) so a// caller-supplied header overrides a built-in default of the same name@@ -369,7 +360,7 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques.requestId = &id.toRequestId(transfer),.loaderId = &id.toLoaderId(req.loader_id),.type = req.resource_type.string(),- .documentURL = document_url,+ .documentURL = frame.url,.request = RequestWriter.init(transfer),.initiator = .{ .type = "other" },.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo@@ -389,7 +380,7 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c// We're missing a bunch of fields, but, for now, this seems like enoughtry bc.cdp.sendEvent("Network.responseReceived", .{- .frameId = &id.toFrameId(req.frame_id),+ .frameId = &id.toFrameId(req.document_frame_id orelse req.frame_id),.requestId = &id.toRequestId(transfer),.loaderId = &id.toLoaderId(req.loader_id),.timestamp = timestamp(.monotonic),@@ -462,12 +453,13 @@ pub const RequestWriter = struct {{try jws.objectField("initialPriority");- try jws.write("High");+ try jws.write(initialPriority(request.resource_type));}{+ // TODO implement proper referrerPolicytry jws.objectField("referrerPolicy");- try jws.write("strict-origin-when-cross-origin");+ try jws.write("unsafe-url");}{@@ -541,13 +533,15 @@ const ResponseWriter = struct {{try jws.objectField("connectionReused");- try jws.write(false);+ try jws.write(transfer._conn_reused);try jws.objectField("connectionId");- try jws.write(0);+ try jws.write(transfer._conn_id);+ // Bytes received so far, which is zero for a streaming response:+ // its headers are reported before any of the body is read.try jws.objectField("encodedDataLength");- try jws.write(transfer._cdp_content_length);+ try jws.write(transfer._content_length);try jws.objectField("securityState");- try jws.write("unknown");+ try jws.write(securityState(transfer.req.url));}{@@ -567,12 +561,14 @@ const ResponseWriter = struct {.sendStart = -1,.sslEnd = -1,.sslStart = -1,+ // -1 is Chrome's "no service worker"; the push pair is 0 when+ // nothing was pushed, not -1..workerStart = -1,.workerReady = -1,.workerFetchStart = -1,.workerRespondWithSettled = -1,- .pushStart = -1,- .pushEnd = -1,+ .pushStart = 0,+ .pushEnd = 0,});}@@ -606,6 +602,23 @@ const ResponseWriter = struct {}};+fn initialPriority(resource_type: HttpClient.Request.ResourceType) []const u8 {+ return switch (resource_type) {+ .document, .stylesheet => "VeryHigh",+ .script, .xhr, .fetch, .eventsource => "High",+ };+}++fn securityState(url: [:0]const u8) []const u8 {+ if (URL.isSecure(url)) {+ return "secure";+ }+ if (std.mem.startsWith(u8, url, "http:") or std.mem.startsWith(u8, url, "ws:")) {+ return "insecure";+ }+ return "unknown";+}+fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedResponseKey {const key = std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;@@ -1077,40 +1090,46 @@ test "cdp.Network: worker requests emit network events" {try ctx.expectSentResult(null, .{ .id = 1 });const fixture_root = "http://127.0.0.1:9582/src/browser/tests/cdp/";+ const page_url = fixture_root ++ "worker_network.html";const worker_url = fixture_root ++ "worker_network.js";const api_url = "http://127.0.0.1:9582/echo_method";const page = try bc.session.createPage();- try page.navigate(fixture_root ++ "worker_network.html", .{});+ try page.navigate(page_url, .{});try testing.waitForPage(bc);+ // Both the worker's script fetch and the fetch the worker itself issues are+ // attributed to the document that created the worker, not to the worker's+ // own frame id, which no client can resolve.try ctx.expectSentEvent("Network.requestWillBeSent", .{- .documentURL = worker_url,+ .documentURL = page_url,.type = "Script",.request = .{.url = worker_url,.initialPriority = "High",- .referrerPolicy = "strict-origin-when-cross-origin",+ .referrerPolicy = "unsafe-url",},}, .{ .session_id = "SID-NW" });try ctx.expectSentEvent("Network.responseReceived", .{.type = "Script",.response = .{.url = worker_url,- .connectionReused = false,- .connectionId = 0,- .securityState = "unknown",+ .securityState = "insecure",+ // The document was fetched from this origin a moment ago, so the+ // worker's script comes off the same socket. The id itself is a+ // process-wide libcurl counter, so it isn't assertable here.+ .connectionReused = true,.timing = .{.workerStart = -1,.workerReady = -1,.workerFetchStart = -1,.workerRespondWithSettled = -1,- .pushStart = -1,- .pushEnd = -1,+ .pushStart = 0,+ .pushEnd = 0,},},}, .{ .session_id = "SID-NW" });try ctx.expectSentEvent("Network.requestWillBeSent", .{- .documentURL = api_url,+ .documentURL = page_url,.type = "Fetch",.request = .{ .url = api_url },}, .{ .session_id = "SID-NW" });diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 543aa891b..bca67a13b 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1543,7 +1543,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T}self.cacheStore(transfer);- transfer._cdp_content_length = transfer.getContentLength() orelse 0;+ transfer._content_length = transfer.getContentLength() orelse 0;try transfer.bufferEvents(transfer.res.buffer.items);return true;}@@ -1644,6 +1644,10 @@ pub const Request = struct {timeout_ms: u32 = 0,skip_cache: bool = false,+ // The document frame this request belongs to, for CDP attribution.+ // This will be different than frame_id for Workers.+ document_frame_id: ?u32 = null,+// Requests that are internal to the browser and skip various layers,// these do not need to be deferred and do not obey robots.txt.internal: bool = false,@@ -1943,7 +1947,10 @@ pub const Transfer = struct {_cache_key: [:0]const u8 = "",// Content length reported on the CDP loadingFinished event.- _cdp_content_length: usize = 0,+ _content_length: usize = 0,++ _conn_id: i64 = 0,+ _conn_reused: bool = false,// Set by the first deinit. A retired transfer is unlinked from// everything and sits on client.graveyard@@ -2374,7 +2381,7 @@ pub const Transfer = struct {self.setResponseHead(cached.metadata.status, cached.metadata.content_type);self.res.headers = cached.metadata.headers;self._from_cache = true;- self._cdp_content_length = body.len;+ self._content_length = body.len;try self.bufferEvents(body);}@@ -2400,7 +2407,7 @@ pub const Transfer = struct {self.setResponseHead(status, content_type);self.res.headers = owned;- self._cdp_content_length = owned_body.len;+ self._content_length = owned_body.len;try self.bufferEvents(owned_body);}@@ -2412,6 +2419,13 @@ pub const Transfer = struct {// Streaming: already materialized at the first delivered chunk.return;}++ // libcurl returns -1 if the connection wasn't used. We want to avoid+ // negative, so -1 -> 0 and everything else += 1;+ const conn_id = conn.getConnId() catch -1;+ self._conn_id = if (conn_id < 0) 0 else conn_id + 1;+ self._conn_reused = conn.isConnReused() catch false;+const arena = self.arena;if (self.res.header == null) {@@ -2913,7 +2927,7 @@ pub const Transfer = struct {if (transfer._notify_cdp) {req.notification.dispatch(.http_request_done, &.{.transfer = transfer,- .content_length = transfer._cdp_content_length,+ .content_length = transfer._content_length,});}req.done_callback(req.ctx) catch |err| {@@ -3045,7 +3059,7 @@ const Synthetic = struct {h[0] = .{ .name = "content-type", .value = content_type };transfer.res.headers = h;}- transfer._cdp_content_length = body.len;+ transfer._content_length = body.len;try transfer.bufferEvents(body);}};diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex fea5fae90..10cc175a8 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -136,6 +136,7 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans.internal = true,.resource_type = .fetch,.frame_id = transfer.req.frame_id,+ .document_frame_id = transfer.req.document_frame_id,.loader_id = transfer.req.loader_id,.notification = transfer.req.notification,.cookie_jar = null,diff --git a/src/network/http.zig b/src/network/http.zigindex 85edfbff3..bbcc7e746 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -622,6 +622,19 @@ pub const Connection = struct {return @intCast(count);}+ // -1 when the transfer used no connection.+ pub fn getConnId(self: *const Connection) !c_long {+ var conn_id: c_long = undefined;+ try libcurl.curl_easy_getinfo(self._easy, .conn_id, &conn_id);+ return conn_id;+ }++ pub fn isConnReused(self: *const Connection) !bool {+ var opened: c_long = undefined;+ try libcurl.curl_easy_getinfo(self._easy, .num_connects, &opened);+ return opened == 0;+ }+// Total transfer time (name lookup to completion) in microseconds.pub fn getTotalTimeMicros(self: *const Connection) !c_long {var micros: c_long = undefined;diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zigindex ae74f82e7..347fa6c27 100644--- a/src/sys/libcurl.zig+++ b/src/sys/libcurl.zig@@ -246,6 +246,8 @@ pub const CurlInfo = enum(c.CURLINFO) {response_code = c.CURLINFO_RESPONSE_CODE,connect_code = c.CURLINFO_HTTP_CONNECTCODE,total_time_t = c.CURLINFO_TOTAL_TIME_T,+ num_connects = c.CURLINFO_NUM_CONNECTS,+ conn_id = c.CURLINFO_CONN_ID,};pub const Error = error{@@ -661,11 +663,14 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err.response_code,.connect_code,.redirect_count,+ .num_connects,=> blk: {const p: *c_long = out;break :blk c.curl_easy_getinfo(easy, inf, p);},- .total_time_t => blk: {+ .total_time_t,+ .conn_id,+ => blk: {const p: *c.curl_off_t = out;break :blk c.curl_easy_getinfo(easy, inf, p);},