Skip to content

Lightpanda: the headless browser designed for AI and automation

Improve CDP response event data

Karl Seguin5acfd3825053
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 7e13f3db3..27312ab43 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1024,7 +1024,9 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {
// Two-phase variant; see HttpClient.newRequest for the ownership contract.
pub fn newRequest(self: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {
- return self._session.browser.http_client.newRequest(req, &self._http_owner);
+ var r = req;
+ r.document_frame_id = self._frame_id;
+ return self._session.browser.http_client.newRequest(r, &self._http_owner);
}
// Synchronously abort every transfer and WebSocket owned by this frame
diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zig
index f9282718c..f6d7d2f8b 100644
--- a/src/browser/webapi/WorkerGlobalScope.zig
+++ b/src/browser/webapi/WorkerGlobalScope.zig
@@ -262,7 +262,9 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {
// Two-phase variant; see HttpClient.newRequest for the ownership contract.
pub fn newRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !*HttpClient.Transfer {
- return self._session.browser.http_client.newRequest(req, &self._http_owner);
+ var r = req;
+ r.document_frame_id = self._frame._frame_id;
+ return self._session.browser.http_client.newRequest(r, &self._http_owner);
}
pub fn getSelf(self: *WorkerGlobalScope) *WorkerGlobalScope {
@@ -399,6 +401,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
.url = resolved_url,
.method = .GET,
.frame_id = self._frame_id,
+ .document_frame_id = self._frame._frame_id,
.loader_id = self._loader_id,
.headers = headers,
.cookie_jar = &session.cookie_jar,
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 744c8856a..508f29c80 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -344,17 +344,8 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
const transfer = msg.transfer;
const req = &transfer.req;
- const frame_id = req.frame_id;
- // Worker requests use their WorkerGlobalScope id rather than a document
- // frame id. They still belong to this page session and must be visible to
- // Network clients; otherwise responseReceived/loadingFinished are emitted
- // without the corresponding requestWillBeSent event and CDP clients drop
- // the whole request. A worker has no document URL, so use its request URL
- // when the id does not resolve to a document frame.
- const document_url = if (bc.session.findFrameByFrameId(frame_id)) |frame|
- frame.url
- else
- req.url;
+ const frame_id = req.document_frame_id orelse req.frame_id;
+ const frame = bc.session.findFrameByFrameId(frame_id) orelse return;
// Modify request with extra CDP headers. Use set (replace by name) so a
// caller-supplied header overrides a built-in default of the same name
@@ -369,7 +360,7 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.loader_id),
.type = req.resource_type.string(),
- .documentURL = document_url,
+ .documentURL = frame.url,
.request = RequestWriter.init(transfer),
.initiator = .{ .type = "other" },
.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo
@@ -389,7 +380,7 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c
// We're missing a bunch of fields, but, for now, this seems like enough
try bc.cdp.sendEvent("Network.responseReceived", .{
- .frameId = &id.toFrameId(req.frame_id),
+ .frameId = &id.toFrameId(req.document_frame_id orelse req.frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.loader_id),
.timestamp = timestamp(.monotonic),
@@ -462,12 +453,13 @@ pub const RequestWriter = struct {
{
try jws.objectField("initialPriority");
- try jws.write("High");
+ try jws.write(initialPriority(request.resource_type));
}
{
+ // TODO implement proper referrerPolicy
try jws.objectField("referrerPolicy");
- try jws.write("strict-origin-when-cross-origin");
+ try jws.write("unsafe-url");
}
{
@@ -541,13 +533,15 @@ const ResponseWriter = struct {
{
try jws.objectField("connectionReused");
- try jws.write(false);
+ try jws.write(transfer._conn_reused);
try jws.objectField("connectionId");
- try jws.write(0);
+ try jws.write(transfer._conn_id);
+ // Bytes received so far, which is zero for a streaming response:
+ // its headers are reported before any of the body is read.
try jws.objectField("encodedDataLength");
- try jws.write(transfer._cdp_content_length);
+ try jws.write(transfer._content_length);
try jws.objectField("securityState");
- try jws.write("unknown");
+ try jws.write(securityState(transfer.req.url));
}
{
@@ -567,12 +561,14 @@ const ResponseWriter = struct {
.sendStart = -1,
.sslEnd = -1,
.sslStart = -1,
+ // -1 is Chrome's "no service worker"; the push pair is 0 when
+ // nothing was pushed, not -1.
.workerStart = -1,
.workerReady = -1,
.workerFetchStart = -1,
.workerRespondWithSettled = -1,
- .pushStart = -1,
- .pushEnd = -1,
+ .pushStart = 0,
+ .pushEnd = 0,
});
}
@@ -606,6 +602,23 @@ const ResponseWriter = struct {
}
};
+fn initialPriority(resource_type: HttpClient.Request.ResourceType) []const u8 {
+ return switch (resource_type) {
+ .document, .stylesheet => "VeryHigh",
+ .script, .xhr, .fetch, .eventsource => "High",
+ };
+}
+
+fn securityState(url: [:0]const u8) []const u8 {
+ if (URL.isSecure(url)) {
+ return "secure";
+ }
+ if (std.mem.startsWith(u8, url, "http:") or std.mem.startsWith(u8, url, "ws:")) {
+ return "insecure";
+ }
+ return "unknown";
+}
+
fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedResponseKey {
const key = std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;
@@ -1077,40 +1090,46 @@ test "cdp.Network: worker requests emit network events" {
try ctx.expectSentResult(null, .{ .id = 1 });
const fixture_root = "http://127.0.0.1:9582/src/browser/tests/cdp/";
+ const page_url = fixture_root ++ "worker_network.html";
const worker_url = fixture_root ++ "worker_network.js";
const api_url = "http://127.0.0.1:9582/echo_method";
const page = try bc.session.createPage();
- try page.navigate(fixture_root ++ "worker_network.html", .{});
+ try page.navigate(page_url, .{});
try testing.waitForPage(bc);
+ // Both the worker's script fetch and the fetch the worker itself issues are
+ // attributed to the document that created the worker, not to the worker's
+ // own frame id, which no client can resolve.
try ctx.expectSentEvent("Network.requestWillBeSent", .{
- .documentURL = worker_url,
+ .documentURL = page_url,
.type = "Script",
.request = .{
.url = worker_url,
.initialPriority = "High",
- .referrerPolicy = "strict-origin-when-cross-origin",
+ .referrerPolicy = "unsafe-url",
},
}, .{ .session_id = "SID-NW" });
try ctx.expectSentEvent("Network.responseReceived", .{
.type = "Script",
.response = .{
.url = worker_url,
- .connectionReused = false,
- .connectionId = 0,
- .securityState = "unknown",
+ .securityState = "insecure",
+ // The document was fetched from this origin a moment ago, so the
+ // worker's script comes off the same socket. The id itself is a
+ // process-wide libcurl counter, so it isn't assertable here.
+ .connectionReused = true,
.timing = .{
.workerStart = -1,
.workerReady = -1,
.workerFetchStart = -1,
.workerRespondWithSettled = -1,
- .pushStart = -1,
- .pushEnd = -1,
+ .pushStart = 0,
+ .pushEnd = 0,
},
},
}, .{ .session_id = "SID-NW" });
try ctx.expectSentEvent("Network.requestWillBeSent", .{
- .documentURL = api_url,
+ .documentURL = page_url,
.type = "Fetch",
.request = .{ .url = api_url },
}, .{ .session_id = "SID-NW" });
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 543aa891b..bca67a13b 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1543,7 +1543,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
}
self.cacheStore(transfer);
- transfer._cdp_content_length = transfer.getContentLength() orelse 0;
+ transfer._content_length = transfer.getContentLength() orelse 0;
try transfer.bufferEvents(transfer.res.buffer.items);
return true;
}
@@ -1644,6 +1644,10 @@ pub const Request = struct {
timeout_ms: u32 = 0,
skip_cache: bool = false,
+ // The document frame this request belongs to, for CDP attribution.
+ // This will be different than frame_id for Workers.
+ document_frame_id: ?u32 = null,
+
// Requests that are internal to the browser and skip various layers,
// these do not need to be deferred and do not obey robots.txt.
internal: bool = false,
@@ -1943,7 +1947,10 @@ pub const Transfer = struct {
_cache_key: [:0]const u8 = "",
// Content length reported on the CDP loadingFinished event.
- _cdp_content_length: usize = 0,
+ _content_length: usize = 0,
+
+ _conn_id: i64 = 0,
+ _conn_reused: bool = false,
// Set by the first deinit. A retired transfer is unlinked from
// everything and sits on client.graveyard
@@ -2374,7 +2381,7 @@ pub const Transfer = struct {
self.setResponseHead(cached.metadata.status, cached.metadata.content_type);
self.res.headers = cached.metadata.headers;
self._from_cache = true;
- self._cdp_content_length = body.len;
+ self._content_length = body.len;
try self.bufferEvents(body);
}
@@ -2400,7 +2407,7 @@ pub const Transfer = struct {
self.setResponseHead(status, content_type);
self.res.headers = owned;
- self._cdp_content_length = owned_body.len;
+ self._content_length = owned_body.len;
try self.bufferEvents(owned_body);
}
@@ -2412,6 +2419,13 @@ pub const Transfer = struct {
// Streaming: already materialized at the first delivered chunk.
return;
}
+
+ // libcurl returns -1 if the connection wasn't used. We want to avoid
+ // negative, so -1 -> 0 and everything else += 1;
+ const conn_id = conn.getConnId() catch -1;
+ self._conn_id = if (conn_id < 0) 0 else conn_id + 1;
+ self._conn_reused = conn.isConnReused() catch false;
+
const arena = self.arena;
if (self.res.header == null) {
@@ -2913,7 +2927,7 @@ pub const Transfer = struct {
if (transfer._notify_cdp) {
req.notification.dispatch(.http_request_done, &.{
.transfer = transfer,
- .content_length = transfer._cdp_content_length,
+ .content_length = transfer._content_length,
});
}
req.done_callback(req.ctx) catch |err| {
@@ -3045,7 +3059,7 @@ const Synthetic = struct {
h[0] = .{ .name = "content-type", .value = content_type };
transfer.res.headers = h;
}
- transfer._cdp_content_length = body.len;
+ transfer._content_length = body.len;
try transfer.bufferEvents(body);
}
};
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index fea5fae90..10cc175a8 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -136,6 +136,7 @@ fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Trans
.internal = true,
.resource_type = .fetch,
.frame_id = transfer.req.frame_id,
+ .document_frame_id = transfer.req.document_frame_id,
.loader_id = transfer.req.loader_id,
.notification = transfer.req.notification,
.cookie_jar = null,
diff --git a/src/network/http.zig b/src/network/http.zig
index 85edfbff3..bbcc7e746 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -622,6 +622,19 @@ pub const Connection = struct {
return @intCast(count);
}
+ // -1 when the transfer used no connection.
+ pub fn getConnId(self: *const Connection) !c_long {
+ var conn_id: c_long = undefined;
+ try libcurl.curl_easy_getinfo(self._easy, .conn_id, &conn_id);
+ return conn_id;
+ }
+
+ pub fn isConnReused(self: *const Connection) !bool {
+ var opened: c_long = undefined;
+ try libcurl.curl_easy_getinfo(self._easy, .num_connects, &opened);
+ return opened == 0;
+ }
+
// Total transfer time (name lookup to completion) in microseconds.
pub fn getTotalTimeMicros(self: *const Connection) !c_long {
var micros: c_long = undefined;
diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zig
index ae74f82e7..347fa6c27 100644
--- a/src/sys/libcurl.zig
+++ b/src/sys/libcurl.zig
@@ -246,6 +246,8 @@ pub const CurlInfo = enum(c.CURLINFO) {
response_code = c.CURLINFO_RESPONSE_CODE,
connect_code = c.CURLINFO_HTTP_CONNECTCODE,
total_time_t = c.CURLINFO_TOTAL_TIME_T,
+ num_connects = c.CURLINFO_NUM_CONNECTS,
+ conn_id = c.CURLINFO_CONN_ID,
};
pub const Error = error{
@@ -661,11 +663,14 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err
.response_code,
.connect_code,
.redirect_count,
+ .num_connects,
=> blk: {
const p: *c_long = out;
break :blk c.curl_easy_getinfo(easy, inf, p);
},
- .total_time_t => blk: {
+ .total_time_t,
+ .conn_id,
+ => blk: {
const p: *c.curl_off_t = out;
break :blk c.curl_easy_getinfo(easy, inf, p);
},