Lightpanda: the headless browser designed for AI and automation
EventSource cannot be intercepted (header/data already processed)
Karl Seguinf2867d27da09
diff --git a/src/Notification.zig b/src/Notification.zigindex 26d48abf9..634fb2863 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -249,8 +249,7 @@ pub const RequestAuthRequired = struct {wait_for_interception: *bool,};-// A final response is complete (status, headers and the whole body are on the-// transfer) and has not been delivered to its consumer yet.+// A final response or a redirect which has not be delivered to its consumer yet.pub const ResponseIntercept = struct {transfer: *Transfer,wait_for_interception: *bool,diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 241621794..685345822 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1326,7 +1326,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {switch (cache_result) {.hit => |cached| {lp.metrics.http_cache.incr(.hit);- try transfer.bufferCached(cached, .local);+ try self.serveCached(transfer, cached, .local);return true;},.revalidate => |cached| {@@ -1383,10 +1383,18 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {};lp.metrics.http_cache.incr(.revalidated);- try transfer.bufferCached(stale, .validated);+ try self.serveCached(transfer, stale, .validated);return true;}+fn serveCached(self: *Client, transfer: *Transfer, cached: Cache.CachedResponse, cache_state: ResourceTiming.CacheState) !void {+ const body = transfer.setCachedResponse(cached, cache_state);+ if (self.interceptResponse(transfer, .{ .cached = body })) {+ return;+ }+ try transfer.bufferEvents(body);+}+// Everything the cache needs about this response. Names and values are// borrowed from the transfer arena, which outlives the put.fn cachePutRequest(transfer: *Transfer, rh: *http.ResponseHead) !?Cache.CachePutRequest {@@ -1957,9 +1965,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tself.removeConn(msg.conn);transfer._conn = null;- // A revalidation (304) is not offered: what it delivers is the cached- // body, which the response on the transfer does not carry.- if (transfer._cache_intent != .revalidate and self.interceptResponse(transfer, .final)) {+ // A revalidation (304) is not offered directly: what it delivers is the+ // cached body+ const not_modified = transfer._cache_intent == .revalidate and transfer.res.header.?.status == 304;+ if (not_modified == false and self.interceptResponse(transfer, .final)) {return false;}@@ -1971,6 +1980,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T// already released) to CDP. True when CDP took it: the transfer is parked// until continueInterceptedResponse, fulfillIntercepted or abortParked.fn interceptResponse(self: *Client, transfer: *Transfer, pause: Transfer.ResponsePause) bool {+ if (transfer.res.stream.started) {+ // A stream (EventSource) has already delivered its header and data.+ return false;+ }transfer._response_pause = pause;var wait_for_interception = false;transfer.notify(@@ -2262,6 +2275,10 @@ pub fn continueInterceptedResponse(self: *Client, transfer: *Transfer) !void {transfer.abortPipelineError(err);return err;},+ .cached => |body| transfer.bufferEvents(body) catch |err| {+ transfer.abortPipelineError(err);+ return err;+ },.redirect => |location| self.followRedirect(transfer, location) catch |err| {transfer.abortPipelineError(err);return err;@@ -2717,6 +2734,9 @@ pub const Transfer = struct {pub const ResponsePause = union(enum) {// A final response, delivered to the consumer on continue.final,+ // A response served from the HTTP cache (a hit, or the stored copy a+ // 304 renewed), with its arena-owned body; delivered on continue.+ cached: []const u8,// A redirect response; the arena-owned Location is followed on continue.redirect: []const u8,// A network failure; on continue it is reported exactly as it would@@ -3504,9 +3524,10 @@ pub const Transfer = struct {}}- // Serve a cache entry as this transfer's response. Takes ownership of- // `cached` (file-backed bodies are read into the arena and closed).- fn bufferCached(self: *Transfer, cached: Cache.CachedResponse, cache_state: ResourceTiming.CacheState) !void {+ // Make a cache entry this transfer's response, returning its body.+ // Takes ownership of `cached` (file-backed bodies are read into the+ // arena and closed).+ fn setCachedResponse(self: *Transfer, cached: Cache.CachedResponse, cache_state: ResourceTiming.CacheState) []const u8 {const body: []const u8 = switch (cached.data) {.buffer => |b| b,};@@ -3516,7 +3537,7 @@ pub const Transfer = struct {self._from_cache = true;self._timing.cache = cache_state;self._content_length = body.len;- try self.bufferEvents(body);+ return body;}// Materialize an interceptor-supplied response (CDP fulfillRequest).@@ -3543,6 +3564,8 @@ pub const Transfer = struct {self.setResponseHead(status, content_type);self.setResponseHeaders(headers);+ // It may replace a cached response paused at the response stage.+ self._from_cache = false;self._content_length = owned_body.len;try self.bufferEvents(owned_body);}diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zigindex 279aabd1e..b32ed5a55 100644--- a/src/server/cdp/domains/fetch.zig+++ b/src/server/cdp/domains/fetch.zig@@ -139,7 +139,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),.request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),- .resourceType = transfer.req.resource_type.string(),+ .resourceType = fetchResourceType(transfer.req.resource_type),.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID}, .{ .session_id = session_id });@@ -172,17 +172,17 @@ pub fn responseIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),.request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),- .resourceType = transfer.req.resource_type.string(),+ .resourceType = fetchResourceType(transfer.req.resource_type),.responseErrorReason = cdpErrorReason(failure.err),.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID}, .{ .session_id = session_id }),- .final, .redirect => {+ .final, .cached, .redirect => {const head = transfer.res.header orelse return error.MissingResponseHead;try bc.cdp.sendEvent("Fetch.requestPaused", .{.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),.request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),- .resourceType = transfer.req.resource_type.string(),+ .resourceType = fetchResourceType(transfer.req.resource_type),.responseStatusCode = head.status,.responseStatusText = transfer.statusText() orelse "",.responseHeaders = transfer.responseHeaders(),@@ -251,22 +251,16 @@ fn getResponseBody(cmd: *CDP.Command) !void {requestId: []const u8, // "INT-{d}"})) orelse return error.InvalidParams;+ const no_body = "Can only get response body on requests captured after headers received.";const transfer = (try pausedResponse(bc, params.requestId)) orelse {- return cmd.sendError(-32000, "Can only get response body on requests captured after headers received.", .{});+ return cmd.sendError(-32000, no_body, .{});};- switch (transfer._response_pause) {- .final => {},- .redirect => return cmd.sendError(-32000, "Response body is not available for redirects.", .{}),- .failed => return cmd.sendError(-32000, "Response body is not available for a failed request.", .{}),- }-- const body = transfer.res.buffer.items;- if (std.unicode.utf8ValidateSlice(body)) {- return cmd.sendResult(.{- .body = body,- .base64Encoded = false,- }, .{ .size_hint = body.len });- }+ const body = switch (transfer._response_pause) {+ .final => transfer.res.buffer.items,+ .cached => |cached| cached,+ .redirect, .failed => return cmd.sendError(-32000, no_body, .{}),+ };+ // Like Chrome, always base64, whatever the body is.return cmd.sendResult(.{.body = network.Base64Body{ .data = body },.base64Encoded = true,@@ -289,10 +283,12 @@ fn continueRequest(cmd: *CDP.Command) !void {}if (try pausedResponse(bc, params.requestId)) |transfer| {- // The request has already been sent; only an unchanged continue is meaningful.- if (params.url != null or params.method != null or params.postData != null or params.headers != null) {- return cmd.sendError(-32602, "Cannot override request at the response stage.", .{});- }+ // The request has already been sent: like Chrome, its overrides are+ // ignored, except that a url on a redirect replaces its Location.+ if (params.url) |url| switch (transfer._response_pause) {+ .redirect => transfer._response_pause = .{ .redirect = try transfer.arena.dupe(u8, url) },+ else => {},+ };_ = bc.intercept_state.remove(try idFromRequestId(params.requestId));try bc.cdp.browser.http_client.continueInterceptedResponse(transfer);return cmd.sendResult(null, .{});@@ -499,7 +495,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(request.frame_id),.request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),- .resourceType = request.resource_type.string(),+ .resourceType = fetchResourceType(request.resource_type),.authChallenge = .{.origin = "", // TODO get origin, could be the proxy address for example..source = if (challenge.source) |s| (if (s == .server) "Server" else "Proxy") else "",@@ -522,7 +518,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:// The CDP Network.ErrorReason for a network failure paused at the response// stage. Failed is CDP's own generic reason, for failures it has no narrower// value for (TLS, protocol and size errors among them).-fn cdpErrorReason(err: anyerror) ErrorReason {+fn cdpErrorReason(err: anyerror) network.ErrorReason {return switch (err) {error.OperationTimedout => .TimedOut,error.CouldntResolveHost, error.CouldntResolveProxy => .NameNotResolved,@@ -533,6 +529,10 @@ fn cdpErrorReason(err: anyerror) ErrorReason {};}+fn fetchResourceType(resource_type: HttpClient.Request.ResourceType) []const u8 {+ return @tagName(InterceptState.cdpResourceType(resource_type));+}+// Get u32 from requestId which is formatted as: "INT-{d}"fn idFromRequestId(request_id: []const u8) !u32 {if (!std.mem.startsWith(u8, request_id, "INT-")) {@@ -631,7 +631,11 @@ pub const InterceptState = struct {continue;}if (pattern.resource_type) |rt| {- if (rt != cdpResourceType(resource_type)) {+ const wanted: ResourceType = switch (rt) {+ .Fetch, .EventSource => .XHR,+ else => rt,+ };+ if (wanted != cdpResourceType(resource_type)) {continue;}}@@ -642,14 +646,15 @@ pub const InterceptState = struct {return false;}+ // Unlike the Network domain, Chrome's Fetch domain doesn't tell XHR,+ // fetch() and EventSource apart (they are all Blink's kXhr): they pause as+ // XHR, and a pattern for any one of the three matches all of them.fn cdpResourceType(resource_type: HttpClient.Request.ResourceType) ResourceType {return switch (resource_type) {.document => .Document,- .xhr => .XHR,+ .xhr, .fetch, .eventsource => .XHR,.script, .worker => .Script,- .fetch => .Fetch,.stylesheet => .Stylesheet,- .eventsource => .EventSource,.image => .Image,};}@@ -819,7 +824,10 @@ test "cdp.Fetch: InterceptState patterns" {});try testing.expectEqual(2, state.patterns.len);try testing.expect(state.matches(.Request, "http://h/api/x", .xhr));- try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .fetch));+ // Like Chrome, XHR also covers fetch() and EventSource+ try testing.expect(state.matches(.Request, "http://h/api/x", .fetch));+ try testing.expect(state.matches(.Request, "http://h/api/x", .eventsource));+ try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .script));try testing.expectEqual(false, state.matches(.Request, "http://h/other", .xhr));// A pattern pauses only at its own stagetry testing.expect(state.matches(.Response, "http://h/a.css", .stylesheet));@@ -850,7 +858,8 @@ test "cdp.Fetch: enable compiles patterns, disable clears them" {try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-P" });try testing.expectEqual(2, bc.intercept_state.patterns.len);try testing.expect(bc.intercept_state.matches(.Request, "http://h/json/a", .fetch));- try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/json/a", .xhr));+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/json/a", .xhr));+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/json/a", .script));try testing.expect(bc.intercept_state.matches(.Request, "http://h/exact", .xhr));try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));@@ -874,6 +883,7 @@ test "cdp.Fetch: enable compiles patterns, disable clears them" {// Response-stage tests drive real requests against the test server and read// the outcome through the request's own callbacks.const ResponseStageProbe = struct {+ streaming: bool = false,body: std.ArrayList(u8) = .empty,done: bool = false,err: ?anyerror = null,@@ -908,6 +918,7 @@ const ResponseStageProbe = struct {.credentials_mode = .omit,.request_mode = .no_cors,.resource_type = .script,+ .streaming = self.streaming,.notification = bc.session.notification,.ctx = self,.data_callback = dataCallback,@@ -990,7 +1001,8 @@ test "cdp.Fetch: response stage pauses a final response and continueResponse deltry testing.expectEqual(0, probe.body.items.len);try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });- try ctx.expectSentResult(.{ .body = repeat("1234567890", 10), .base64Encoded = false }, .{ .id = 2 });+ // repeat("1234567890", 10), base64 like Chrome even for text+ try ctx.expectSentResult(.{ .body = "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MA==", .base64Encoded = true }, .{ .id = 2 });try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });try ctx.expectSentResult(null, .{ .id = 3 });@@ -1052,7 +1064,7 @@ test "cdp.Fetch: response stage fulfillRequest replaces the response" {try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);}-test "cdp.Fetch: response stage getResponseBody base64-encodes a non-UTF-8 body" {+test "cdp.Fetch: response stage getResponseBody returns a binary body byte-exact" {var ctx = try testing.context();defer ctx.deinit();const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });@@ -1104,7 +1116,7 @@ test "cdp.Fetch: response stage pauses a redirect response, then the response ittry testing.expectEqualSlices(u8, "http://127.0.0.1:9582/xhr", location.?);try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });- try ctx.expectSentError(-32000, "Response body is not available for redirects.", .{ .id = 2 });+ try ctx.expectSentError(-32000, "Can only get response body on requests captured after headers received.", .{ .id = 2 });// Releasing the redirect follows it; the redirected request's response pauses in turn.try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });@@ -1124,6 +1136,41 @@ test "cdp.Fetch: response stage pauses a redirect response, then the response ittry testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);}+test "cdp.Fetch: response stage continueRequest ignores request overrides, but a url replaces a redirect's Location" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr/redirect");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.continueRequest", .params = .{+ .requestId = "INT-0000000001",+ .url = "http://127.0.0.1:9582/images/small.png",+ .method = "POST",+ } });+ try ctx.expectSentResult(null, .{ .id = 2 });+ try ResponseStageProbe.tickUntilPaused(bc, 1);+ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000002",+ .request = .{ .url = "http://127.0.0.1:9582/images/small.png", .method = "GET" },+ }, .{ .session_id = "SID-RS" });++ // Not a redirect: the url is ignored, like the rest.+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueRequest", .params = .{+ .requestId = "INT-0000000002",+ .url = "http://127.0.0.1:9582/xhr",+ } });+ try ctx.expectSentResult(null, .{ .id = 3 });+ try probe.tickUntilSettled(bc);+ try testing.expectEqual(null, probe.err);+ try testing.expectEqual(1024, probe.body.items.len);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}+test "cdp.Fetch: response stage pauses a network failure with responseErrorReason" {var ctx = try testing.context();defer ctx.deinit();@@ -1146,7 +1193,7 @@ test "cdp.Fetch: response stage pauses a network failure with responseErrorReasotry testing.expectEqual(null, paused.get("responseHeaders"));try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });- try ctx.expectSentError(-32000, "Response body is not available for a failed request.", .{ .id = 2 });+ try ctx.expectSentError(-32000, "Can only get response body on requests captured after headers received.", .{ .id = 2 });// Releasing it unchanged reports the original failure.try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });@@ -1156,6 +1203,23 @@ test "cdp.Fetch: response stage pauses a network failure with responseErrorReasotry testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);}+test "cdp.Fetch: response stage does not pause a stream it already delivered" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{ .streaming = true };+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/sse/simple");+ try probe.tickUntilSettled(bc);++ try testing.expectEqual(null, probe.err);+ try testing.expectEqual(73, probe.body.items.len);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+ try testing.expectEqual(error.PausedEventNotSent, sentPausedParams(&ctx, "INT-0000000001"));+}+test "cdp.Fetch: closing the browser context releases a response-stage pause" {var ctx = try testing.context();defer ctx.deinit();@@ -1174,12 +1238,12 @@ test "cdp.Fetch: closing the browser context releases a response-stage pause" {}test "cdp.Fetch: cdpErrorReason" {- try testing.expectEqual(ErrorReason.TimedOut, cdpErrorReason(error.OperationTimedout));- try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveHost));- try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveProxy));- try testing.expectEqual(ErrorReason.ConnectionFailed, cdpErrorReason(error.CouldntConnect));- try testing.expectEqual(ErrorReason.ConnectionClosed, cdpErrorReason(error.GotNothing));- try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.RecvError));- try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.SendError));- try testing.expectEqual(ErrorReason.Failed, cdpErrorReason(error.SslConnectError));+ try testing.expectEqual(network.ErrorReason.TimedOut, cdpErrorReason(error.OperationTimedout));+ try testing.expectEqual(network.ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveHost));+ try testing.expectEqual(network.ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveProxy));+ try testing.expectEqual(network.ErrorReason.ConnectionFailed, cdpErrorReason(error.CouldntConnect));+ try testing.expectEqual(network.ErrorReason.ConnectionClosed, cdpErrorReason(error.GotNothing));+ try testing.expectEqual(network.ErrorReason.ConnectionReset, cdpErrorReason(error.RecvError));+ try testing.expectEqual(network.ErrorReason.ConnectionReset, cdpErrorReason(error.SendError));+ try testing.expectEqual(network.ErrorReason.Failed, cdpErrorReason(error.SslConnectError));}