Lightpanda: the headless browser designed for AI and automation
webapi: include Sec-Fetch-Site and Sec-Fetch-Mode headers
Karl Seguinf70705b2ceb5
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 62696cb87..befb536a1 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -803,6 +803,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo.skip_cache = self.parent == null,.throttle = self.parent == null,.origin = self.origin,+ .initiator_origin = opts.initiator_origin,// Our own url is already the destination, so the owner's site for// cookies would say "same-site" for any top-level navigation..cookie_origin = opts.initiator_url,diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.htmlindex 785a5ba61..6d902ace5 100644--- a/src/browser/tests/net/fetch.html+++ b/src/browser/tests/net/fetch.html@@ -507,6 +507,35 @@}</script>+<script id=fetch_metadata_headers type=module>+ {+ // The test server is loopback, so potentially trustworthy.+ const state = await testing.async();+ const parse = (text) => {+ const got = {};+ for (const line of text.trim().split('\n')) {+ const sep = line.indexOf(': ');+ got[line.slice(0, sep).toLowerCase()] = line.slice(sep + 2);+ }+ return got;+ };+ const same = parse(await (await fetch('/echo_headers')).text());+ const same_origin_mode = parse(await (await fetch('/echo_headers', { mode: 'same-origin' })).text());+ // 127.0.0.1 -> localhost is cross-site, and the chain keeps it.+ const cross = parse(await (await fetch('/redirect_cross_echo_headers')).text());+ state.resolve();++ await state.done(() => {+ testing.expectEqual('same-origin', same['sec-fetch-site']);+ testing.expectEqual('cors', same['sec-fetch-mode']);+ testing.expectEqual('empty', same['sec-fetch-dest']);+ testing.expectEqual(undefined, same['sec-fetch-user']);+ testing.expectEqual('same-origin', same_origin_mode['sec-fetch-mode']);+ testing.expectEqual('cross-site', cross['sec-fetch-site']);+ });+ }+</script>+<script id=fetch_redirect_referer type=module>{const state = await testing.async();diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zigindex f30bffa50..7302ab3b9 100644--- a/src/network/CorsGate.zig+++ b/src/network/CorsGate.zig@@ -517,7 +517,7 @@ fn fetchThenResume(self: *CorsGate, transfer: *Transfer) !void {.notification = transfer.req.notification,.origin = transfer.req.origin,.credentials_mode = .omit,- .request_mode = .no_cors,+ .request_mode = .cors,.ctx = ctx,.header_callback = CorsPreflightContext.headerCallback,.done_callback = CorsPreflightContext.doneCallback,@@ -537,6 +537,10 @@ fn fetchThenResume(self: *CorsGate, transfer: *Transfer) !void {try fetch_transfer.setHeader("Referer", r, .{});}+ // internal requests skip this step, so we do it manually here+ fetch_transfer._fetch_site = transfer._fetch_site;+ try HttpClient.setFetchMetadataHeaders(fetch_transfer, transfer.destination());+// Access-Control-Allow-Methodstry fetch_transfer.setHeader(ACCESS_CONTROL_REQUEST_METHOD,diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex aa5f22b48..8365edc51 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -675,6 +675,9 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfeconst raw_origin: ?[]const u8 = req.origin orelse if (owner) |o| o.scope.origin() else null;owned.origin = if (raw_origin) |origin| try arena.dupe(u8, origin) else null;+ if (req.initiator_origin) |origin| {+ owned.initiator_origin = try arena.dupe(u8, origin);+ }// The body can be larger, so callers can signal, via the// `body_outlives_request` flag that they guarantee that the body@@ -1060,6 +1063,7 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {if (transfer.req.internal == false) {try setOriginHeader(transfer);+ try setFetchMetadataHeaders(transfer, transfer.destination());}if (self.obey_cors and !transfer.req.internal) {@@ -1153,6 +1157,45 @@ fn setOriginHeader(transfer: *Transfer) !void {try transfer.setHeader("Origin", transfer.effectiveOrigin(), .{});}+pub fn setFetchMetadataHeaders(transfer: *Transfer, dest: []const u8) !void {+ const req = &transfer.req;++ const site: Transfer.FetchSite = if (transfer._fetch_site) |previous|+ previous.next(req)+ else if (req.request_mode == .navigate and req.initiator_origin == null)+ .none+ else+ .forRequest(req);+ transfer._fetch_site = site;++ if (URL.isPotentiallyTrustworthy(req.url) == false) {+ var i: usize = 0;+ while (i < transfer.req_headers.items.len) {+ const hdr = transfer.req_headers.items[i];+ if (hdr.source == .user_agent and std.ascii.startsWithIgnoreCase(hdr.name, "sec-fetch-")) {+ _ = transfer.req_headers.orderedRemove(i);+ } else {+ i += 1;+ }+ }+ return;+ }++ try transfer.setHeader("Sec-Fetch-Site", @tagName(site), .{});+ try transfer.setHeader("Sec-Fetch-Mode", switch (req.request_mode) {+ .cors => "cors",+ .no_cors => "no-cors",+ .same_origin => "same-origin",+ .navigate => "navigate",+ }, .{});+ // Only a navigation the user started (address bar, CDP) has an activation+ // to report; we don't track the transient kind a click gives a script.+ if (req.request_mode == .navigate and req.initiator_origin == null) {+ try transfer.setHeader("Sec-Fetch-User", "?1", .{});+ }+ try transfer.setHeader("Sec-Fetch-Dest", dest, .{});+}+// RobotsGate resumption.pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void {return self.pipeline(transfer, .throttle);@@ -1982,6 +2025,10 @@ pub const Request = struct {// The Origin of the Request.origin: ?[]const u8,+ // The origin of the document that started a navigation. null for a+ // navigation the user started (address bar, CDP).+ initiator_origin: ?[]const u8 = null,+// Requests that are internal to the browser and skip various layers,// these do not need to be deferred and do not obey robots.txt.internal: bool = false,@@ -2018,6 +2065,13 @@ pub const Request = struct {// knowingly.shutdown_callback: ShutdownCallback,+ // What Sec-Fetch-Site is measured against: a navigation's initiator,+ // everything else's own origin.+ fn initiatorOrigin(req: *const Request) []const u8 {+ const origin = if (req.request_mode == .navigate) req.initiator_origin else req.origin;+ return origin orelse "null";+ }+pub fn credentialsAllowed(req: *const Request) bool {return switch (req.credentials_mode) {.omit => false,@@ -2359,6 +2413,50 @@ pub const Transfer = struct {// that redirected to it._cors_origin_tainted: bool = false,+ // Sec-Fetch-Site so far. null until the first hop is evaluated.+ _fetch_site: ?FetchSite = null,++ // Ordered matters, from closest to furthest+ const FetchSite = enum {+ @"same-origin",+ @"same-site",+ @"cross-site",+ none,++ fn forRequest(req: *const Request) FetchSite {+ const origin = req.initiatorOrigin();+ if (URL.isSameOrigin(req.url, origin)) {+ return .@"same-origin";+ }+ const same_scheme = std.mem.eql(u8, URL.getProtocol(req.url), URL.getProtocol(origin));+ if (same_scheme and Cookie.areHostsSameSite(URL.getHostname(req.url), URL.getHostname(origin))) {+ return .@"same-site";+ }+ return .@"cross-site";+ }++ fn next(self: FetchSite, req: *const Request) FetchSite {+ if (self == .none) {+ return .none;+ }+ // it can only get further, so redirect a -> b -> a doesn't appear as a -> a+ return @enumFromInt(@max(@intFromEnum(self), @intFromEnum(forRequest(req))));+ }+ };++ // Fetch's request destination, as Sec-Fetch-Dest spells it.+ pub fn destination(self: *const Transfer) []const u8 {+ return switch (self.req.resource_type) {+ // The owner of a document request is the frame being navigated.+ .document => if (self.owner != null and self.owner.?.parent != null) "iframe" else "document",+ .xhr, .fetch, .eventsource => "empty",+ .script => "script",+ .stylesheet => "style",+ .image => "image",+ .worker => "worker",+ };+ }+pub const State = union(enum) {// Pre-commit. Only valid inside the request flow (Client.request// or a re-entry like continueTransfer / unpark) before any commit@@ -5262,6 +5360,85 @@ test "HttpClient: cors redirect to a URL with credentials" {}}+test "HttpClient: fetch metadata headers" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();+ var client: Client = undefined;+ initTestClient(&client, &pool);++ {+ // A navigation the user started.+ const arena = try pool.acquire(.small, "fetch metadata test");+ defer arena.release();+ var transfer: Transfer = .{+ .arena = arena,+ .owner = null,+ .req = .{+ .method = .GET,+ .url = "https://a.test/",+ .origin = "https://a.test",+ .credentials_mode = .include,+ .request_mode = .navigate,+ .resource_type = .document,+ .shutdown_callback = noopShutdown,+ .ctx = undefined,+ },+ .client = &client,+ .id = 1,+ .start_time = 0,+ };+ try setFetchMetadataHeaders(&transfer, transfer.destination());+ try testing.expectEqual("none", transfer.findRequestHeader("sec-fetch-site").?);+ try testing.expectEqual("navigate", transfer.findRequestHeader("sec-fetch-mode").?);+ try testing.expectEqual("?1", transfer.findRequestHeader("sec-fetch-user").?);+ try testing.expectEqual("document", transfer.findRequestHeader("sec-fetch-dest").?);++ // none sticks across redirects+ transfer.req.url = "https://b.test/";+ try setFetchMetadataHeaders(&transfer, transfer.destination());+ try testing.expectEqual("none", transfer.findRequestHeader("sec-fetch-site").?);+ }++ {+ const arena = try pool.acquire(.small, "fetch metadata test");+ defer arena.release();+ var transfer: Transfer = .{+ .arena = arena,+ .owner = null,+ .req = .{+ .method = .GET,+ .url = "https://www.a.test/",+ .origin = "https://a.test",+ .initiator_origin = "https://b.test",+ .credentials_mode = .include,+ .request_mode = .no_cors,+ .resource_type = .image,+ .shutdown_callback = noopShutdown,+ .ctx = undefined,+ },+ .client = &client,+ .id = 1,+ .start_time = 0,+ };+ try setFetchMetadataHeaders(&transfer, transfer.destination());+ try testing.expectEqual("same-site", transfer.findRequestHeader("sec-fetch-site").?);+ try testing.expectEqual("no-cors", transfer.findRequestHeader("sec-fetch-mode").?);+ try testing.expectEqual(null, transfer.findRequestHeader("sec-fetch-user"));+ try testing.expectEqual("image", transfer.findRequestHeader("sec-fetch-dest").?);++ // an insecure hop drops them+ transfer.req.url = "http://a.test/";+ try setFetchMetadataHeaders(&transfer, transfer.destination());+ try testing.expectEqual(null, transfer.findRequestHeader("sec-fetch-site"));+ try testing.expectEqual(null, transfer.findRequestHeader("sec-fetch-dest"));++ // coming back to a same-origin URL keeps the furthest site so far+ transfer.req.url = "https://a.test/";+ try setFetchMetadataHeaders(&transfer, transfer.destination());+ try testing.expectEqual("cross-site", transfer.findRequestHeader("sec-fetch-site").?);+ }+}+test "HttpClient: fulfillIntercepted follows a 3xx redirect" {// Regression for #2828: a CDP Fetch.fulfillRequest with a 3xx status + a// Location header must be followed like a real network redirect (re-issued