Lightpanda: the headless browser designed for AI and automation
webapi: Origin header + request header validation
Karl Seguin66af1dc58d14
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.htmlindex a644937e0..785a5ba61 100644--- a/src/browser/tests/net/fetch.html+++ b/src/browser/tests/net/fetch.html@@ -459,10 +459,12 @@headers.set('Accept-Language', 'fr');headers.append('X-Multi', 'a');headers.append('X-Multi', 'b');+ // Cookie and anything sec-* are forbidden request headers: the Request+ // guard drops them, so our own Sec-Ch-Ua survives.headers.append('Cookie', 'a=1');headers.append('Cookie', 'b=2');- headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');headers.set('Sec-Ch-Ua', '"Chromium";v="140"');+ headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');const response = await fetch('http://127.0.0.1:9582/echo_headers', { headers });const text = await response.text();state.resolve();@@ -475,7 +477,7 @@}testing.expectEqual('fr', got['accept-language']);testing.expectEqual('a, b', got['x-multi']);- testing.expectEqual('a=1; b=2', got['cookie']);+ testing.expectEqual(undefined, got['cookie']);testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));testing.expectEqual(false, got['user-agent'].includes('Mozilla'));testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex 2ca69c53e..a0db01e94 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -317,7 +317,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {const testing = @import("../../../testing.zig");test "WebApi: fetch" {- testing.expectLog(&.{ .http, .http });+ testing.expectLog(&.{.http});try testing.htmlRunner("net/fetch.html", .{});try testing.htmlRunner("net/fetch_hash_route.html", .{});}diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zigindex 6d52e8904..e06c33517 100644--- a/src/browser/webapi/net/Headers.zig+++ b/src/browser/webapi/net/Headers.zig@@ -27,6 +27,8 @@ _guard: Guard = .none,// What mutation JS can makepub const Guard = enum {none, // don't block anything+ request, // block forbidden request headers+ request_no_cors, // block anything but the no-cors safelistresponse, // block forbidden response headersimmutable, // block everythig};@@ -65,10 +67,25 @@ pub fn initGuarded(opts_: ?InitOpts, guard: Guard, exec: *const Execution) !*Healist.delete("set-cookie2", null);}- return exec._factory.create(Headers{+ const self = try exec._factory.create(Headers{._list = list,._guard = guard,});++ if (guard == .request or guard == .request_no_cors) {+ var i: usize = 0;+ const list_entries = &self._list._entries;+ while (i < list_entries.items.len) {+ const entry = &list_entries.items[i];+ if (try self.checkGuard(entry.name.str(), entry.value.str()) == .ignore) {+ _ = list_entries.orderedRemove(i);+ continue;+ }+ i += 1;+ }+ }++ return self;}pub fn isForbiddenResponseHeaderName(name: []const u8) bool {@@ -86,19 +103,138 @@ pub fn isForbiddenResponseHeaderName(name: []const u8) bool {const Mutation = enum { proceed, ignore };-fn checkGuard(self: *const Headers, name: []const u8) !Mutation {- return switch (self._guard) {- .none => .proceed,- .immutable => error.TypeError,- .response => if (isForbiddenResponseHeaderName(name)) .ignore else .proceed,+fn checkGuard(self: *const Headers, name: []const u8, value: ?[]const u8) !Mutation {+ const allowed = switch (self._guard) {+ .none => true,+ .immutable => return error.TypeError,+ .request => isForbiddenRequestHeader(name, value orelse "") == false,+ .request_no_cors => if (value) |v|+ isNoCorsSafelistedRequestHeader(name, v)+ else+ isNoCorsSafelistedRequestHeaderName(name) or isPrivilegedNoCorsRequestHeaderName(name),+ .response => isForbiddenResponseHeaderName(name) == false,};+ return if (allowed) .proceed else .ignore;+}++// https://fetch.spec.whatwg.org/#forbidden-request-header+fn isForbiddenRequestHeader(name: []const u8, value: []const u8) bool {+ const Set = std.StaticStringMapWithEql(void, std.static_string_map.eqlAsciiIgnoreCase);+ const forbidden = Set.initComptime(.{+ .{"accept-charset"}, .{"accept-encoding"},+ .{"access-control-request-headers"}, .{"access-control-request-method"},+ .{"connection"}, .{"content-length"},+ .{"cookie"}, .{"cookie2"},+ .{"date"}, .{"dnt"},+ .{"expect"}, .{"host"},+ .{"keep-alive"}, .{"origin"},+ .{"referer"}, .{"set-cookie"},+ .{"te"}, .{"trailer"},+ .{"transfer-encoding"}, .{"upgrade"},+ .{"via"},+ });+ if (forbidden.has(name)) {+ return true;+ }++ if (std.ascii.startsWithIgnoreCase(name, "proxy-") or std.ascii.startsWithIgnoreCase(name, "sec-")) {+ return true;+ }++ const Overrides = std.StaticStringMapWithEql(void, std.static_string_map.eqlAsciiIgnoreCase);+ const overrides = Overrides.initComptime(.{+ .{"x-http-method"}, .{"x-http-method-override"}, .{"x-method-override"},+ });+ if (overrides.has(name) == false) {+ return false;+ }++ // value is a method override list.+ var it = std.mem.splitScalar(u8, value, ',');+ while (it.next()) |part| {+ const method = std.mem.trim(u8, part, &Mime.HTTP_WHITESPACE);+ if (std.ascii.eqlIgnoreCase(method, "connect")) {+ return true;+ }+ if (std.ascii.eqlIgnoreCase(method, "trace")) {+ return true;+ }+ if (std.ascii.eqlIgnoreCase(method, "track")) {+ return true;+ }+ }+ return false;+}++// https://fetch.spec.whatwg.org/#no-cors-safelisted-request-header-name+fn isNoCorsSafelistedRequestHeaderName(name: []const u8) bool {+ return std.ascii.eqlIgnoreCase(name, "accept") or+ std.ascii.eqlIgnoreCase(name, "accept-language") or+ std.ascii.eqlIgnoreCase(name, "content-language") or+ std.ascii.eqlIgnoreCase(name, "content-type");+}++// https://fetch.spec.whatwg.org/#privileged-no-cors-request-header-name+fn isPrivilegedNoCorsRequestHeaderName(name: []const u8) bool {+ return std.ascii.eqlIgnoreCase(name, "range") or std.ascii.eqlIgnoreCase(name, "authorization");+}++// https://fetch.spec.whatwg.org/#no-cors-safelisted-request-header+fn isNoCorsSafelistedRequestHeader(name: []const u8, value: []const u8) bool {+ if (value.len > 128) {+ return false;+ }++ if (std.ascii.eqlIgnoreCase(name, "accept")) {+ return hasCorsUnsafeByte(value) == false;+ }++ if (std.ascii.eqlIgnoreCase(name, "accept-language") or std.ascii.eqlIgnoreCase(name, "content-language")) {+ for (value) |c| switch (c) {+ '0'...'9', 'A'...'Z', 'a'...'z', ' ', '*', ',', '-', '.', ';', '=' => {},+ else => return false,+ };+ return true;+ }++ if (std.ascii.eqlIgnoreCase(name, "content-type") == false) {+ return false;+ }++ if (hasCorsUnsafeByte(value)) {+ return false;+ }++ // Only the essence matters, and only these three are safelisted. Anything+ // that doesn't parse as a mime type can't match one of them.+ const essence = std.mem.trim(u8, std.mem.sliceTo(value, ';'), &Mime.HTTP_WHITESPACE);+ return std.ascii.eqlIgnoreCase(essence, "application/x-www-form-urlencoded") or+ std.ascii.eqlIgnoreCase(essence, "multipart/form-data") or+ std.ascii.eqlIgnoreCase(essence, "text/plain");+}++// https://fetch.spec.whatwg.org/#cors-unsafe-request-header-byte+fn hasCorsUnsafeByte(value: []const u8) bool {+ for (value) |c| switch (c) {+ 0x00...0x08, 0x0a...0x1f => return true,+ '"', '(', ')', ':', '<', '>', '?', '@', '[', '\\', ']', '{', '}', 0x7f => return true,+ else => {},+ };+ return false;}pub fn append(self: *Headers, name: []const u8, value: []const u8, exec: *const Execution) !void {const normalized_name = try validateAndNormalizeName(name, exec);const normalized_value = try normalizeValue(value, exec);- const mutation = try self.checkGuard(normalized_name);- if (mutation == .ignore) {++ // The guard sees the combined value, since that's what a get would return+ // once this append lands.+ const combined = if (self._guard == .request_no_cors) blk: {+ const existing = try self.get(normalized_name, exec) orelse break :blk normalized_value;+ break :blk try std.mem.join(exec.local_arena, ", ", &.{ existing, normalized_value });+ } else normalized_value;++ if (try self.checkGuard(normalized_name, combined) == .ignore) {return;}try self._list.append(exec.arena, normalized_name, normalized_value);@@ -106,8 +242,7 @@ pub fn append(self: *Headers, name: []const u8, value: []const u8, exec: *constpub fn delete(self: *Headers, name: []const u8, exec: *const Execution) !void {const normalized_name = try validateAndNormalizeName(name, exec);- const mutation = try self.checkGuard(normalized_name);- if (mutation == .ignore) {+ if (try self.checkGuard(normalized_name, null) == .ignore) {return;}self._list.delete(normalized_name, null);@@ -138,8 +273,7 @@ pub fn has(self: *const Headers, name: []const u8, exec: *const Execution) !boolpub fn set(self: *Headers, name: []const u8, value_: []const u8, exec: *const Execution) !void {const normalized_name = try validateAndNormalizeName(name, exec);const value = try normalizeValue(value_, exec);- const mutation = try self.checkGuard(normalized_name);- if (mutation == .ignore) {+ if (try self.checkGuard(normalized_name, value) == .ignore) {return;}try self._list.set(exec.arena, normalized_name, value);diff --git a/src/browser/webapi/net/Request.zig b/src/browser/webapi/net/Request.zigindex 6447add9e..b90ab38e2 100644--- a/src/browser/webapi/net/Request.zig+++ b/src/browser/webapi/net/Request.zig@@ -123,12 +123,17 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {.request => |r| r._method,};- var headers = if (opts.headers) |headers_init| switch (headers_init) {- .obj => |h| h,- else => try Headers.init(headers_init, exec),- } else switch (input) {+ const mode = switch (input) {+ .url => opts.mode,+ .request => |r| if (opts_ != null) opts.mode else r._mode,+ };++ const guard = headerGuard(mode);+ var headers = if (opts.headers) |headers_init|+ try Headers.initGuarded(headers_init, guard, exec)+ else switch (input) {.url => null,- .request => |r| r._headers,+ .request => |r| if (r._headers) |h| try Headers.initGuarded(.{ .obj = h }, guard, exec) else null,};const body = if (opts.body) |b| blk: {@@ -136,7 +141,7 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {// Per Fetch §6.5 step 11, the default Content-Type only applies if// the user has not already set one via the headers init dict.if (extracted.content_type) |ct| {- const hs = headers orelse try Headers.init(null, exec);+ const hs = headers orelse try Headers.initGuarded(null, guard, exec);if (try hs.has("content-type", exec) == false) {try hs.append("content-type", ct, exec);}@@ -157,11 +162,6 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {.request => |r| r._signal,};- const mode = switch (input) {- .url => opts.mode,- .request => |r| if (opts_ != null) opts.mode else r._mode,- };-const self = try arena.create(Request);self.* = .{._url = url,@@ -244,11 +244,15 @@ pub fn getHeaders(self: *Request, exec: *const Execution) !*Headers {return headers;}- const headers = try Headers.init(null, exec);+ const headers = try Headers.initGuarded(null, headerGuard(self._mode), exec);self._headers = headers;return headers;}+fn headerGuard(mode: Mode) Headers.Guard {+ return if (mode == .@"no-cors") .request_no_cors else .request;+}+pub fn getBodyUsed(self: *const Request) bool {if (self._body == null) {return false;diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zigindex b9a7b5d8f..d035c8a25 100644--- a/src/network/CorsGate.zig+++ b/src/network/CorsGate.zig@@ -192,14 +192,6 @@ pub fn check(self: *CorsGate, transfer: *Transfer) !Result {const origin = transfer.effectiveOrigin();transfer._cors_cross_origin = true;- // https://fetch.spec.whatwg.org/#append-a-request-origin-header- //- // If the request is no cors, we only add the origin if it is not HEAD or GET.- // TODO: Should use referrer policy.- if (req.request_mode != .no_cors or (req.method != .HEAD and req.method != .GET)) {- try transfer.setHeader("Origin", origin, .{});- }-if (req.request_mode == .no_cors) {log.debug(.cors, "cross origin", .{.url = req.url,diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 2ff673156..f60b60ff2 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1030,6 +1030,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {return transfer.failAsync(error.UrlBlocked);}+ if (transfer.req.internal == false) {+ try setOriginHeader(transfer);+ }+if (self.obey_cors and !transfer.req.internal) {if (!isCrossOriginModeAllowed(transfer)) {log.warn(.http, "blocked by mode", .{@@ -1087,6 +1091,40 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {}}+// A cors request always carries an Origin. Everything else only carries one+// for an unsafe method - same origin or not. Can't go in CorsGate since it can+// be disabled.+fn setOriginHeader(transfer: *Transfer) !void {+ const req = &transfer.req;++ const cross_origin = transfer._cors_origin_tainted or blk: {+ const origin = req.origin orelse break :blk true;+ break :blk URL.isSameOrigin(req.url, origin) == false;+ };++ const cors_tainted = cross_origin and switch (req.request_mode) {+ .cors, .same_origin => true,+ // A navigation is never cors-tainted. Chrome only sends an Origin on+ // an unsafe one (a form POST), which the method check below covers.+ .no_cors, .navigate => false,+ };++ const unsafe_method = req.method != .GET and req.method != .HEAD;+ if (cors_tainted == false and unsafe_method == false) {+ // A 301/302/303 rewrites the method to GET, leaving the previous hop's+ // Origin behind. Only drop one we put there ourselves.+ for (transfer.req_headers.items, 0..) |hdr, i| {+ if (hdr.source == .user_agent and std.ascii.eqlIgnoreCase(hdr.name, "origin")) {+ _ = transfer.req_headers.orderedRemove(i);+ break;+ }+ }+ return;+ }++ try transfer.setHeader("Origin", transfer.effectiveOrigin(), .{});+}+// RobotsGate resumption.pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void {return self.pipeline(transfer, .throttle);@@ -3145,6 +3183,16 @@ pub const Transfer = struct {for (self.req_headers.items) |hdr| {try conn.addHeader(arena, hdr.name, hdr.value);}+ if (req.body != null and self.findRequestHeader("content-type") == null) {+ // Prevent libcurl from always setting application/x-www-form-urlencoded+ try conn.addRawHeader("Content-Type:");+ }+ if (req.body == null and (req.method == .POST or req.method == .PUT) and+ self.findRequestHeader("content-length") == null)+ {+ // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 10+ try conn.addRawHeader("Content-Length: 0");+ }if (req.body != null) {// Browsers never send Expect: 100-continue; libcurl generates it// for HTTP/1.1 requests whose body exceeds 1MB