Skip to content

Lightpanda: the headless browser designed for AI and automation

webapi: Origin header + request header validation

Karl Seguin66af1dc58d14
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.html
index a644937e0..785a5ba61 100644
--- a/src/browser/tests/net/fetch.html
+++ b/src/browser/tests/net/fetch.html
@@ -459,10 +459,12 @@
headers.set('Accept-Language', 'fr');
headers.append('X-Multi', 'a');
headers.append('X-Multi', 'b');
+ // Cookie and anything sec-* are forbidden request headers: the Request
+ // guard drops them, so our own Sec-Ch-Ua survives.
headers.append('Cookie', 'a=1');
headers.append('Cookie', 'b=2');
- headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');
headers.set('Sec-Ch-Ua', '"Chromium";v="140"');
+ headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');
const response = await fetch('http://127.0.0.1:9582/echo_headers', { headers });
const text = await response.text();
state.resolve();
@@ -475,7 +477,7 @@
}
testing.expectEqual('fr', got['accept-language']);
testing.expectEqual('a, b', got['x-multi']);
- testing.expectEqual('a=1; b=2', got['cookie']);
+ testing.expectEqual(undefined, got['cookie']);
testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));
testing.expectEqual(false, got['user-agent'].includes('Mozilla'));
testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 2ca69c53e..a0db01e94 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -317,7 +317,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
const testing = @import("../../../testing.zig");
test "WebApi: fetch" {
- testing.expectLog(&.{ .http, .http });
+ testing.expectLog(&.{.http});
try testing.htmlRunner("net/fetch.html", .{});
try testing.htmlRunner("net/fetch_hash_route.html", .{});
}
diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zig
index 6d52e8904..e06c33517 100644
--- a/src/browser/webapi/net/Headers.zig
+++ b/src/browser/webapi/net/Headers.zig
@@ -27,6 +27,8 @@ _guard: Guard = .none,
// What mutation JS can make
pub const Guard = enum {
none, // don't block anything
+ request, // block forbidden request headers
+ request_no_cors, // block anything but the no-cors safelist
response, // block forbidden response headers
immutable, // block everythig
};
@@ -65,10 +67,25 @@ pub fn initGuarded(opts_: ?InitOpts, guard: Guard, exec: *const Execution) !*Hea
list.delete("set-cookie2", null);
}
- return exec._factory.create(Headers{
+ const self = try exec._factory.create(Headers{
._list = list,
._guard = guard,
});
+
+ if (guard == .request or guard == .request_no_cors) {
+ var i: usize = 0;
+ const list_entries = &self._list._entries;
+ while (i < list_entries.items.len) {
+ const entry = &list_entries.items[i];
+ if (try self.checkGuard(entry.name.str(), entry.value.str()) == .ignore) {
+ _ = list_entries.orderedRemove(i);
+ continue;
+ }
+ i += 1;
+ }
+ }
+
+ return self;
}
pub fn isForbiddenResponseHeaderName(name: []const u8) bool {
@@ -86,19 +103,138 @@ pub fn isForbiddenResponseHeaderName(name: []const u8) bool {
const Mutation = enum { proceed, ignore };
-fn checkGuard(self: *const Headers, name: []const u8) !Mutation {
- return switch (self._guard) {
- .none => .proceed,
- .immutable => error.TypeError,
- .response => if (isForbiddenResponseHeaderName(name)) .ignore else .proceed,
+fn checkGuard(self: *const Headers, name: []const u8, value: ?[]const u8) !Mutation {
+ const allowed = switch (self._guard) {
+ .none => true,
+ .immutable => return error.TypeError,
+ .request => isForbiddenRequestHeader(name, value orelse "") == false,
+ .request_no_cors => if (value) |v|
+ isNoCorsSafelistedRequestHeader(name, v)
+ else
+ isNoCorsSafelistedRequestHeaderName(name) or isPrivilegedNoCorsRequestHeaderName(name),
+ .response => isForbiddenResponseHeaderName(name) == false,
};
+ return if (allowed) .proceed else .ignore;
+}
+
+// https://fetch.spec.whatwg.org/#forbidden-request-header
+fn isForbiddenRequestHeader(name: []const u8, value: []const u8) bool {
+ const Set = std.StaticStringMapWithEql(void, std.static_string_map.eqlAsciiIgnoreCase);
+ const forbidden = Set.initComptime(.{
+ .{"accept-charset"}, .{"accept-encoding"},
+ .{"access-control-request-headers"}, .{"access-control-request-method"},
+ .{"connection"}, .{"content-length"},
+ .{"cookie"}, .{"cookie2"},
+ .{"date"}, .{"dnt"},
+ .{"expect"}, .{"host"},
+ .{"keep-alive"}, .{"origin"},
+ .{"referer"}, .{"set-cookie"},
+ .{"te"}, .{"trailer"},
+ .{"transfer-encoding"}, .{"upgrade"},
+ .{"via"},
+ });
+ if (forbidden.has(name)) {
+ return true;
+ }
+
+ if (std.ascii.startsWithIgnoreCase(name, "proxy-") or std.ascii.startsWithIgnoreCase(name, "sec-")) {
+ return true;
+ }
+
+ const Overrides = std.StaticStringMapWithEql(void, std.static_string_map.eqlAsciiIgnoreCase);
+ const overrides = Overrides.initComptime(.{
+ .{"x-http-method"}, .{"x-http-method-override"}, .{"x-method-override"},
+ });
+ if (overrides.has(name) == false) {
+ return false;
+ }
+
+ // value is a method override list.
+ var it = std.mem.splitScalar(u8, value, ',');
+ while (it.next()) |part| {
+ const method = std.mem.trim(u8, part, &Mime.HTTP_WHITESPACE);
+ if (std.ascii.eqlIgnoreCase(method, "connect")) {
+ return true;
+ }
+ if (std.ascii.eqlIgnoreCase(method, "trace")) {
+ return true;
+ }
+ if (std.ascii.eqlIgnoreCase(method, "track")) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// https://fetch.spec.whatwg.org/#no-cors-safelisted-request-header-name
+fn isNoCorsSafelistedRequestHeaderName(name: []const u8) bool {
+ return std.ascii.eqlIgnoreCase(name, "accept") or
+ std.ascii.eqlIgnoreCase(name, "accept-language") or
+ std.ascii.eqlIgnoreCase(name, "content-language") or
+ std.ascii.eqlIgnoreCase(name, "content-type");
+}
+
+// https://fetch.spec.whatwg.org/#privileged-no-cors-request-header-name
+fn isPrivilegedNoCorsRequestHeaderName(name: []const u8) bool {
+ return std.ascii.eqlIgnoreCase(name, "range") or std.ascii.eqlIgnoreCase(name, "authorization");
+}
+
+// https://fetch.spec.whatwg.org/#no-cors-safelisted-request-header
+fn isNoCorsSafelistedRequestHeader(name: []const u8, value: []const u8) bool {
+ if (value.len > 128) {
+ return false;
+ }
+
+ if (std.ascii.eqlIgnoreCase(name, "accept")) {
+ return hasCorsUnsafeByte(value) == false;
+ }
+
+ if (std.ascii.eqlIgnoreCase(name, "accept-language") or std.ascii.eqlIgnoreCase(name, "content-language")) {
+ for (value) |c| switch (c) {
+ '0'...'9', 'A'...'Z', 'a'...'z', ' ', '*', ',', '-', '.', ';', '=' => {},
+ else => return false,
+ };
+ return true;
+ }
+
+ if (std.ascii.eqlIgnoreCase(name, "content-type") == false) {
+ return false;
+ }
+
+ if (hasCorsUnsafeByte(value)) {
+ return false;
+ }
+
+ // Only the essence matters, and only these three are safelisted. Anything
+ // that doesn't parse as a mime type can't match one of them.
+ const essence = std.mem.trim(u8, std.mem.sliceTo(value, ';'), &Mime.HTTP_WHITESPACE);
+ return std.ascii.eqlIgnoreCase(essence, "application/x-www-form-urlencoded") or
+ std.ascii.eqlIgnoreCase(essence, "multipart/form-data") or
+ std.ascii.eqlIgnoreCase(essence, "text/plain");
+}
+
+// https://fetch.spec.whatwg.org/#cors-unsafe-request-header-byte
+fn hasCorsUnsafeByte(value: []const u8) bool {
+ for (value) |c| switch (c) {
+ 0x00...0x08, 0x0a...0x1f => return true,
+ '"', '(', ')', ':', '<', '>', '?', '@', '[', '\\', ']', '{', '}', 0x7f => return true,
+ else => {},
+ };
+ return false;
}
pub fn append(self: *Headers, name: []const u8, value: []const u8, exec: *const Execution) !void {
const normalized_name = try validateAndNormalizeName(name, exec);
const normalized_value = try normalizeValue(value, exec);
- const mutation = try self.checkGuard(normalized_name);
- if (mutation == .ignore) {
+
+ // The guard sees the combined value, since that's what a get would return
+ // once this append lands.
+ const combined = if (self._guard == .request_no_cors) blk: {
+ const existing = try self.get(normalized_name, exec) orelse break :blk normalized_value;
+ break :blk try std.mem.join(exec.local_arena, ", ", &.{ existing, normalized_value });
+ } else normalized_value;
+
+ if (try self.checkGuard(normalized_name, combined) == .ignore) {
return;
}
try self._list.append(exec.arena, normalized_name, normalized_value);
@@ -106,8 +242,7 @@ pub fn append(self: *Headers, name: []const u8, value: []const u8, exec: *const
pub fn delete(self: *Headers, name: []const u8, exec: *const Execution) !void {
const normalized_name = try validateAndNormalizeName(name, exec);
- const mutation = try self.checkGuard(normalized_name);
- if (mutation == .ignore) {
+ if (try self.checkGuard(normalized_name, null) == .ignore) {
return;
}
self._list.delete(normalized_name, null);
@@ -138,8 +273,7 @@ pub fn has(self: *const Headers, name: []const u8, exec: *const Execution) !bool
pub fn set(self: *Headers, name: []const u8, value_: []const u8, exec: *const Execution) !void {
const normalized_name = try validateAndNormalizeName(name, exec);
const value = try normalizeValue(value_, exec);
- const mutation = try self.checkGuard(normalized_name);
- if (mutation == .ignore) {
+ if (try self.checkGuard(normalized_name, value) == .ignore) {
return;
}
try self._list.set(exec.arena, normalized_name, value);
diff --git a/src/browser/webapi/net/Request.zig b/src/browser/webapi/net/Request.zig
index 6447add9e..b90ab38e2 100644
--- a/src/browser/webapi/net/Request.zig
+++ b/src/browser/webapi/net/Request.zig
@@ -123,12 +123,17 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {
.request => |r| r._method,
};
- var headers = if (opts.headers) |headers_init| switch (headers_init) {
- .obj => |h| h,
- else => try Headers.init(headers_init, exec),
- } else switch (input) {
+ const mode = switch (input) {
+ .url => opts.mode,
+ .request => |r| if (opts_ != null) opts.mode else r._mode,
+ };
+
+ const guard = headerGuard(mode);
+ var headers = if (opts.headers) |headers_init|
+ try Headers.initGuarded(headers_init, guard, exec)
+ else switch (input) {
.url => null,
- .request => |r| r._headers,
+ .request => |r| if (r._headers) |h| try Headers.initGuarded(.{ .obj = h }, guard, exec) else null,
};
const body = if (opts.body) |b| blk: {
@@ -136,7 +141,7 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {
// Per Fetch §6.5 step 11, the default Content-Type only applies if
// the user has not already set one via the headers init dict.
if (extracted.content_type) |ct| {
- const hs = headers orelse try Headers.init(null, exec);
+ const hs = headers orelse try Headers.initGuarded(null, guard, exec);
if (try hs.has("content-type", exec) == false) {
try hs.append("content-type", ct, exec);
}
@@ -157,11 +162,6 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {
.request => |r| r._signal,
};
- const mode = switch (input) {
- .url => opts.mode,
- .request => |r| if (opts_ != null) opts.mode else r._mode,
- };
-
const self = try arena.create(Request);
self.* = .{
._url = url,
@@ -244,11 +244,15 @@ pub fn getHeaders(self: *Request, exec: *const Execution) !*Headers {
return headers;
}
- const headers = try Headers.init(null, exec);
+ const headers = try Headers.initGuarded(null, headerGuard(self._mode), exec);
self._headers = headers;
return headers;
}
+fn headerGuard(mode: Mode) Headers.Guard {
+ return if (mode == .@"no-cors") .request_no_cors else .request;
+}
+
pub fn getBodyUsed(self: *const Request) bool {
if (self._body == null) {
return false;
diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zig
index b9a7b5d8f..d035c8a25 100644
--- a/src/network/CorsGate.zig
+++ b/src/network/CorsGate.zig
@@ -192,14 +192,6 @@ pub fn check(self: *CorsGate, transfer: *Transfer) !Result {
const origin = transfer.effectiveOrigin();
transfer._cors_cross_origin = true;
- // https://fetch.spec.whatwg.org/#append-a-request-origin-header
- //
- // If the request is no cors, we only add the origin if it is not HEAD or GET.
- // TODO: Should use referrer policy.
- if (req.request_mode != .no_cors or (req.method != .HEAD and req.method != .GET)) {
- try transfer.setHeader("Origin", origin, .{});
- }
-
if (req.request_mode == .no_cors) {
log.debug(.cors, "cross origin", .{
.url = req.url,
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 2ff673156..f60b60ff2 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1030,6 +1030,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
return transfer.failAsync(error.UrlBlocked);
}
+ if (transfer.req.internal == false) {
+ try setOriginHeader(transfer);
+ }
+
if (self.obey_cors and !transfer.req.internal) {
if (!isCrossOriginModeAllowed(transfer)) {
log.warn(.http, "blocked by mode", .{
@@ -1087,6 +1091,40 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
}
}
+// A cors request always carries an Origin. Everything else only carries one
+// for an unsafe method - same origin or not. Can't go in CorsGate since it can
+// be disabled.
+fn setOriginHeader(transfer: *Transfer) !void {
+ const req = &transfer.req;
+
+ const cross_origin = transfer._cors_origin_tainted or blk: {
+ const origin = req.origin orelse break :blk true;
+ break :blk URL.isSameOrigin(req.url, origin) == false;
+ };
+
+ const cors_tainted = cross_origin and switch (req.request_mode) {
+ .cors, .same_origin => true,
+ // A navigation is never cors-tainted. Chrome only sends an Origin on
+ // an unsafe one (a form POST), which the method check below covers.
+ .no_cors, .navigate => false,
+ };
+
+ const unsafe_method = req.method != .GET and req.method != .HEAD;
+ if (cors_tainted == false and unsafe_method == false) {
+ // A 301/302/303 rewrites the method to GET, leaving the previous hop's
+ // Origin behind. Only drop one we put there ourselves.
+ for (transfer.req_headers.items, 0..) |hdr, i| {
+ if (hdr.source == .user_agent and std.ascii.eqlIgnoreCase(hdr.name, "origin")) {
+ _ = transfer.req_headers.orderedRemove(i);
+ break;
+ }
+ }
+ return;
+ }
+
+ try transfer.setHeader("Origin", transfer.effectiveOrigin(), .{});
+}
+
// RobotsGate resumption.
pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void {
return self.pipeline(transfer, .throttle);
@@ -3145,6 +3183,16 @@ pub const Transfer = struct {
for (self.req_headers.items) |hdr| {
try conn.addHeader(arena, hdr.name, hdr.value);
}
+ if (req.body != null and self.findRequestHeader("content-type") == null) {
+ // Prevent libcurl from always setting application/x-www-form-urlencoded
+ try conn.addRawHeader("Content-Type:");
+ }
+ if (req.body == null and (req.method == .POST or req.method == .PUT) and
+ self.findRequestHeader("content-length") == null)
+ {
+ // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 10
+ try conn.addRawHeader("Content-Length: 0");
+ }
if (req.body != null) {
// Browsers never send Expect: 100-continue; libcurl generates it
// for HTTP/1.1 requests whose body exceeds 1MB