Lightpanda: the headless browser designed for AI and automation
perf: Cookie Jar stores cookies per site
Karl Seguin6a0ff3fd56cc
diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 4dc133fd1..d935f7726 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -2626,8 +2626,8 @@ fn execGetCookies(arena: std.mem.Allocator, session: *lp.Session, arguments: ?stconst Params = struct { url: ?[]const u8 = null, all: bool = false };const args = try parseArgsOrDefault(Params, arena, arguments);- const cookies = session.cookie_jar.cookies.items;- if (cookies.len == 0) return "No cookies.";+ const jar = &session.cookie_jar;+ if (jar.len == 0) return "No cookies.";const filter_url: ?[:0]const u8 = blk: {if (args.all) break :blk null;@@ -2640,7 +2640,8 @@ fn execGetCookies(arena: std.mem.Allocator, session: *lp.Session, arguments: ?stvar aw: std.Io.Writer.Allocating = .init(arena);const writer = &aw.writer;var count: usize = 0;- for (cookies) |*cookie| {+ var it = jar.iterator();+ while (it.next()) |cookie| {if (host) |h| if (!cookie.matchesHost(h)) continue;writer.print("{s}={s}", .{ cookie.name, cookie.value }) catch return ToolError.InternalError;writer.print("; domain={s}; path={s}", .{ cookie.domain, cookie.path }) catch return ToolError.InternalError;diff --git a/src/browser/webapi/Document.zig b/src/browser/webapi/Document.zigindex fe74ff528..581d94051 100644--- a/src/browser/webapi/Document.zig+++ b/src/browser/webapi/Document.zig@@ -1932,11 +1932,12 @@ test "Document: a non-secure document can't set or overwrite a Secure cookie" {frame.url = "http://victim.example/";_ = try doc.setCookie("s=1; Secure", frame);- try testing.expectEqual(0, jar.cookies.items.len);+ try testing.expectEqual(0, jar.len);try jar.populateFromResponse("https://victim.example/", "s=1; Secure");_ = try doc.setCookie("s=2", frame);- try testing.expectEqual("1", jar.cookies.items[0].value);+ var it = jar.iterator();+ try testing.expectEqual("1", it.next().?.value);}test "Document: isRelaxableTo" {diff --git a/src/browser/webapi/WebDriver.zig b/src/browser/webapi/WebDriver.zigindex 6bbebdbbb..a40253bf1 100644--- a/src/browser/webapi/WebDriver.zig+++ b/src/browser/webapi/WebDriver.zig@@ -82,7 +82,7 @@ fn getNamedCookie(_: *const WebDriver, name: []const u8, frame: *Frame) ?WebDriv}jar.removeExpired(null);- for (jar.cookies.items) |*cookie| {+ for (jar.siteCookies(target.host)) |*cookie| {if (cookie.appliesTo(&target, .{ .same_site = true, .is_http = true, .kind = .navigation }) == false) {continue;}diff --git a/src/browser/webapi/storage/Cookie.zig b/src/browser/webapi/storage/Cookie.zigindex 3f57456e6..423cc9b91 100644--- a/src/browser/webapi/storage/Cookie.zig+++ b/src/browser/webapi/storage/Cookie.zig@@ -639,7 +639,10 @@ pub const SiteForCookies = union(enum) {pub const Jar = struct {allocator: Allocator,- cookies: std.ArrayList(Cookie),+ // site => cookie list+ sites: std.StringArrayHashMapUnmanaged(std.ArrayList(Cookie)),+ // Cookies across all sites.+ len: usize,// Optional notifier for cookie change events. When set, `add` dispatches// `cookie_changed` after a real mutation. Session populates this; the// standalone test construction leaves it null.@@ -647,24 +650,81 @@ pub const Jar = struct {pub fn init(allocator: Allocator, notification: ?*Notification) Jar {return .{- .cookies = .empty,+ .len = 0,+ .sites = .empty,.allocator = allocator,.notification = notification,};}pub fn deinit(self: *Jar) void {- for (self.cookies.items) |c| {- c.deinit();- }- self.cookies.deinit(self.allocator);+ self.freeSites();+ self.sites.deinit(self.allocator);}pub fn clearRetainingCapacity(self: *Jar) void {- for (self.cookies.items) |c| {- c.deinit();+ self.freeSites();+ self.sites.clearRetainingCapacity();+ self.len = 0;+ }++ fn freeSites(self: *Jar) void {+ var it = self.sites.iterator();+ while (it.next()) |entry| {+ for (entry.value_ptr.items) |c| {+ c.deinit();+ }+ entry.value_ptr.deinit(self.allocator);+ self.allocator.free(entry.key_ptr.*);+ }+ }++ /// The only cookies that can apply to `host` (lower case, as in+ /// PreparedUri): those of its site, in send order.+ pub fn siteCookies(self: *const Jar, host: []const u8) []Cookie {+ const group = self.sites.getPtr(siteKey(host)) orelse return &.{};+ return group.items;+ }++ /// Every cookie, site by site. The jar must not change while iterating.+ pub fn iterator(self: *const Jar) Iterator {+ return .{ .groups = self.sites.values() };+ }++ pub const Iterator = struct {+ group: usize = 0,+ index: usize = 0,+ groups: []std.ArrayList(Cookie),++ pub fn next(self: *Iterator) ?*Cookie {+ while (self.group < self.groups.len) {+ const items = self.groups[self.group].items;+ if (self.index < items.len) {+ defer self.index += 1;+ return &items[self.index];+ }+ self.group += 1;+ self.index = 0;+ }+ return null;+ }+ };++ /// Removes the cookies `matches` accepts, `context` being its first argument.+ pub fn removeWhere(self: *Jar, context: anytype, comptime matches: fn (@TypeOf(context), *const Cookie) bool) void {+ var g = self.sites.count();+ while (g > 0) {+ g -= 1;+ // Going backwards, a group is only dropped once its last cookie,+ // index 0, is removed, and only groups already visited shift.+ var i = self.sites.values()[g].items.len;+ while (i > 0) {+ i -= 1;+ if (matches(context, &self.sites.values()[g].items[i])) {+ self.remove(g, i);+ }+ }}- self.cookies.clearRetainingCapacity();}pub fn add(@@ -687,51 +747,73 @@ pub const Jar = struct {}const is_expired = isCookieExpired(&cookie, request_time);+ const key = siteKey(cookie.domain);+ const site = self.sites.getIndex(key);++ if (site) |g| {+ const group = &self.sites.values()[g];+ for (group.items, 0..) |*c, i| {+ // We're only looking for the equal one.+ if (areCookiesEqual(&cookie, c) == false) {+ continue;+ }- for (self.cookies.items, 0..) |*c, i| {- // We're only looking for the equal one.- if (areCookiesEqual(&cookie, c) == false) {- continue;- }+ // RFC 6265bis 5.7.2: a non-HTTP API (e.g. document.cookie) must+ // not replace an HttpOnly cookie.+ if (c.http_only and is_http == false) {+ return;+ }- // RFC 6265bis 5.7.2: a non-HTTP API (e.g. document.cookie) must- // not replace an HttpOnly cookie.- if (c.http_only and is_http == false) {+ if (is_expired) {+ self.remove(g, i);+ } else {+ // RFC 6265bis 5.7 step 23: the replacement keeps the+ // creation time of the cookie it replaces.+ cookie.creation_time = c.creation_time;+ // Free the old cookie's arena before overwriting the slot;+ // after the assignment, c points at the new cookie.+ c.deinit();+ group.items[i] = cookie;+ stored = true;+ self.dispatchChange(.changed, g, i);+ }return;}-- if (is_expired) {- self.remove(i);- } else {- // RFC 6265bis 5.7 step 23: the replacement keeps the- // creation time of the cookie it replaces.- cookie.creation_time = c.creation_time;- // Free the old cookie's arena before overwriting the slot;- // after the assignment, c points at the new cookie.- c.deinit();- self.cookies.items[i] = cookie;- stored = true;- self.dispatchChange(.changed, &self.cookies.items[i]);- }- return;}if (!is_expired) {// Chrome's CookieMonster::GarbageCollect, before the insert so the// new cookie is never evicted: expired cookies go first, then the// least recently accessed (RFC 6265bis 5.7).- self.removeExpired(request_time);- self.evict(cookie.domain, max_domain_cookies, purge_domain_cookies);- self.evict(null, max_cookies, purge_cookies);+ if (site) |g| {+ if (self.sites.values()[g].items.len >= max_domain_cookies) {+ self.removeExpiredAt(g, request_time);+ self.evictSite(key);+ }+ }+ if (self.len >= max_cookies) {+ self.removeExpired(request_time);+ self.evictJar();+ }++ // Looked up after eviction, which can drop this site's group.+ const g = self.sites.getIndex(key) orelse blk: {+ const owned_key = try self.allocator.dupe(u8, key);+ errdefer self.allocator.free(owned_key);+ try self.sites.putNoClobber(self.allocator, owned_key, .empty);+ break :blk self.sites.count() - 1;+ };+ const group = &self.sites.values()[g];// keep it in send order: longest path then creation time.- var pos = self.cookies.items.len;- while (pos > 0 and sendsBefore(&cookie, &self.cookies.items[pos - 1])) {+ var pos = group.items.len;+ while (pos > 0 and sendsBefore(&cookie, &group.items[pos - 1])) {pos -= 1;}- try self.cookies.insert(self.allocator, pos, cookie);+ try group.insert(self.allocator, pos, cookie);stored = true;- self.dispatchChange(.changed, &self.cookies.items[pos]);+ self.len += 1;+ self.dispatchChange(.changed, g, pos);}}@@ -756,9 +838,10 @@ pub const Jar = struct {// cookie's or the other way around (a leading `.` isn't compared), and// a path that cookie's path path-matches. An expired cookie is already// gone as far as the RFC is concerned, even if the jar still holds it.+ // Domains that domain-match are of the same site.fn shadowsSecureCookie(self: *const Jar, cookie: *const Cookie, now: u64) bool {const domain = std.mem.trimStart(u8, cookie.domain, ".");- for (self.cookies.items) |*c| {+ for (self.siteCookies(cookie.domain)) |*c| {if (isCookieExpired(c, now)) {continue;}@@ -774,47 +857,86 @@ pub const Jar = struct {return false;}- // Past `max` cookies of `domain`'s site (of the jar when null), evicts- // the least recently accessed, non-Secure ones first, until `purge`- // fewer are left, counting the one being added.- fn evict(self: *Jar, domain: ?[]const u8, max: usize, purge: usize) void {- // A site can't be at `max` before the jar is.- if (self.cookies.items.len < max) return;+ // RFC 6265bis 5.7 and Chrome: non-Secure cookies go first, then the least+ // recently accessed.+ fn evictsBefore(a: *const Cookie, b: *const Cookie) bool {+ if (a.secure != b.secure) return b.secure;+ return a.last_access_time < b.last_access_time;+ }- var count: usize = 0;- for (self.cookies.items) |*c| {- if (domain == null or areHostsSameSite(c.domain, domain.?)) count += 1;+ // Past `max_domain_cookies` for the site, evicts down to `purge_domain_cookies`+ // fewer, counting the one being added. Never empties the group.+ fn evictSite(self: *Jar, key: []const u8) void {+ const g = self.sites.getIndex(key) orelse return;+ const count = self.sites.values()[g].items.len;+ if (count < max_domain_cookies) return;+ for (max_domain_cookies - purge_domain_cookies - 1..count) |_| {+ const items = self.sites.values()[g].items;+ var oldest: usize = 0;+ for (items[1..], 1..) |*c, i| {+ if (evictsBefore(c, &items[oldest])) oldest = i;+ }+ self.remove(g, oldest);}- if (count < max) return;- for (max - purge - 1..count) |_| {- var oldest: ?usize = null;- for (self.cookies.items, 0..) |*c, i| {- if (domain != null and !areHostsSameSite(c.domain, domain.?)) continue;- if (oldest) |o| {- const b = &self.cookies.items[o];- // RFC 6265bis 5.7 and Chrome: non-Secure cookies go first.- const first = if (c.secure != b.secure) b.secure else c.last_access_time < b.last_access_time;- if (!first) continue;+ }++ // The same, past `max_cookies` in the jar, across sites.+ fn evictJar(self: *Jar) void {+ const count = self.len;+ if (count < max_cookies) return;+ for (max_cookies - purge_cookies - 1..count) |_| {+ var oldest: ?struct { g: usize, i: usize } = null;+ for (self.sites.values(), 0..) |*group, g| {+ for (group.items, 0..) |*c, i| {+ if (oldest) |o| {+ if (!evictsBefore(c, &self.sites.values()[o.g].items[o.i])) continue;+ }+ oldest = .{ .g = g, .i = i };}- oldest = i;}- self.remove(oldest.?);+ const o = oldest.?;+ self.remove(o.g, o.i);}}- fn remove(self: *Jar, index: usize) void {- const c = &self.cookies.items[index];- self.dispatchChange(.deleted, c);+ // Removes cookie `i` of group `g`, and the group once empty, which+ // shifts the groups after it.+ fn remove(self: *Jar, g: usize, i: usize) void {+ self.dispatchChange(.deleted, g, i);+ const group = &self.sites.values()[g];+ const c = &group.items[i];c.deinit();- _ = self.cookies.orderedRemove(index);+ _ = group.orderedRemove(i);+ self.len -= 1;+ if (group.items.len == 0) {+ self.dropSite(g);+ }}+ fn dropSite(self: *Jar, g: usize) void {+ self.sites.values()[g].deinit(self.allocator);+ const key = self.sites.keys()[g];+ self.sites.orderedRemoveAt(g);+ self.allocator.free(key);+ }++ // Listeners run while the caller can be mid-loop over the jar, so they+ // must not change it synchronously. In safe builds, the locks turn a+ // change that would move the cookie or the groups into a panic.fn dispatchChange(self: *Jar,kind: Notification.CookieChanged.Kind,- cookie: *const Cookie,+ g: usize,+ i: usize,) void {const notification = self.notification orelse return;+ const group = &self.sites.values()[g];+ self.sites.lockPointers();+ defer self.sites.unlockPointers();+ group.lockPointers();+ defer group.unlockPointers();++ const cookie = &group.items[i];notification.dispatch(.cookie_changed, &.{.kind = kind,.name = cookie.name,@@ -828,16 +950,28 @@ pub const Jar = struct {}pub fn removeExpired(self: *Jar, request_time: ?u64) void {- if (self.cookies.items.len == 0) return;const time = request_time orelse lp.datetime.timestamp(.real);- var i: usize = self.cookies.items.len;+ var g = self.sites.count();+ while (g > 0) {+ g -= 1;+ self.removeExpiredAt(g, time);+ }+ }++ // Can drop group `g`, shifting the groups after it.+ fn removeExpiredAt(self: *Jar, g: usize, now: u64) void {+ const group = &self.sites.values()[g];+ var i: usize = group.items.len;while (i > 0) {i -= 1;- const cookie = &self.cookies.items[i];- if (isCookieExpired(cookie, time)) {- self.cookies.orderedRemove(i).deinit();+ if (isCookieExpired(&group.items[i], now)) {+ group.orderedRemove(i).deinit();+ self.len -= 1;}}+ if (group.items.len == 0) {+ self.dropSite(g);+ }}const LookupOpts = struct {@@ -854,10 +988,13 @@ pub const Jar = struct {const same_site = areSameSite(opts.origin_url, target.host);const now = opts.request_time orelse lp.datetime.timestamp(.real);- removeExpired(self, now);+ const key = siteKey(target.host);+ if (self.sites.getIndex(key)) |g| {+ self.removeExpiredAt(g, now);+ }var first = true;- for (self.cookies.items) |*cookie| {+ for (self.siteCookies(target.host)) |*cookie| {const applies = cookie.appliesTo(&target, .{.same_site = same_site,.is_http = opts.is_http,@@ -943,6 +1080,16 @@ pub fn areHostsSameSite(target_host: []const u8, origin_host: []const u8) bool {return std.mem.eql(u8, findSecondLevelDomain(target_host), findSecondLevelDomain(origin_host));}+// The site of a cookie domain or a host: its registrable domain, like+// Chrome's CookieMonster key. A cookie only applies to hosts with its key.+fn siteKey(domain: []const u8) []const u8 {+ const host = if (domain.len > 0 and domain[0] == '.') domain[1..] else domain;+ // findSecondLevelDomain keeps a single label as is, and Domain=localhost+ // is the one single-label domain parseDomain accepts.+ if (std.mem.endsWith(u8, host, ".localhost")) return "localhost";+ return findSecondLevelDomain(host);+}+fn findSecondLevelDomain(host: []const u8) []const u8 {var i = std.mem.findScalarLast(u8, host, '.') orelse return host;while (true) {@@ -1062,9 +1209,10 @@ test "cookie: findSecondLevelDomain" {test "Jar: add" {const expectCookies = struct {fn expect(expected: []const struct { []const u8, []const u8 }, jar: Jar) !void {- try testing.expectEqual(expected.len, jar.cookies.items.len);+ try testing.expectEqual(expected.len, jar.len);LOOP: for (expected) |e| {- for (jar.cookies.items) |c| {+ var it = jar.iterator();+ while (it.next()) |c| {if (std.mem.eql(u8, e.@"0", c.name) and std.mem.eql(u8, e.@"1", c.value)) {continue :LOOP;}@@ -1113,16 +1261,16 @@ test "Jar: non-HTTP add must not replace or duplicate an HttpOnly cookie" {defer jar.deinit();try jar.add(try Cookie.parse(testing.allocator, test_url, "session=REAL;Path=/;HttpOnly"), now, true);- try testing.expectEqual(@as(usize, 1), jar.cookies.items.len);+ try testing.expectEqual(@as(usize, 1), jar.len);try jar.add(try Cookie.parse(testing.allocator, test_url, "session=ATTACKER;Path=/"), now, false);- try testing.expectEqual(@as(usize, 1), jar.cookies.items.len);- try testing.expectEqual("REAL", jar.cookies.items[0].value);- try testing.expectEqual(true, jar.cookies.items[0].http_only);+ try testing.expectEqual(@as(usize, 1), jar.len);+ try testing.expectEqual("REAL", nthCookie(&jar, 0).value);+ try testing.expectEqual(true, nthCookie(&jar, 0).http_only);try jar.add(try Cookie.parse(testing.allocator, test_url, "session=REFRESHED;Path=/;HttpOnly"), now, true);- try testing.expectEqual(@as(usize, 1), jar.cookies.items.len);- try testing.expectEqual("REFRESHED", jar.cookies.items[0].value);+ try testing.expectEqual(@as(usize, 1), jar.len);+ try testing.expectEqual("REFRESHED", nthCookie(&jar, 0).value);}test "Jar: add limit" {@@ -1143,12 +1291,19 @@ test "Jar: add limit" {}fn jarHas(jar: *const Jar, name: []const u8) bool {- for (jar.cookies.items) |*c| {+ var it = jar.iterator();+ while (it.next()) |c| {if (std.mem.eql(u8, c.name, name)) return true;}return false;}+fn nthCookie(jar: *const Jar, n: usize) *Cookie {+ var it = jar.iterator();+ for (0..n) |_| _ = it.next();+ return it.next().?;+}+test "Jar: past the limit of a site, the least recently accessed are evicted" {var jar = Jar.init(testing.allocator, null);defer jar.deinit();@@ -1181,7 +1336,7 @@ test "Jar: past the limit of a site, the least recently accessed are evicted" {// Down to 150 for lightpanda.io, the new cookie included: the 31 least// recently accessed non-Secure cookies, c11...c41, go.- try testing.expectEqual(max_domain_cookies - purge_domain_cookies + 1, jar.cookies.items.len);+ try testing.expectEqual(max_domain_cookies - purge_domain_cookies + 1, jar.len);try testing.expectEqual(true, jarHas(&jar, "o"));try testing.expectEqual(true, jarHas(&jar, "new"));for (0..max_domain_cookies) |i| {@@ -1205,7 +1360,7 @@ test "Jar: past a limit, expired cookies are evicted first" {try jar.add(try Cookie.parse(testing.allocator, test_url, "new=v"), now + 1000, true);// Removing them is enough: nothing else goes.- try testing.expectEqual(max_domain_cookies - 10 + 1, jar.cookies.items.len);+ try testing.expectEqual(max_domain_cookies - 10 + 1, jar.len);for (0..max_domain_cookies) |i| {const name = try std.fmt.bufPrint(&buf, "c{d}", .{i});try testing.expectEqual(i < 170, jarHas(&jar, name));@@ -1228,7 +1383,7 @@ test "Jar: past the limit of the jar, the least recently accessed are evicted" {}try jar.add(try Cookie.parse(testing.allocator, "http://new.test/", "new=v"), now + max_cookies, true);- try testing.expectEqual(max_cookies - purge_cookies, jar.cookies.items.len);+ try testing.expectEqual(max_cookies - purge_cookies, jar.len);try testing.expectEqual(true, jarHas(&jar, "new"));for (0..max_cookies) |i| {const name = try std.fmt.bufPrint(&buf, "c{d}", .{i});@@ -1237,7 +1392,38 @@ test "Jar: past the limit of the jar, the least recently accessed are evicted" {// Replacing a cookie evicts nothing.try jar.add(try Cookie.parse(testing.allocator, "http://new.test/", "new=w"), now + max_cookies + 1, true);- try testing.expectEqual(max_cookies - purge_cookies, jar.cookies.items.len);+ try testing.expectEqual(max_cookies - purge_cookies, jar.len);+}++test "Jar: cookies are grouped by site" {+ var jar = Jar.init(testing.allocator, null);+ defer jar.deinit();++ try jar.populateFromResponse("http://lightpanda.io/", "a=1");+ try jar.populateFromResponse("http://www.lightpanda.io/", "b=2; Domain=lightpanda.io");+ try jar.populateFromResponse("http://docs.lightpanda.io/", "c=3");+ try jar.populateFromResponse("http://lightpanda.dev/", "d=4");+ // Domain=localhost applies to its subdomains, which findSecondLevelDomain+ // alone would put in sites of their own.+ try jar.populateFromResponse("http://localhost/", "e=5; Domain=localhost");+ try testing.expectEqual(5, jar.len);+ try testing.expectEqual(3, jar.sites.count());++ try testing.expectEqual(3, jar.siteCookies("www.lightpanda.io").len);+ try testing.expectEqual(1, jar.siteCookies("lightpanda.dev").len);+ try testing.expectEqual(0, jar.siteCookies("lightpanda.com").len);+ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();+ try jar.forRequest("http://a.b.localhost/", &aw.writer, .{+ .origin_url = .{ .url = "http://a.b.localhost/" },+ .is_http = true,+ });+ try testing.expectEqual("e=5", aw.written());++ // Removing a site's last cookie drops its group.+ try jar.populateFromResponse("http://lightpanda.dev/", "d=; Max-Age=0");+ try testing.expectEqual(4, jar.len);+ try testing.expectEqual(2, jar.sites.count());}test "Jar: a non-secure origin can't set or overwrite a Secure cookie" {@@ -1245,7 +1431,7 @@ test "Jar: a non-secure origin can't set or overwrite a Secure cookie" {defer jar.deinit();try jar.populateFromResponse("http://lightpanda.io/", "s=1; Secure");- try testing.expectEqual(0, jar.cookies.items.len);+ try testing.expectEqual(0, jar.len);try jar.populateFromResponse("https://lightpanda.io/", "s=1; Secure");// No overwrite, no shadowing on a matching domain or a longer path, no@@ -1255,19 +1441,19 @@ test "Jar: a non-secure origin can't set or overwrite a Secure cookie" {try jar.populateFromResponse("http://lightpanda.io/", "s=4; Domain=lightpanda.io");try jar.populateFromResponse("http://www.lightpanda.io/", "s=5");try jar.populateFromResponse("http://lightpanda.io/", "s=; Max-Age=0");- try testing.expectEqual(1, jar.cookies.items.len);- try testing.expectEqual("1", jar.cookies.items[0].value);+ try testing.expectEqual(1, jar.len);+ try testing.expectEqual("1", nthCookie(&jar, 0).value);// Another name, domain, or a path the Secure cookie's path doesn't cover.try jar.populateFromResponse("http://lightpanda.io/", "t=1");try jar.populateFromResponse("http://lightpanda.dev/", "s=6");try jar.populateFromResponse("https://lightpanda.io/", "u=1; Secure; Path=/sub");try jar.populateFromResponse("http://lightpanda.io/", "u=2");- try testing.expectEqual(5, jar.cookies.items.len);+ try testing.expectEqual(5, jar.len);// Like Chrome, loopback origins are secure ones.try jar.populateFromResponse("http://localhost:3000/", "s=7; Secure");- try testing.expectEqual(6, jar.cookies.items.len);+ try testing.expectEqual(6, jar.len);}test "Jar: an expired Secure cookie doesn't block a non-secure origin" {@@ -1280,9 +1466,9 @@ test "Jar: an expired Secure cookie doesn't block a non-secure origin" {// Still in the jar (nothing purged it), but expired by `later`.const later = now + 20;try jar.addFromUrl("http://lightpanda.io/", try Cookie.parse(testing.allocator, "http://lightpanda.io/", "s=2"), later, true);- try testing.expectEqual(1, jar.cookies.items.len);- try testing.expectEqual("2", jar.cookies.items[0].value);- try testing.expectEqual(false, jar.cookies.items[0].secure);+ try testing.expectEqual(1, jar.len);+ try testing.expectEqual("2", nthCookie(&jar, 0).value);+ try testing.expectEqual(false, nthCookie(&jar, 0).secure);}test "Jar: forRequest" {@@ -1424,13 +1610,13 @@ test "Jar: forRequest" {.is_http = true,});- const l = jar.cookies.items.len;+ const l = jar.len;try expectCookies("global1=1", &jar, test_url, .{.request_time = now + 100,.origin_url = .{ .url = test_url },.is_http = true,});- try testing.expectEqual(l - 1, jar.cookies.items.len);+ try testing.expectEqual(l - 1, jar.len);// If you add more cases after this point, note that the above test removes// the 'global2' cookiediff --git a/src/browser/webapi/storage/CookieStore.zig b/src/browser/webapi/storage/CookieStore.zigindex 748846ba2..feb40ad5f 100644--- a/src/browser/webapi/storage/CookieStore.zig+++ b/src/browser/webapi/storage/CookieStore.zig@@ -355,7 +355,7 @@ fn matchCookies(const same_site = Cookie.areSameSite(exec.siteForCookies(), target.host);var items: std.ArrayList(CookieListItem) = .empty;- for (session.cookie_jar.cookies.items) |*cookie| {+ for (session.cookie_jar.siteCookies(target.host)) |*cookie| {// CookieStore exposes only cookies that script would see for the// current document. HttpOnly cookies stay hidden.if (cookie.appliesTo(&target, .{ .same_site = same_site, .is_http = false }) == false) {diff --git a/src/cookies.zig b/src/cookies.zigindex 2237336de..fc722f676 100644--- a/src/cookies.zig+++ b/src/cookies.zig@@ -102,12 +102,14 @@ fn _saveToFile(jar: *Cookie.Jar, path: []const u8) !void {try writer.end();try af.replace(lp.io);- log.info(.app, "Cookie.saveToFile", .{ .path = path, .count = jar.cookies.items.len });+ log.info(.app, "Cookie.saveToFile", .{ .path = path, .count = jar.len });}fn writeJson(jar: *const Cookie.Jar, w: *std.Io.Writer) !void {try w.writeByte('[');- for (jar.cookies.items, 0..) |c, i| {+ var it = jar.iterator();+ var i: usize = 0;+ while (it.next()) |c| : (i += 1) {if (i > 0) {try w.writeByte(',');}@@ -127,7 +129,7 @@ fn writeJson(jar: *const Cookie.Jar, w: *std.Io.Writer) !void {}, .{ .emit_null_optional_fields = false }, w);}- if (jar.cookies.items.len > 0) {+ if (jar.len > 0) {try w.writeByte('\n');}try w.writeAll("]\n");@@ -151,7 +153,8 @@ const JsonCookie = struct {// Allocate before the struct literal copies `arena` into the result.const name = try a.dupe(u8, self.name);const value = try a.dupe(u8, self.value);- const domain = try a.dupe(u8, self.domain);+ // Cookie domains are case-insensitive; the jar keys them lower case.+ const domain = std.ascii.lowerString(try a.alloc(u8, self.domain.len), self.domain);const path = if (self.path) |p| try a.dupe(u8, p) else "/";// A missing or unrecognised sameSite leaves the cookie unspecified.@@ -289,6 +292,8 @@ pub const NetscapeFormat = struct {try allocator.print(".{s}", .{bare_domain})elsetry allocator.dupe(u8, bare_domain);+ // Cookie domains are case-insensitive; the jar keys them lower case.+ _ = std.ascii.lowerString(domain, domain);// Bake a cookie.const cookie = Cookie{@@ -324,6 +329,12 @@ pub const NetscapeFormat = struct {}};+fn nthCookie(jar: *const Cookie.Jar, n: usize) *Cookie {+ var it = jar.iterator();+ for (0..n) |_| _ = it.next();+ return it.next().?;+}+test "cookies: netscape include_subdomains drives the domain's leading dot" {var jar = Cookie.Jar.init(std.testing.allocator, null);defer jar.deinit();@@ -338,15 +349,15 @@ test "cookies: netscape include_subdomains drives the domain's leading dot" {".example.com\tFALSE\t/\tFALSE\t4102444800\td\t4\n";try std.testing.expectEqual(@as(usize, 4), try NetscapeFormat.parse(&jar, content));- try std.testing.expectEqualStrings(".example.com", jar.cookies.items[0].domain);- try std.testing.expectEqualStrings(".example.com", jar.cookies.items[1].domain);- try std.testing.expectEqualStrings("example.com", jar.cookies.items[2].domain);- try std.testing.expectEqualStrings("example.com", jar.cookies.items[3].domain);+ try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 0).domain);+ try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 1).domain);+ try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 2).domain);+ try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 3).domain);// That dot is what `Cookie.matchesHost` keys tail-matching off.- try std.testing.expect(jar.cookies.items[0].matchesHost("www.example.com"));- try std.testing.expect(jar.cookies.items[2].matchesHost("www.example.com") == false);- try std.testing.expect(jar.cookies.items[2].matchesHost("example.com"));+ try std.testing.expect(nthCookie(&jar, 0).matchesHost("www.example.com"));+ try std.testing.expect(nthCookie(&jar, 2).matchesHost("www.example.com") == false);+ try std.testing.expect(nthCookie(&jar, 2).matchesHost("example.com"));}test "cookies: netscape #HttpOnly_ prefix sets http_only" {@@ -362,13 +373,13 @@ test "cookies: netscape #HttpOnly_ prefix sets http_only" {try std.testing.expectEqual(@as(usize, 3), try NetscapeFormat.parse(&jar, content));- try std.testing.expect(jar.cookies.items[0].http_only);- try std.testing.expectEqualStrings("example.com", jar.cookies.items[0].domain);+ try std.testing.expect(nthCookie(&jar, 0).http_only);+ try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 0).domain);- try std.testing.expect(jar.cookies.items[1].http_only);- try std.testing.expectEqualStrings(".example.com", jar.cookies.items[1].domain);+ try std.testing.expect(nthCookie(&jar, 1).http_only);+ try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 1).domain);- try std.testing.expect(jar.cookies.items[2].http_only == false);+ try std.testing.expect(nthCookie(&jar, 2).http_only == false);}test "cookies: netscape treats #HttpOnly_ near-misses as comments" {@@ -387,7 +398,7 @@ test "cookies: netscape treats #HttpOnly_ near-misses as comments" {"#HttpOnly_\n";try std.testing.expectEqual(@as(usize, 0), try NetscapeFormat.parse(&jar, content));- try std.testing.expectEqual(@as(usize, 0), jar.cookies.items.len);+ try std.testing.expectEqual(@as(usize, 0), jar.len);}test "cookies: netscape 0 and empty expires load as session cookies" {@@ -408,16 +419,16 @@ test "cookies: netscape 0 and empty expires load as session cookies" {"example.com\tFALSE\t/\tFALSE\t4102444800\tfuture\t6\n";_ = try NetscapeFormat.parse(&jar, content);- try std.testing.expectEqual(@as(usize, 3), jar.cookies.items.len);+ try std.testing.expectEqual(@as(usize, 3), jar.len);- try std.testing.expectEqualStrings("session", jar.cookies.items[0].name);- try std.testing.expectEqual(@as(?f64, null), jar.cookies.items[0].expires);+ try std.testing.expectEqualStrings("session", nthCookie(&jar, 0).name);+ try std.testing.expectEqual(@as(?f64, null), nthCookie(&jar, 0).expires);- try std.testing.expectEqualStrings("python", jar.cookies.items[1].name);- try std.testing.expectEqual(@as(?f64, null), jar.cookies.items[1].expires);+ try std.testing.expectEqualStrings("python", nthCookie(&jar, 1).name);+ try std.testing.expectEqual(@as(?f64, null), nthCookie(&jar, 1).expires);- try std.testing.expectEqualStrings("future", jar.cookies.items[2].name);- try std.testing.expectEqual(@as(?f64, 4102444800), jar.cookies.items[2].expires);+ try std.testing.expectEqualStrings("future", nthCookie(&jar, 2).name);+ try std.testing.expectEqual(@as(?f64, 4102444800), nthCookie(&jar, 2).expires);}test "cookies: load JSON accepts CDP SameSite casing" {@@ -452,6 +463,6 @@ test "cookies: load JSON keeps a session cookie with expires -1" {var jar = Cookie.Jar.init(std.testing.allocator, null);defer jar.deinit();try jar.add(try parsed[0].toCookie(jar.allocator), lp.datetime.timestamp(.real), true);- try std.testing.expectEqual(1, jar.cookies.items.len);- try std.testing.expectEqual(null, jar.cookies.items[0].expires);+ try std.testing.expectEqual(1, jar.len);+ try std.testing.expectEqual(null, nthCookie(&jar, 0).expires);}diff --git a/src/server/bidi/http_command.zig b/src/server/bidi/http_command.zigindex f3315c02f..c25679088 100644--- a/src/server/bidi/http_command.zig+++ b/src/server/bidi/http_command.zig@@ -865,7 +865,7 @@ fn getAllCookies(cmd: *BiDi.Command) !void {const associated: storage.Associated = .init(frame.url, null);var cookies: std.ArrayList(storage.HttpCookie) = .empty;- for (jar.cookies.items) |*cookie| {+ for (jar.siteCookies(associated.url.host)) |*cookie| {if (associated.has(cookie)) {try cookies.append(cmd.arena, .{ .cookie = cookie });}@@ -883,7 +883,7 @@ fn getNamedCookie(cmd: *BiDi.Command, p: CookieName) !void {jar.removeExpired(null);const associated: storage.Associated = .init(frame.url, p.name);- for (jar.cookies.items) |*cookie| {+ for (jar.siteCookies(associated.url.host)) |*cookie| {if (associated.has(cookie)) {return cmd.sendResult(storage.HttpCookie{ .cookie = cookie });}@@ -936,7 +936,7 @@ fn addCookie(cmd: *BiDi.Command, p: AddCookie) !void {fn deleteCookies(cmd: *BiDi.Command, name: ?[]const u8) !void {const frame = (try currentFrame(cmd)) orelse return;const associated: storage.Associated = .init(frame.url, name);- storage.remove(&cmd.bidi.user_context.session.cookie_jar, &associated, storage.Associated.has);+ cmd.bidi.user_context.session.cookie_jar.removeWhere(&associated, storage.Associated.has);return cmd.sendDone();}diff --git a/src/server/bidi/storage.zig b/src/server/bidi/storage.zigindex f6aa71068..53604c9ab 100644--- a/src/server/bidi/storage.zig+++ b/src/server/bidi/storage.zig@@ -81,18 +81,6 @@ pub const Associated = struct {}};-// Removes the cookies `matches` accepts, `context` being its first argument.-pub fn remove(jar: *Cookie.Jar, context: anytype, comptime matches: fn (@TypeOf(context), *const Cookie) bool) void {- const cookies = &jar.cookies;- var i = cookies.items.len;- while (i > 0) {- i -= 1;- if (matches(context, &cookies.items[i])) {- cookies.orderedRemove(i).deinit();- }- }-}-// A cookie in the HTTP session's shape.pub const HttpCookie = struct {cookie: *const Cookie,@@ -277,7 +265,8 @@ fn getCookies(cmd: *BiDi.Command) !void {jar.removeExpired(null);var cookies: std.ArrayList(BiDiCookie) = .empty;- for (jar.cookies.items) |*cookie| {+ var it = jar.iterator();+ while (it.next()) |cookie| {if (filter.matches(cookie)) {try cookies.append(cmd.arena, .{ .cookie = cookie });}@@ -337,7 +326,7 @@ fn deleteCookies(cmd: *BiDi.Command) !void {var filter = p.filter;filter.decode(cmd.arena) catch return cmd.sendError("invalid argument", "invalid filter.value");- remove(&cmd.bidi.user_context.session.cookie_jar, &filter, Filter.matches);+ cmd.bidi.user_context.session.cookie_jar.removeWhere(&filter, Filter.matches);return cmd.sendResult(.{ .partitionKey = .{ .userContext = user_context } });}@@ -386,8 +375,9 @@ test "bidi.storage: cookies" {try ctx.processMessage(.{ .id = 5, .method = "storage.deleteCookies", .params = .{ .filter = .{ .name = "a" } } });try ctx.expectSentResult(.{ .partitionKey = .{ .userContext = "default" } }, .{ .id = 5 });- try testing.expectEqual(1, ctx.bidi().user_context.session.cookie_jar.cookies.items.len);- try testing.expectEqual("b", ctx.bidi().user_context.session.cookie_jar.cookies.items[0].name);+ try testing.expectEqual(1, ctx.bidi().user_context.session.cookie_jar.len);+ var it = ctx.bidi().user_context.session.cookie_jar.iterator();+ try testing.expectEqual("b", it.next().?.name);// SameSite=None needs Securetry ctx.processMessage(.{@@ -419,5 +409,5 @@ test "bidi.storage: cookies" {// clears the resttry ctx.processMessage(.{ .id = 11, .method = "storage.deleteCookies", .params = struct {}{} });try ctx.expectSentResult(.{ .partitionKey = .{ .userContext = "default" } }, .{ .id = 11 });- try testing.expectEqual(0, ctx.bidi().user_context.session.cookie_jar.cookies.items.len);+ try testing.expectEqual(0, ctx.bidi().user_context.session.cookie_jar.len);}diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 6ec87c9ee..a82aa4d4a 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -1830,14 +1830,15 @@ test "cdp: --cookie-jar saves on dispose and --cookie loads it back" {{// no file yet: the context starts with an empty jarconst bc = try ctx.loadBrowserContext(.{});- try testing.expectEqual(0, bc.session.cookie_jar.cookies.items.len);+ try testing.expectEqual(0, bc.session.cookie_jar.len);try bc.session.cookie_jar.populateFromResponse("https://example.com/", "sid=abc; Path=/; Max-Age=3600; HttpOnly");ctx.cdp().disposeBrowserContext();}// a new context (a reconnect, or a restart) starts logged inconst bc = try ctx.loadBrowserContext(.{});- const cookies = bc.session.cookie_jar.cookies.items;- try testing.expectEqual(1, cookies.len);- try testing.expectEqualSlices(u8, "abc", cookies[0].value);+ const jar = &bc.session.cookie_jar;+ try testing.expectEqual(1, jar.len);+ var it = jar.iterator();+ try testing.expectEqualSlices(u8, "abc", it.next().?.value);}diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex 90269b84f..3ccb16250 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -280,20 +280,28 @@ fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {const Cookie = @import("../../../browser/webapi/storage/storage.zig").Cookie;-// Only matches the cookie on provided parameters-fn cookieMatches(cookie: *const Cookie, name: []const u8, domain: ?[]const u8, path: ?[]const u8) bool {- if (!std.mem.eql(u8, cookie.name, name)) return false;-- if (domain) |domain_| {- const c_no_dot = if (std.mem.startsWith(u8, cookie.domain, ".")) cookie.domain[1..] else cookie.domain;- const d_no_dot = if (std.mem.startsWith(u8, domain_, ".")) domain_[1..] else domain_;- if (!std.mem.eql(u8, c_no_dot, d_no_dot)) return false;- }- if (path) |path_| {- if (!std.mem.eql(u8, cookie.path, path_)) return false;+// Only matches the cookie on provided parameters. We do not want to use+// Cookie.appliesTo here, as a Cookie with a shorter path would match. Similar+// to deduplicating with areCookiesEqual, except domain and path are optional.+const CookieMatch = struct {+ name: []const u8,+ domain: ?[]const u8,+ path: ?[]const u8,++ fn matches(self: *const CookieMatch, cookie: *const Cookie) bool {+ if (!std.mem.eql(u8, cookie.name, self.name)) return false;++ if (self.domain) |domain_| {+ const c_no_dot = if (std.mem.startsWith(u8, cookie.domain, ".")) cookie.domain[1..] else cookie.domain;+ const d_no_dot = if (std.mem.startsWith(u8, domain_, ".")) domain_[1..] else domain_;+ if (!std.mem.eql(u8, c_no_dot, d_no_dot)) return false;+ }+ if (self.path) |path_| {+ if (!std.mem.eql(u8, cookie.path, path_)) return false;+ }+ return true;}- return true;-}+};fn deleteCookies(cmd: *CDP.Command) !void {const params = (try cmd.params(struct {@@ -311,21 +319,12 @@ fn deleteCookies(cmd: *CDP.Command) !void {}const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;- const cookies = &bc.session.cookie_jar.cookies;-- var index = cookies.items.len;- while (index > 0) {- index -= 1;- const cookie = &cookies.items[index];- const domain = try Cookie.parseDomain(cmd.arena, params.url, params.domain);- const path = try Cookie.parsePath(cmd.arena, params.url, params.path);-- // We do not want to use Cookie.appliesTo here. As a Cookie with a shorter path would match.- // Similar to deduplicating with areCookiesEqual, except domain and path are optional.- if (cookieMatches(cookie, params.name, domain, path)) {- cookies.orderedRemove(index).deinit();- }- }+ const match: CookieMatch = .{+ .name = params.name,+ .domain = try Cookie.parseDomain(cmd.arena, params.url, params.domain),+ .path = try Cookie.parsePath(cmd.arena, params.url, params.path),+ };+ bc.session.cookie_jar.removeWhere(&match, CookieMatch.matches);return cmd.sendResult(null, .{});}@@ -401,7 +400,7 @@ fn getCookies(cmd: *CDP.Command) !void {var jar = &bc.session.cookie_jar;jar.removeExpired(null);- const writer = CdpStorage.CookieWriter{ .cookies = jar.cookies.items, .urls = urls.items };+ const writer = CdpStorage.CookieWriter{ .jar = jar, .urls = urls.items };try cmd.sendResult(.{ .cookies = writer }, .{});}@@ -413,7 +412,7 @@ fn getAllCookies(cmd: *CDP.Command) !void {const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;var jar = &bc.session.cookie_jar;jar.removeExpired(null);- const writer = CdpStorage.CookieWriter{ .cookies = jar.cookies.items };+ const writer = CdpStorage.CookieWriter{ .jar = jar };try cmd.sendResult(.{ .cookies = writer }, .{});}diff --git a/src/server/cdp/domains/storage.zig b/src/server/cdp/domains/storage.zigindex e38c7a6f7..14a216032 100644--- a/src/server/cdp/domains/storage.zig+++ b/src/server/cdp/domains/storage.zig@@ -69,7 +69,7 @@ fn getCookies(cmd: *CDP.Command) !void {}}bc.session.cookie_jar.removeExpired(null);- const writer = CookieWriter{ .cookies = bc.session.cookie_jar.cookies.items };+ const writer = CookieWriter{ .jar = &bc.session.cookie_jar };try cmd.sendResult(.{ .cookies = writer }, .{});}@@ -234,7 +234,7 @@ fn parseSameSite(value: ?[]const u8) ?Cookie.SameSite {}pub const CookieWriter = struct {- cookies: []const Cookie,+ jar: *const Cookie.Jar,urls: ?[]const PreparedUri = null,pub fn jsonStringify(self: *const CookieWriter, w: anytype) !void {@@ -247,8 +247,9 @@ pub const CookieWriter = struct {fn writeCookies(self: CookieWriter, w: anytype) !void {try w.beginArray();+ var it = self.jar.iterator();if (self.urls) |urls| {- for (self.cookies) |*cookie| {+ while (it.next()) |cookie| {for (urls) |*url| {if (cookie.appliesTo(url, .{ .same_site = true, .is_http = true, .kind = .navigation })) { // TBD same_site, should we compare to the pages url?try writeCookie(cookie, w);@@ -257,7 +258,7 @@ pub const CookieWriter = struct {}}} else {- for (self.cookies) |*cookie| {+ while (it.next()) |cookie| {try writeCookie(cookie, w);}}@@ -323,13 +324,13 @@ test "cdp.Storage: setCookies takes back the cookies getCookies gave" {const cookie: CdpCookie = .{ .name = "a", .value = "v", .domain = "example.com", .secure = true, .priority = "High", .sourceScheme = .Secure, .sourcePort = 443 };try ctx.processMessage(.{ .id = 1, .method = "Storage.setCookies", .params = .{ .cookies = &[_]CdpCookie{cookie} } });try ctx.expectSentResult(null, .{ .id = 1 });- try testing.expectEqual(1, ctx.cdp().browser_context.?.session.cookie_jar.cookies.items.len);+ try testing.expectEqual(1, ctx.cdp().browser_context.?.session.cookie_jar.len);var expired = cookie;expired.expires = 1;try ctx.processMessage(.{ .id = 2, .method = "Storage.setCookies", .params = .{ .cookies = &[_]CdpCookie{expired} } });try ctx.expectSentResult(null, .{ .id = 2 });- try testing.expectEqual(0, ctx.cdp().browser_context.?.session.cookie_jar.cookies.items.len);+ try testing.expectEqual(0, ctx.cdp().browser_context.?.session.cookie_jar.len);// getCookies reports a session cookie with expires: -1.var session = cookie;@@ -337,8 +338,9 @@ test "cdp.Storage: setCookies takes back the cookies getCookies gave" {try ctx.processMessage(.{ .id = 5, .method = "Storage.setCookies", .params = .{ .cookies = &[_]CdpCookie{session} } });try ctx.expectSentResult(null, .{ .id = 5 });const jar = &ctx.cdp().browser_context.?.session.cookie_jar;- try testing.expectEqual(1, jar.cookies.items.len);- try testing.expectEqual(null, jar.cookies.items[0].expires);+ try testing.expectEqual(1, jar.len);+ var it = jar.iterator();+ try testing.expectEqual(null, it.next().?.expires);// What Chrome refuses among those fields.var bad_port = cookie;