Lightpanda: the headless browser designed for AI and automation
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const Session = @import("browser/Session.zig");
const Cookie = @import("browser/webapi/storage/Cookie.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
/// Load cookies from a JSON file into the cookie jar.
/// The file format is an array of objects with: name, value, domain, path,
/// expires (optional, float), secure (optional, bool), httpOnly (optional, bool),
/// sameSite (optional, Strict/Lax/None).
/// This matches the CDP Network.Cookie format used by Puppeteer and Playwright.
pub fn loadFromFile(session: *Session, path: []const u8) void {
_loadFromFile(session, path) catch |err| {
log.err(.app, "Cookie.loadFromFile", .{ .err = err, .path = path });
};
}
fn _loadFromFile(session: *Session, path: []const u8) !void {
const arena = try session.getArena(.medium, "Cookies.loadFromFile");
defer arena.release();
const content = std.Io.Dir.cwd().readFileAlloc(lp.io, path, arena.allocator(), .limited(1024 * 1024)) catch |err| {
switch (err) {
error.FileNotFound => log.debug(.app, "Cookie.readFile", .{ .path = path, .note = "file not found" }),
else => log.err(.app, "Cookie.readFile", .{ .path = path, .err = err }),
}
return;
};
const jar = &session.cookie_jar;
// The file is either a CDP-style JSON array or the Netscape format. Sniff which one it is.
const head = std.mem.trimStart(u8, content, &std.ascii.whitespace);
if (head.len == 0) {
log.debug(.app, "Cookie.parseFile", .{ .path = path, .note = "empty file" });
return;
}
if (head[0] != '[' and head[0] != '{') {
const loaded = try NetscapeFormat.parse(jar, content);
log.info(.app, "Cookie.loadFromFile", .{ .path = path, .count = loaded });
return;
}
const json_cookies = std.json.parseFromSliceLeaky([]const JsonCookie, arena.allocator(), content, .{
.ignore_unknown_fields = true,
}) catch |err| {
log.err(.app, "Cookie.parseFile", .{ .path = path, .err = err });
return;
};
const now = lp.datetime.timestamp(.real);
var loaded: usize = 0;
for (json_cookies) |jc| {
const cookie = try jc.toCookie(jar.allocator);
jar.add(cookie, now, true) catch |err| {
log.warn(.app, "invalid cookie", .{ .name = jc.name, .err = err });
continue;
};
loaded += 1;
}
log.info(.app, "Cookie.loadFromFile", .{ .path = path, .count = loaded });
}
/// Save all cookies from the jar to a JSON file.
pub fn saveToFile(jar: *Cookie.Jar, path: []const u8) void {
_saveToFile(jar, path) catch |err| {
log.err(.app, "Cookie.saveToFile", .{ .path = path, .err = err });
};
}
fn _saveToFile(jar: *Cookie.Jar, path: []const u8) !void {
jar.removeExpired(null);
// Concurrent serve connections may save the same path: the last rename
// wins, but no reader sees a torn file.
var af = try std.Io.Dir.cwd().createFileAtomic(lp.io, path, .{ .replace = true });
defer af.deinit(lp.io);
var buf: [8192]u8 = undefined;
var writer = af.file.writer(lp.io, &buf);
try writeJson(jar, &writer.interface);
try writer.end();
try af.replace(lp.io);
log.info(.app, "Cookie.saveToFile", .{ .path = path, .count = jar.len });
}
fn writeJson(jar: *const Cookie.Jar, w: *std.Io.Writer) !void {
try w.writeByte('[');
var it = jar.iterator();
var i: usize = 0;
while (it.next()) |c| : (i += 1) {
if (i > 0) {
try w.writeByte(',');
}
try w.writeAll("\n ");
try std.json.Stringify.value(JsonCookie{
.name = c.name,
.value = c.value,
.domain = c.domain,
.path = c.path,
.expires = c.expires,
.secure = c.secure,
.httpOnly = c.http_only,
// Left out for an unspecified cookie, as Network.getCookies
// does, so that it's still Lax by default once loaded back.
.sameSite = if (c.same_site_default) null else @tagName(c.same_site),
}, .{ .emit_null_optional_fields = false }, w);
}
if (jar.len > 0) {
try w.writeByte('\n');
}
try w.writeAll("]\n");
}
const JsonCookie = struct {
name: []const u8,
value: []const u8,
domain: []const u8,
path: ?[]const u8 = "/",
expires: ?f64 = null,
secure: ?bool = null,
httpOnly: ?bool = null,
sameSite: ?[]const u8 = null,
fn toCookie(self: JsonCookie, allocator: Allocator) !Cookie {
var arena = std.heap.ArenaAllocator.init(allocator);
errdefer arena.deinit();
const a = arena.allocator();
// Allocate before the struct literal copies `arena` into the result.
const name = try a.dupe(u8, self.name);
const value = try a.dupe(u8, self.value);
// Cookie domains are case-insensitive; the jar keys them lower case.
const domain = std.ascii.lowerString(try a.alloc(u8, self.domain.len), self.domain);
const path = if (self.path) |p| try a.dupe(u8, p) else "/";
// A missing or unrecognised sameSite leaves the cookie unspecified.
const same_site = Cookie.SameSite.parse(self.sameSite);
return .{
.arena = arena,
.name = name,
.value = value,
.domain = domain,
.path = path,
// A session cookie is written with expires: -1.
.expires = if (self.expires) |e| (if (e < 0) null else e) else null,
.secure = self.secure orelse false,
.http_only = self.httpOnly orelse false,
.same_site = same_site orelse .lax,
.same_site_default = same_site == null,
};
}
};
/// Netscape cookie file format parser with `#HttpOnly_` addition from curl.
/// https://docs.cyotek.com/cyowcopy/1.10/netscapecookieformat.html
/// https://curl.se/docs/http-cookies.html
pub const NetscapeFormat = struct {
/// Parses and loads cookies in Netscape format.
/// Returns number of cookies loaded.
pub fn parse(jar: *Cookie.Jar, slice: []const u8) !usize {
const now = lp.datetime.timestamp(.real);
var loaded: usize = 0;
var line_iterator = std.mem.splitScalar(u8, slice, '\n');
iterate_lines: while (line_iterator.next()) |line| {
if (line.len == 0) {
continue :iterate_lines;
}
// Remove CR if there's one.
var s = if (line[line.len - 1] == '\r') line[0 .. line.len - 1] else line;
// Skip if nothing left.
if (s.len == 0) {
continue :iterate_lines;
}
// Computed here since this doesn't have it's own column.
var is_http_only = false;
if (s[0] == '#') {
// Is it continued with `HttpOnly_`?
is_http_only =
s.len >= 10 and
@as(u64, @bitCast(s[1..9].*)) == @as(u64, @bitCast(@as([]const u8, "HttpOnly")[0..8].*)) and
s[9] == '_';
// Regular comment; skip the line.
if (!is_http_only) {
continue :iterate_lines;
}
// Advance.
s = s[10..];
}
var fields: struct {
domain: []const u8 = undefined,
include_subdomains: bool = false,
/// null means "/".
path: ?[]const u8 = null,
secure: bool = false,
/// null means a session cookie.
expires: ?f64 = null,
name: []const u8 = undefined,
value: []const u8 = "",
} = .{};
// Iterate over columns.
var column_index: usize = 0;
var column_iterator = std.mem.splitScalar(u8, s, '\t');
while (column_iterator.next()) |column| {
defer column_index += 1;
switch (column_index) {
0 => fields.domain = column,
1 => fields.include_subdomains = parseBool(column) catch continue :iterate_lines,
2 => {
// If this is a boolean, we have to set `secure` field here instead.
const secure = parseBool(column) catch {
// Not a boolean, set path.
fields.path = column;
continue;
};
fields.secure = secure;
// Parsed secure early, we have to advance once more.
column_index += 1;
},
3 => fields.secure = parseBool(column) catch continue :iterate_lines,
4 => {
// A session cookie is written as `0` by curl, wget and
// yt-dlp, and as an empty column by Python's
// http.cookiejar. Both leave `expires` null: `Jar.add`
// drops anything that expires at or before now, so a
// literal 0 would be discarded on the spot.
if (column.len == 0) {
continue;
}
const expires = std.fmt.parseFloat(f64, column) catch continue :iterate_lines;
if (!std.math.isFinite(expires)) {
continue :iterate_lines;
}
if (expires != 0) {
fields.expires = expires;
}
},
5 => fields.name = column,
6 => fields.value = column,
// Indicates we got columns more than we expected.
else => continue :iterate_lines,
}
}
// We need at least 6 columns filled (value can be empty).
if (column_index < 6) {
continue :iterate_lines;
}
var cookie_arena = std.heap.ArenaAllocator.init(jar.allocator);
errdefer cookie_arena.deinit();
const allocator = cookie_arena.allocator();
const name = try allocator.dupe(u8, fields.name);
const value = try allocator.dupe(u8, fields.value);
const _path = if (fields.path) |path| try allocator.dupe(u8, path) else "/";
// The domain column's leading dot and the `include_subdomains`
// column can disagree. curl strips the dot on read and re-derives
// it from the column, and `Cookie.matchesHost` keys tail-matching
// off that dot, so the column is what decides it here.
const bare_domain = if (fields.domain.len > 0 and fields.domain[0] == '.') fields.domain[1..] else fields.domain;
const domain = if (fields.include_subdomains)
try allocator.print(".{s}", .{bare_domain})
else
try allocator.dupe(u8, bare_domain);
// Cookie domains are case-insensitive; the jar keys them lower case.
_ = std.ascii.lowerString(domain, domain);
// Bake a cookie.
const cookie = Cookie{
.arena = cookie_arena,
.name = name,
.value = value,
.domain = domain,
.path = _path,
.expires = fields.expires,
.secure = fields.secure,
.http_only = is_http_only,
.same_site = .none,
};
jar.add(cookie, now, true) catch |err| {
log.warn(.app, "invalid cookie", .{ .name = fields.name, .err = err });
continue :iterate_lines;
};
loaded += 1;
}
return loaded;
}
fn parseBool(s: []const u8) error{Invalid}!bool {
if (std.ascii.eqlIgnoreCase(s, "false")) {
return false;
}
if (std.ascii.eqlIgnoreCase(s, "true")) {
return true;
}
return error.Invalid;
}
};
fn nthCookie(jar: *const Cookie.Jar, n: usize) *Cookie {
var it = jar.iterator();
for (0..n) |_| _ = it.next();
return it.next().?;
}
test "cookies: netscape include_subdomains drives the domain's leading dot" {
var jar = Cookie.Jar.init(std.testing.allocator, null);
defer jar.deinit();
// The domain column and the flag disagree on two of these four lines; the
// flag wins either way. Expiry is far future so nothing is dropped as stale.
const content =
"# Netscape HTTP Cookie File\n" ++
"example.com\tTRUE\t/\tFALSE\t4102444800\ta\t1\n" ++
".example.com\tTRUE\t/\tFALSE\t4102444800\tb\t2\n" ++
"example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n" ++
".example.com\tFALSE\t/\tFALSE\t4102444800\td\t4\n";
try std.testing.expectEqual(@as(usize, 4), try NetscapeFormat.parse(&jar, content));
try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 0).domain);
try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 1).domain);
try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 2).domain);
try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 3).domain);
// That dot is what `Cookie.matchesHost` keys tail-matching off.
try std.testing.expect(nthCookie(&jar, 0).matchesHost("www.example.com"));
try std.testing.expect(nthCookie(&jar, 2).matchesHost("www.example.com") == false);
try std.testing.expect(nthCookie(&jar, 2).matchesHost("example.com"));
}
test "cookies: netscape #HttpOnly_ prefix sets http_only" {
var jar = Cookie.Jar.init(std.testing.allocator, null);
defer jar.deinit();
// curl writes the prefix ahead of the domain column and strips it before
// reading the columns, so it composes with the include_subdomains dot.
const content =
"#HttpOnly_example.com\tFALSE\t/\tFALSE\t4102444800\ta\t1\n" ++
"#HttpOnly_.example.com\tTRUE\t/\tFALSE\t4102444800\tb\t2\n" ++
"example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n";
try std.testing.expectEqual(@as(usize, 3), try NetscapeFormat.parse(&jar, content));
try std.testing.expect(nthCookie(&jar, 0).http_only);
try std.testing.expectEqualStrings("example.com", nthCookie(&jar, 0).domain);
try std.testing.expect(nthCookie(&jar, 1).http_only);
try std.testing.expectEqualStrings(".example.com", nthCookie(&jar, 1).domain);
try std.testing.expect(nthCookie(&jar, 2).http_only == false);
}
test "cookies: netscape treats #HttpOnly_ near-misses as comments" {
var jar = Cookie.Jar.init(std.testing.allocator, null);
defer jar.deinit();
// Every line here is a comment. curl's check is a case-sensitive
// `strncmp(lineptr, "#HttpOnly_", 10)`, so none of these are the prefix,
// and a bare prefix with no columns after it has nothing to load.
const content =
"# Netscape HTTP Cookie File\n" ++
"#\n" ++
"# HttpOnly_example.com\tFALSE\t/\tFALSE\t4102444800\ta\t1\n" ++
"#HttpOnlyX example.com\tFALSE\t/\tFALSE\t4102444800\tb\t2\n" ++
"#httponly_example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n" ++
"#HttpOnly_\n";
try std.testing.expectEqual(@as(usize, 0), try NetscapeFormat.parse(&jar, content));
try std.testing.expectEqual(@as(usize, 0), jar.len);
}
test "cookies: netscape 0 and empty expires load as session cookies" {
var jar = Cookie.Jar.init(std.testing.allocator, null);
defer jar.deinit();
// curl, wget and yt-dlp write `0` for a session cookie and Python's
// http.cookiejar writes an empty column. Neither may be handed to
// `Jar.add` as a real timestamp, since it drops anything at or before
// now. A genuine past timestamp is still dropped, and a non-finite or
// non-numeric one rejects the line.
const content =
"example.com\tFALSE\t/\tFALSE\t0\tsession\t1\n" ++
"example.com\tFALSE\t/\tFALSE\t\tpython\t2\n" ++
"example.com\tFALSE\t/\tFALSE\t1\tstale\t3\n" ++
"example.com\tFALSE\t/\tFALSE\tnan\tnan\t4\n" ++
"example.com\tFALSE\t/\tFALSE\tsoon\tbogus\t5\n" ++
"example.com\tFALSE\t/\tFALSE\t4102444800\tfuture\t6\n";
_ = try NetscapeFormat.parse(&jar, content);
try std.testing.expectEqual(@as(usize, 3), jar.len);
try std.testing.expectEqualStrings("session", nthCookie(&jar, 0).name);
try std.testing.expectEqual(@as(?f64, null), nthCookie(&jar, 0).expires);
try std.testing.expectEqualStrings("python", nthCookie(&jar, 1).name);
try std.testing.expectEqual(@as(?f64, null), nthCookie(&jar, 1).expires);
try std.testing.expectEqualStrings("future", nthCookie(&jar, 2).name);
try std.testing.expectEqual(@as(?f64, 4102444800), nthCookie(&jar, 2).expires);
}
test "cookies: load JSON accepts CDP SameSite casing" {
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena.deinit();
const parsed = try std.json.parseFromSliceLeaky(
[]const JsonCookie,
arena.allocator(),
"[{\"name\":\"sid\",\"value\":\"1\",\"domain\":\"example.com\",\"sameSite\":\"Lax\"}]",
.{ .ignore_unknown_fields = true },
);
const cookie = try parsed[0].toCookie(std.testing.allocator);
defer cookie.deinit();
try std.testing.expectEqual(Cookie.SameSite.lax, cookie.same_site);
}
test "cookies: load JSON keeps a session cookie with expires -1" {
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena.deinit();
// Puppeteer's page.cookies() and CDP getCookies write a session cookie
// with expires: -1.
const parsed = try std.json.parseFromSliceLeaky(
[]const JsonCookie,
arena.allocator(),
"[{\"name\":\"sid\",\"value\":\"1\",\"domain\":\"example.com\",\"expires\":-1,\"session\":true}]",
.{ .ignore_unknown_fields = true },
);
var jar = Cookie.Jar.init(std.testing.allocator, null);
defer jar.deinit();
try jar.add(try parsed[0].toCookie(jar.allocator), lp.datetime.timestamp(.real), true);
try std.testing.expectEqual(1, jar.len);
try std.testing.expectEqual(null, nthCookie(&jar, 0).expires);
}