Lightpanda: the headless browser designed for AI and automation
`cookies`: accept Netscape cookie format when loading from CLI
Halil Durake67301a0fbc8
diff --git a/src/cookies.zig b/src/cookies.zigindex d888af183..05b0eb32e 100644--- a/src/cookies.zig+++ b/src/cookies.zig@@ -43,6 +43,20 @@ fn _loadFromFile(session: *Session, path: []const u8) !void {return;};+ const jar = &session.cookie_jar;++ // The file is either a CDP-style JSON array or the Netscape format. Sniff which one it is.+ const head = std.mem.trimStart(u8, content, &std.ascii.whitespace);+ if (head.len == 0) {+ log.debug(.app, "Cookie.parseFile", .{ .path = path, .note = "empty file" });+ return;+ }+ if (head[0] != '[' and head[0] != '{') {+ const loaded = try NetscapeFormat.parse(jar, content);+ log.info(.app, "Cookie.loadFromFile", .{ .path = path, .count = loaded });+ return;+ }+const json_cookies = std.json.parseFromSliceLeaky([]const JsonCookie, arena.allocator(), content, .{.ignore_unknown_fields = true,}) catch |err| {@@ -50,7 +64,6 @@ fn _loadFromFile(session: *Session, path: []const u8) !void {return;};- const jar = &session.cookie_jar;const now = lp.datetime.timestamp(.real);var loaded: usize = 0;@@ -150,6 +163,253 @@ fn parseJsonSameSite(value: ?[]const u8) Cookie.SameSite {return .none;}+/// Netscape cookie file format parser with `#HttpOnly_` addition from curl.+/// https://docs.cyotek.com/cyowcopy/1.10/netscapecookieformat.html+/// https://curl.se/docs/http-cookies.html+pub const NetscapeFormat = struct {+ /// Parses and loads cookies in Netscape format.+ /// Returns number of cookies loaded.+ pub fn parse(jar: *Cookie.Jar, slice: []const u8) !usize {+ const now = lp.datetime.timestamp(.real);+ var loaded: usize = 0;++ var line_iterator = std.mem.splitScalar(u8, slice, '\n');+ iterate_lines: while (line_iterator.next()) |line| {+ if (line.len == 0) {+ continue :iterate_lines;+ }+ // Remove CR if there's one.+ var s = if (line[line.len - 1] == '\r') line[0 .. line.len - 1] else line;+ // Skip if nothing left.+ if (s.len == 0) {+ continue :iterate_lines;+ }++ // Computed here since this doesn't have it's own column.+ var is_http_only = false;+ if (s[0] == '#') {+ // Is it continued with `HttpOnly_`?+ is_http_only =+ s.len >= 10 and+ @as(u64, @bitCast(s[1..9].*)) == @as(u64, @bitCast(@as([]const u8, "HttpOnly")[0..8].*)) and+ s[9] == '_';+ // Regular comment; skip the line.+ if (!is_http_only) {+ continue :iterate_lines;+ }+ // Advance.+ s = s[10..];+ }++ var fields: struct {+ domain: []const u8 = undefined,+ include_subdomains: bool = false,+ /// null means "/".+ path: ?[]const u8 = null,+ secure: bool = false,+ /// null means a session cookie.+ expires: ?f64 = null,+ name: []const u8 = undefined,+ value: []const u8 = "",+ } = .{};++ // Iterate over columns.+ var column_index: usize = 0;+ var column_iterator = std.mem.splitScalar(u8, s, '\t');+ while (column_iterator.next()) |column| {+ defer column_index += 1;++ switch (column_index) {+ 0 => fields.domain = column,+ 1 => fields.include_subdomains = parseBool(column) catch continue :iterate_lines,+ 2 => {+ // If this is a boolean, we have to set `secure` field here instead.+ const secure = parseBool(column) catch {+ // Not a boolean, set path.+ fields.path = column;+ continue;+ };+ fields.secure = secure;+ // Parsed secure early, we have to advance once more.+ column_index += 1;+ },+ 3 => fields.secure = parseBool(column) catch continue :iterate_lines,+ 4 => {+ // A session cookie is written as `0` by curl, wget and+ // yt-dlp, and as an empty column by Python's+ // http.cookiejar. Both leave `expires` null: `Jar.add`+ // drops anything that expires at or before now, so a+ // literal 0 would be discarded on the spot.+ if (column.len == 0) {+ continue;+ }+ const expires = std.fmt.parseFloat(f64, column) catch continue :iterate_lines;+ if (!std.math.isFinite(expires)) {+ continue :iterate_lines;+ }+ if (expires != 0) {+ fields.expires = expires;+ }+ },+ 5 => fields.name = column,+ 6 => fields.value = column,+ // Indicates we got columns more than we expected.+ else => continue :iterate_lines,+ }+ }++ // We need at least 6 columns filled (value can be empty).+ if (column_index < 6) {+ continue :iterate_lines;+ }++ var cookie_arena = std.heap.ArenaAllocator.init(jar.allocator);+ errdefer cookie_arena.deinit();++ const allocator = cookie_arena.allocator();+ const name = try allocator.dupe(u8, fields.name);+ const value = try allocator.dupe(u8, fields.value);+ const _path = if (fields.path) |path| try allocator.dupe(u8, path) else "/";+ // The domain column's leading dot and the `include_subdomains`+ // column can disagree. curl strips the dot on read and re-derives+ // it from the column, and `Cookie.matchesHost` keys tail-matching+ // off that dot, so the column is what decides it here.+ const bare_domain = if (fields.domain.len > 0 and fields.domain[0] == '.') fields.domain[1..] else fields.domain;+ const domain = if (fields.include_subdomains)+ try std.fmt.allocPrint(allocator, ".{s}", .{bare_domain})+ else+ try allocator.dupe(u8, bare_domain);++ // Bake a cookie.+ const cookie = Cookie{+ .arena = cookie_arena,+ .name = name,+ .value = value,+ .domain = domain,+ .path = _path,+ .expires = fields.expires,+ .secure = fields.secure,+ .http_only = is_http_only,+ .same_site = .none,+ };++ jar.add(cookie, now, true) catch |err| {+ log.warn(.app, "invalid cookie", .{ .name = fields.name, .err = err });+ continue :iterate_lines;+ };+ loaded += 1;+ }++ return loaded;+ }++ fn parseBool(s: []const u8) error{Invalid}!bool {+ if (std.ascii.eqlIgnoreCase(s, "false")) {+ return false;+ }+ if (std.ascii.eqlIgnoreCase(s, "true")) {+ return true;+ }+ return error.Invalid;+ }+};++test "cookies: netscape include_subdomains drives the domain's leading dot" {+ var jar = Cookie.Jar.init(std.testing.allocator, null);+ defer jar.deinit();++ // The domain column and the flag disagree on two of these four lines; the+ // flag wins either way. Expiry is far future so nothing is dropped as stale.+ const content =+ "# Netscape HTTP Cookie File\n" +++ "example.com\tTRUE\t/\tFALSE\t4102444800\ta\t1\n" +++ ".example.com\tTRUE\t/\tFALSE\t4102444800\tb\t2\n" +++ "example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n" +++ ".example.com\tFALSE\t/\tFALSE\t4102444800\td\t4\n";++ try std.testing.expectEqual(@as(usize, 4), try NetscapeFormat.parse(&jar, content));+ try std.testing.expectEqualStrings(".example.com", jar.cookies.items[0].domain);+ try std.testing.expectEqualStrings(".example.com", jar.cookies.items[1].domain);+ try std.testing.expectEqualStrings("example.com", jar.cookies.items[2].domain);+ try std.testing.expectEqualStrings("example.com", jar.cookies.items[3].domain);++ // That dot is what `Cookie.matchesHost` keys tail-matching off.+ try std.testing.expect(jar.cookies.items[0].matchesHost("www.example.com"));+ try std.testing.expect(jar.cookies.items[2].matchesHost("www.example.com") == false);+ try std.testing.expect(jar.cookies.items[2].matchesHost("example.com"));+}++test "cookies: netscape #HttpOnly_ prefix sets http_only" {+ var jar = Cookie.Jar.init(std.testing.allocator, null);+ defer jar.deinit();++ // curl writes the prefix ahead of the domain column and strips it before+ // reading the columns, so it composes with the include_subdomains dot.+ const content =+ "#HttpOnly_example.com\tFALSE\t/\tFALSE\t4102444800\ta\t1\n" +++ "#HttpOnly_.example.com\tTRUE\t/\tFALSE\t4102444800\tb\t2\n" +++ "example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n";++ try std.testing.expectEqual(@as(usize, 3), try NetscapeFormat.parse(&jar, content));++ try std.testing.expect(jar.cookies.items[0].http_only);+ try std.testing.expectEqualStrings("example.com", jar.cookies.items[0].domain);++ try std.testing.expect(jar.cookies.items[1].http_only);+ try std.testing.expectEqualStrings(".example.com", jar.cookies.items[1].domain);++ try std.testing.expect(jar.cookies.items[2].http_only == false);+}++test "cookies: netscape treats #HttpOnly_ near-misses as comments" {+ var jar = Cookie.Jar.init(std.testing.allocator, null);+ defer jar.deinit();++ // Every line here is a comment. curl's check is a case-sensitive+ // `strncmp(lineptr, "#HttpOnly_", 10)`, so none of these are the prefix,+ // and a bare prefix with no columns after it has nothing to load.+ const content =+ "# Netscape HTTP Cookie File\n" +++ "#\n" +++ "# HttpOnly_example.com\tFALSE\t/\tFALSE\t4102444800\ta\t1\n" +++ "#HttpOnlyX example.com\tFALSE\t/\tFALSE\t4102444800\tb\t2\n" +++ "#httponly_example.com\tFALSE\t/\tFALSE\t4102444800\tc\t3\n" +++ "#HttpOnly_\n";++ try std.testing.expectEqual(@as(usize, 0), try NetscapeFormat.parse(&jar, content));+ try std.testing.expectEqual(@as(usize, 0), jar.cookies.items.len);+}++test "cookies: netscape 0 and empty expires load as session cookies" {+ var jar = Cookie.Jar.init(std.testing.allocator, null);+ defer jar.deinit();++ // curl, wget and yt-dlp write `0` for a session cookie and Python's+ // http.cookiejar writes an empty column. Neither may be handed to+ // `Jar.add` as a real timestamp, since it drops anything at or before+ // now. A genuine past timestamp is still dropped, and a non-finite or+ // non-numeric one rejects the line.+ const content =+ "example.com\tFALSE\t/\tFALSE\t0\tsession\t1\n" +++ "example.com\tFALSE\t/\tFALSE\t\tpython\t2\n" +++ "example.com\tFALSE\t/\tFALSE\t1\tstale\t3\n" +++ "example.com\tFALSE\t/\tFALSE\tnan\tnan\t4\n" +++ "example.com\tFALSE\t/\tFALSE\tsoon\tbogus\t5\n" +++ "example.com\tFALSE\t/\tFALSE\t4102444800\tfuture\t6\n";++ _ = try NetscapeFormat.parse(&jar, content);+ try std.testing.expectEqual(@as(usize, 3), jar.cookies.items.len);++ try std.testing.expectEqualStrings("session", jar.cookies.items[0].name);+ try std.testing.expectEqual(@as(?f64, null), jar.cookies.items[0].expires);++ try std.testing.expectEqualStrings("python", jar.cookies.items[1].name);+ try std.testing.expectEqual(@as(?f64, null), jar.cookies.items[1].expires);++ try std.testing.expectEqualStrings("future", jar.cookies.items[2].name);+ try std.testing.expectEqual(@as(?f64, 4102444800), jar.cookies.items[2].expires);+}+test "cookies: load JSON accepts CDP SameSite casing" {var arena = std.heap.ArenaAllocator.init(std.testing.allocator);defer arena.deinit();diff --git a/src/help.zon b/src/help.zonindex c6aba3f2a..950b1c3b9 100644--- a/src/help.zon+++ b/src/help.zon@@ -37,7 +37,10 @@\\ Maximum pending connections in the accept queue.\\ Defaults to 128.\\ --cookie <PATH>- \\ Path to a JSON file to load cookies from (read-only).+ \\ Path to a file to load cookies from (read-only). Either a JSON+ \\ array in the CDP/Puppeteer format, or a Netscape cookie file (the+ \\ cookies.txt format written by curl and wget). The format is+ \\ detected from the file's contents.\\ Defaults to no cookie loading.\\ --disable-metrics\\ Disables the /metrics endpoint (Prometheus text format).@@ -70,7 +73,10 @@\\\\options:\\ --cookie <PATH>- \\ Path to a JSON file to load cookies from (read-only).+ \\ Path to a file to load cookies from (read-only). Either a JSON+ \\ array in the CDP/Puppeteer format, or a Netscape cookie file (the+ \\ cookies.txt format written by curl and wget). The format is+ \\ detected from the file's contents.\\ Defaults to no cookie loading.\\ --cookie-jar <PATH>\\ Path to a JSON file to save cookies to on exit (write-only).@@ -177,7 +183,10 @@\\ combined with --port (they share one network listener).\\ Defaults to disabled.\\ --cookie <PATH>- \\ Path to a JSON file to load cookies from (read-only).+ \\ Path to a file to load cookies from (read-only). Either a JSON+ \\ array in the CDP/Puppeteer format, or a Netscape cookie file (the+ \\ cookies.txt format written by curl and wget). The format is+ \\ detected from the file's contents.\\ Defaults to no cookie loading.\\ --cookie-jar <PATH>\\ Path to a JSON file to save cookies to on exit (write-only).@@ -378,7 +387,10 @@\\ passed multiple times. When any --ca-cert or --ca-path is given,\\ the system trust store is replaced by these certificates.\\ --cookie <PATH>- \\ Path to a JSON file to load cookies from (read-only).+ \\ Path to a file to load cookies from (read-only). Either a JSON+ \\ array in the CDP/Puppeteer format, or a Netscape cookie file (the+ \\ cookies.txt format written by curl and wget). The format is+ \\ detected from the file's contents.\\ Defaults to no cookie loading.\\ --cookie-jar <PATH>\\ Path to a JSON file to save cookies to on exit (write-only).