Lightpanda: the headless browser designed for AI and automation
http: improve caching
Karl Seguin321334205562
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 8365edc51..f16b350e3 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1249,15 +1249,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {req.url;transfer._cache_key = key;- const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);- for (transfer.req_headers.items, req_headers) |hdr, *out| {- out.* = .{ .name = hdr.name, .value = hdr.value };- }-const cache_result = cache.get(arena.allocator(), .{.url = key,.timestamp = lp.datetime.timestamp(.real),- .request_headers = req_headers,+ .request_headers = try transfer.cacheRequestHeaders(),}) catch |e| blk: {log.err(.cache, "failed to get", .{ .url = req.url, .err = e });break :blk .miss;@@ -1327,6 +1322,19 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {return true;}+// Everything the cache needs about this response. Names and values are+// borrowed from the transfer arena, which outlives the put.+fn cachePutRequest(transfer: *Transfer, rh: *http.ResponseHead) !?Cache.CachePutRequest {+ return Cache.tryCache(transfer.arena.allocator(), .{+ .timestamp = lp.datetime.timestamp(.real),+ .url = transfer._cache_key,+ .status = rh.status,+ .content_type = rh.contentType(),+ .headers = transfer.res.headers,+ .request_headers = try transfer.cacheRequestHeaders(),+ });+}+// Store a cacheable response at completion, from the materialized response// headers and the buffered body. Failures are logged, never fatal — the// consumer gets its response either way.@@ -1347,49 +1355,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {// could have been disabled while waiting of the responseconst cache = self.cache.active() orelse return;- const arena = transfer.arena;const rh = &(transfer.res.header orelse return);- const headers = transfer.res.headers;-- const vary = findHeader(headers, "vary");- const maybe_req = Cache.tryCache(- arena.allocator(),- lp.datetime.timestamp(.real),- transfer._cache_key,- rh.status,- rh.contentType(),- findHeader(headers, "cache-control"),- vary,- findHeader(headers, "age"),- findHeader(headers, "etag"),- findHeader(headers, "last-modified"),- findHeader(headers, "set-cookie") != null,- findHeader(headers, "authorization") != null,- ) catch |err| {++ const maybe_req = cachePutRequest(transfer, rh) catch |err| {log.warn(.http, "cache eligibility", .{ .err = err });return;};- var req = maybe_req orelse return;-- var vary_headers: std.ArrayList(http.Header) = .empty;- if (vary) |vary_str| {- for (transfer.req_headers.items) |hdr| {- var vary_iter = std.mem.splitScalar(u8, vary_str, ',');- while (vary_iter.next()) |part| {- const name = std.mem.trim(u8, part, &std.ascii.whitespace);- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {- // name/value already live in transfer.arena- vary_headers.append(arena.allocator(), .{- .name = hdr.name,- .value = hdr.value,- }) catch return;- }- }- }- }-- req.headers = headers;- req.vary_headers = vary_headers.items;+ const req = maybe_req orelse return;if (comptime lp.IS_DEBUG) {log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .put = req });@@ -3646,6 +3618,16 @@ pub const Transfer = struct {return null;}+ // The cache works in plain headers; req_headers carry a `source` alongside+ // them that it has no use for.+ fn cacheRequestHeaders(self: *const Transfer) ![]http.Header {+ const headers = try self.arena.alloc(http.Header, self.req_headers.items.len);+ for (self.req_headers.items, headers) |hdr, *out| {+ out.* = .{ .name = hdr.name, .value = hdr.value };+ }+ return headers;+ }+fn removeHeader(self: *Transfer, name: []const u8) void {var i: usize = 0;while (i < self.req_headers.items.len) {diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex 006642fed..6a21fe4fb 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -131,53 +131,44 @@ pub fn parseDeltaSeconds(value: []const u8) ?u64 {pub const CacheControl = struct {max_age: u64,must_revalidate: bool = false,+};- pub fn parse(value: []const u8) ?CacheControl {- var cc: CacheControl = .{ .max_age = undefined };+pub const ResponseDirectives = struct {+ no_store: bool = false,+ no_cache: bool = false,+ private: bool = false,+ public: bool = false,+ max_age: ?u64 = null,- var max_age_set = false;- var max_s_age_set = false;+ pub fn parse(value: []const u8) ResponseDirectives {+ var directives: ResponseDirectives = .{};var iter = std.mem.splitScalar(u8, value, ',');while (iter.next()) |part| {const directive = std.mem.trim(u8, part, &std.ascii.whitespace);- if (std.ascii.eqlIgnoreCase(directive, "no-store")) {- return null;- }- if (std.ascii.eqlIgnoreCase(directive, "no-cache")) {- if (!max_age_set) {- cc.max_age = 0;- max_age_set = true;- }-- cc.must_revalidate = true;- continue;- }- if (std.ascii.eqlIgnoreCase(directive, "private")) {- return null;- }-- if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) {- if (!max_s_age_set) {- if (parseDeltaSeconds(directive[8..])) |max_age| {- cc.max_age = max_age;- max_age_set = true;- }- }- } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) {- if (parseDeltaSeconds(directive[9..])) |max_age| {- cc.max_age = max_age;- max_age_set = true;- max_s_age_set = true;- }+ // We only care about argument for max-age. For something like+ // `no-cache="set-cookie"` we ignore it and just treat it as "no-cache"+ // which is on the safe side.+ const name, const argument = if (std.mem.indexOfScalar(u8, directive, '=')) |i|+ .{ directive[0..i], directive[i + 1 ..] }+ else+ .{ directive, "" };++ if (std.ascii.eqlIgnoreCase(name, "no-store")) {+ directives.no_store = true;+ } else if (std.ascii.eqlIgnoreCase(name, "no-cache")) {+ directives.no_cache = true;+ } else if (std.ascii.eqlIgnoreCase(name, "private")) {+ directives.private = true;+ } else if (std.ascii.eqlIgnoreCase(name, "public")) {+ directives.public = true;+ } else if (std.ascii.eqlIgnoreCase(name, "max-age")) {+ directives.max_age = parseDeltaSeconds(argument);}}- if (!max_age_set) return null;- if (cc.max_age == 0 and !cc.must_revalidate) return null;-- return cc;+ return directives;}};@@ -232,6 +223,33 @@ pub const RenewResponse = struct {url: []const u8,timestamp: u64,headers: []const Http.Header,++ // What a revalidation response tells us about the cache entry (a null+ // value means: we were told nothing, keep what we have)+ pub const Directive = struct {+ timestamp: u64,+ age_at_store: u64,+ max_age: ?u64,+ must_revalidate: ?bool,+ content_type: ?[]const u8,+ etag: ?[]const u8,+ last_modified: ?[]const u8,+ };++ pub fn directive(self: RenewResponse) Directive {+ const response: ResponseHeaders = .parse(self.headers);+ const cache_control = explicitFreshness(self.timestamp, response.directives, response.expires, response.date);++ return .{+ .timestamp = self.timestamp,+ .age_at_store = if (response.age) |a| parseDeltaSeconds(a) orelse 0 else 0,+ .max_age = if (cache_control) |cc| cc.max_age else null,+ .must_revalidate = if (cache_control) |cc| cc.must_revalidate else null,+ .content_type = response.content_type,+ .etag = response.etag,+ .last_modified = response.last_modified,+ };+ }};const CachedData = union(enum) {@@ -281,197 +299,419 @@ pub const CachedResponse = struct {}};-pub fn tryCache(- arena: std.mem.Allocator,+// Cache-related headers (for storing a new one and/or renewing)+const ResponseHeaders = struct {+ directives: ResponseDirectives = .{},+ content_type: ?[]const u8 = null,+ date: ?[]const u8 = null,+ expires: ?[]const u8 = null,+ vary: ?[]const u8 = null,+ age: ?[]const u8 = null,+ etag: ?[]const u8 = null,+ last_modified: ?[]const u8 = null,+ has_set_cookie: bool = false,+ has_authorization: bool = false,++ fn parse(headers: []const Http.Header) ResponseHeaders {+ var self: ResponseHeaders = .{};++ for (headers) |h| {+ switch (h.name.len) {+ 3 => if (std.ascii.eqlIgnoreCase(h.name, "Age")) {+ self.age = h.value;+ },+ 4 => {+ if (std.ascii.eqlIgnoreCase(h.name, "Date")) {+ self.date = h.value;+ } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {+ self.etag = h.value;+ } else if (std.ascii.eqlIgnoreCase(h.name, "Vary")) {+ self.vary = h.value;+ }+ },+ 7 => if (std.ascii.eqlIgnoreCase(h.name, "Expires")) {+ self.expires = h.value;+ },+ 10 => if (std.ascii.eqlIgnoreCase(h.name, "Set-Cookie")) {+ self.has_set_cookie = true;+ },+ 12 => if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {+ self.content_type = h.value;+ },+ 13 => {+ if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {+ self.directives = .parse(h.value);+ } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {+ self.last_modified = h.value;+ } else if (std.ascii.eqlIgnoreCase(h.name, "Authorization")) {+ self.has_authorization = true;+ }+ },+ else => {},+ }+ }++ return self;+ }+};++const CacheCandidate = struct {timestamp: u64,url: [:0]const u8,status: u16,- content_type: ?[]const u8,- cache_control: ?[]const u8,- vary: ?[]const u8,- age: ?[]const u8,- etag: ?[]const u8,- last_modified: ?[]const u8,- has_set_cookie: bool,- has_authorization: bool,-) !?CachePutRequest {- if (status != 200) {+ content_type: ?[]const u8, // from ResponseHead, can be truncated+ headers: []const Http.Header,+ request_headers: []const Http.Header, // needed for vary headers+};++fn explicitFreshness(timestamp: u64, directives: ResponseDirectives, expires_: ?[]const u8, date: ?[]const u8) ?CacheControl {+ if (directives.no_cache) {+ // Storable, but every use has to revalidate first.+ return .{ .max_age = directives.max_age orelse 0, .must_revalidate = true };+ }++ if (directives.max_age) |max_age| {+ return .{ .max_age = max_age };+ }++ const expires = expires_ orelse return null;++ return .{+ .max_age = blk: {+ // an unparsable value is considered expired+ const expires_at = parseHttpDate(expires) orelse break :blk 0;+ const sent_at = if (date) |d| parseHttpDate(d) orelse @as(i64, @intCast(timestamp)) else @as(i64, @intCast(timestamp));+ if (expires_at <= sent_at) {+ break :blk 0;+ }+ break :blk @min(@as(u64, @intCast(expires_at - sent_at)), max_delta_seconds);+ },+ };+}++pub fn tryCache(arena: std.mem.Allocator, candidate: CacheCandidate) !?CachePutRequest {+ const url = candidate.url;+ const status = candidate.status;+ const response: ResponseHeaders = .parse(candidate.headers);++ if (status == 206 or (status >= 300 and status < 400)) {+ // TODO+ // Could be cached, but HttpClient doesn't handle this correctly, it+ // just writes the response instead of actually processing it.log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" });return null;}- if (has_set_cookie) {++ if (response.has_set_cookie) {log.debug(.cache, "no store", .{ .url = url, .reason = "has_cookies" });return null;}- if (has_authorization) {++ if (response.has_authorization) {log.debug(.cache, "no store", .{ .url = url, .reason = "has_authorization" });return null;}- if (vary) |v| if (std.mem.eql(u8, v, "*")) {++ if (response.vary) |v| if (std.mem.eql(u8, v, "*")) {log.debug(.cache, "no store", .{ .url = url, .vary = v, .reason = "vary" });return null;};- const cc = blk: {- if (cache_control) |c| {- if (CacheControl.parse(c)) |cc| {- break :blk cc;+ const cc: CacheControl = blk: {+ const directives = response.directives;+ // "private" bars shared cache, which is how our caching works - tied+ // to the app while the cookie jar is on the Session. Safer not to cache.+ if (directives.no_store or directives.private) {+ break :blk null;+ }+ const timestamp = candidate.timestamp;+ if (explicitFreshness(timestamp, directives, response.expires, response.date)) |cc| {+ if (cc.max_age == 0 and cc.must_revalidate == false) {+ // Already stale on arrival, with nothing to revalidate it against.+ break :blk null;}- } else if (last_modified != null) {- // Requires Last-Modified to be present to heuristically cache.- log.debug(.cache, "heuristic cache", .{ .url = url, .max_age = 86400 });- break :blk CacheControl{ .max_age = 86400, .must_revalidate = false };+ break :blk cc;}+ if (heuristicallyCacheable(status) == false and directives.public == false) {+ // Need a cacheable status and a public cache to use heuristic caching+ break :blk null;+ }+ // also need a last_modified+ const last_modified = response.last_modified orelse break :blk null;+ const lifetime = heuristicLifetime(last_modified, timestamp);+ break :blk if (lifetime == 0) null else CacheControl{ .max_age = lifetime };+ } orelse {log.debug(.cache, "no store", .{.url = url,- .cache_control = cache_control orelse "null",- .last_modified = last_modified orelse "null",+ .reason = "not fresh",+ .expires = response.expires orelse "null",+ .last_modified = response.last_modified orelse "null",});return null;};// get() treats must_revalidate as always-expired, so without validators// the entry could never be served, only purged.- if (cc.must_revalidate and etag == null and last_modified == null) {+ if (cc.must_revalidate and response.etag == null and response.last_modified == null) {log.debug(.cache, "no store", .{ .url = url, .reason = "must_revalidate without validators" });return null;}return .{.url = try arena.dupeZ(u8, url),- .content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",+ .content_type = if (candidate.content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",.status = status,- .stored_at = timestamp,- .age_at_store = if (age) |a| parseDeltaSeconds(a) orelse 0 else 0,+ .stored_at = candidate.timestamp,+ .age_at_store = if (response.age) |a| parseDeltaSeconds(a) orelse 0 else 0,.cache_control = cc,- .headers = &.{},- .vary_headers = &.{},- .etag = if (etag) |e| try arena.dupe(u8, e) else null,- .last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null,+ .headers = candidate.headers,+ .vary_headers = try varyHeaders(arena, response.vary, candidate.request_headers),+ .etag = if (response.etag) |e| try arena.dupe(u8, e) else null,+ .last_modified = if (response.last_modified) |lm| try arena.dupe(u8, lm) else null,+ };+}++fn heuristicallyCacheable(status: u16) bool {+ return switch (status) {+ // tryCache discards 206 and redirects, so skip them early here+ 200, 203, 204, 404, 405, 410, 414, 501 => true,+ else => false,};}-const testing = @import("../../testing.zig");-test "Cache: CacheControl.parse" {- try testing.expectEqual(300, CacheControl.parse("max-age=300").?.max_age);+/// RFC 9111 §4.2.2 suggests 10% of the interval since Last-Modified. The cap+/// keeps a document that has not changed in years from being held forever.+fn heuristicLifetime(last_modified: []const u8, now: u64) u64 {+ const modified_at = parseHttpDate(last_modified) orelse return 0;+ const now_seconds: i64 = @intCast(now);+ if (modified_at >= now_seconds) {+ return 0;+ }+ const heuristic_max_age: u64 = 86400;+ return @min(@as(u64, @intCast(now_seconds - modified_at)) / 10, heuristic_max_age);+}- try testing.expectEqual(300, CacheControl.parse("Max-Age=300").?.max_age);- try testing.expectEqual(300, CacheControl.parse("MAX-AGE=300").?.max_age);+fn parseHttpDate(value: []const u8) ?i64 {+ const date_time = lp.datetime.DateTime.parse(value, .rfc822) catch return null;+ return date_time.unix(.seconds);+}- try testing.expectEqual(300, CacheControl.parse("public, max-age=300").?.max_age);- try testing.expectEqual(300, CacheControl.parse(" max-age=300 ").?.max_age);+/// The request headers the Vary list names, which a later lookup has to match+/// against. Names and values stay borrowed from the caller's headers.+fn varyHeaders(arena: std.mem.Allocator, vary: ?[]const u8, request_headers: []const Http.Header) ![]const Http.Header {+ const vary_value = vary orelse return &.{};- try testing.expectEqual(600, CacheControl.parse("max-age=300, s-maxage=600").?.max_age);- try testing.expectEqual(600, CacheControl.parse("s-maxage=600, max-age=300").?.max_age);+ var headers: std.ArrayList(Http.Header) = .empty;+ for (request_headers) |hdr| {+ var iter = std.mem.splitScalar(u8, vary_value, ',');+ while (iter.next()) |part| {+ const name = std.mem.trim(u8, part, &std.ascii.whitespace);+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {+ try headers.append(arena, hdr);+ }+ }+ }+ return headers.items;+}++const testing = @import("../../testing.zig");+test "Cache: ResponseDirectives.parse" {+ try testing.expectEqual(300, ResponseDirectives.parse("max-age=300").max_age);+ try testing.expectEqual(300, ResponseDirectives.parse("Max-Age=300").max_age);+ try testing.expectEqual(300, ResponseDirectives.parse("MAX-AGE=300").max_age);+ try testing.expectEqual(300, ResponseDirectives.parse("public, max-age=300").max_age);+ try testing.expectEqual(300, ResponseDirectives.parse(" max-age=300 ").max_age);++ // we are a private cache: s-maxage is for shared caches only+ try testing.expectEqual(300, ResponseDirectives.parse("max-age=300, s-maxage=600").max_age);+ try testing.expectEqual(null, ResponseDirectives.parse("s-maxage=600").max_age);++ try testing.expectEqual(true, ResponseDirectives.parse("no-store").no_store);+ try testing.expectEqual(true, ResponseDirectives.parse("max-age=300, no-store").no_store);+ try testing.expectEqual(true, ResponseDirectives.parse("no-cache").no_cache);+ try testing.expectEqual(true, ResponseDirectives.parse("no-cache=\"set-cookie\"").no_cache);+ try testing.expectEqual(true, ResponseDirectives.parse("public").public);- try testing.expectEqual(null, CacheControl.parse("no-store"));- try testing.expectEqual(- CacheControl{ .max_age = 0, .must_revalidate = true },- CacheControl.parse("no-cache"),- );- try testing.expectEqual(null, CacheControl.parse("private"));- try testing.expectEqual(null, CacheControl.parse("max-age=300, no-store"));try testing.expectEqual(- CacheControl{ .max_age = 300, .must_revalidate = true },- CacheControl.parse("no-cache, max-age=300"),+ ResponseDirectives{ .private = true, .max_age = 300 },+ ResponseDirectives.parse("Private, max-age=300"),);- try testing.expectEqual(null, CacheControl.parse("Private, max-age=300"));-- try testing.expectEqual(null, CacheControl.parse("max-age=0"));-- try testing.expectEqual(null, CacheControl.parse("public"));- try testing.expectEqual(null, CacheControl.parse(""));- try testing.expectEqual(null, CacheControl.parse("max-age=abc"));- try testing.expectEqual(null, CacheControl.parse("max-age="));+ try testing.expectEqual(ResponseDirectives{}, ResponseDirectives.parse(""));+ try testing.expectEqual(null, ResponseDirectives.parse("max-age=abc").max_age);+ try testing.expectEqual(null, ResponseDirectives.parse("max-age=").max_age);// values longer than 8 digits must not be truncated- try testing.expectEqual(315360000, CacheControl.parse("max-age=315360000").?.max_age);+ try testing.expectEqual(315360000, ResponseDirectives.parse("max-age=315360000").max_age);// delta-seconds too large to represent are capped at 2^31 (RFC 9111 §1.2.2)- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=2147483649").?.max_age);- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=9999999999999999999").?.max_age);- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=99999999999999999999999").?.max_age);- try testing.expectEqual(max_delta_seconds, CacheControl.parse("s-maxage=9999999999999999999").?.max_age);+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=2147483649").max_age);+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=9999999999999999999").max_age);+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=99999999999999999999999").max_age);}-test "Cache: tryCache heuristic when no cache-control" {+// 2026-09-21T07:00:00Z, so that the HTTP-dates below land on either side of it.+const test_now: u64 = 1789974000;++/// Spells a response out as the header list `tryCache` reads it from.+const TestResponse = struct {+ status: u16 = 200,+ cache_control: ?[]const u8 = null,+ expires: ?[]const u8 = null,+ date: ?[]const u8 = null,+ etag: ?[]const u8 = null,+ last_modified: ?[]const u8 = null,++ fn run(self: TestResponse, arena: std.mem.Allocator) !?CachePutRequest {+ var headers: std.ArrayList(Http.Header) = .empty;+ inline for (.{+ .{ "Cache-Control", self.cache_control },+ .{ "Expires", self.expires },+ .{ "Date", self.date },+ .{ "ETag", self.etag },+ .{ "Last-Modified", self.last_modified },+ }) |field| {+ if (field[1]) |value| {+ try headers.append(arena, .{ .name = field[0], .value = value });+ }+ }++ return tryCache(arena, .{+ .timestamp = test_now,+ .url = "https://example.com",+ .status = self.status,+ .content_type = "text/html",+ .headers = headers.items,+ .request_headers = &.{},+ });+ }+};++test "Cache: tryCache freshness" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();- const result = try tryCache(- arena.allocator(),- 1000,- "https://example.com",- 200,- "text/html",- null,- null,- null,- null,- null,- false,- false,- );- try testing.expectEqual(null, result);+ const cases = [_]struct { max_age: ?u64, response: TestResponse }{+ // nothing to go on+ .{ .max_age = null, .response = .{} },++ .{ .max_age = 300, .response = .{ .cache_control = "max-age=300" } },+ .{ .max_age = null, .response = .{ .cache_control = "max-age=0" } },+ .{ .max_age = null, .response = .{ .cache_control = "no-store, max-age=300" } },++ // Expires is relative to Date when the response carries one+ .{ .max_age = 3600, .response = .{+ .expires = "Mon, 21 Sep 2026 08:00:00 GMT",+ .date = "Mon, 21 Sep 2026 07:00:00 GMT",+ } },+ .{ .max_age = 3600, .response = .{ .expires = "Mon, 21 Sep 2026 08:00:00 GMT" } },++ // a past Expires, and an unparsable one, are both already expired+ .{ .max_age = null, .response = .{ .expires = "Mon, 21 Sep 2026 06:00:00 GMT" } },+ .{ .max_age = null, .response = .{ .expires = "0" } },++ // a broken Expires must not fall back to the Last-Modified heuristic+ .{ .max_age = null, .response = .{+ .expires = "0",+ .last_modified = "Mon, 21 Sep 2026 04:13:20 GMT",+ } },++ // Cache-Control wins over Expires+ .{ .max_age = 300, .response = .{+ .cache_control = "max-age=300",+ .expires = "Mon, 21 Sep 2026 06:00:00 GMT",+ } },+ };++ for (cases, 0..) |c, i| {+ const result = try c.response.run(arena.allocator());+ testing.expectEqual(c.max_age, if (result) |r| r.cache_control.max_age else null) catch |err| {+ testing.print("case {d}\n", .{i});+ return err;+ };+ }+}++test "Cache: tryCache heuristic freshness" {+ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ // 10% of the 10000 seconds since it was last modified+ const modified = "Mon, 21 Sep 2026 04:13:20 GMT";++ const result = try (TestResponse{ .last_modified = modified }).run(arena.allocator());+ try testing.expectEqual(1000, result.?.cache_control.max_age);+ try testing.expectEqual(false, result.?.cache_control.must_revalidate);++ // a status with no heuristic of its own, unless the origin said `public`+ const unknown_status = try (TestResponse{ .status = 299, .last_modified = modified }).run(arena.allocator());+ try testing.expectEqual(null, unknown_status);++ const public_unknown_status = try (TestResponse{+ .status = 299,+ .cache_control = "public",+ .last_modified = modified,+ }).run(arena.allocator());+ try testing.expectEqual(1000, public_unknown_status.?.cache_control.max_age);++ // never stored, however fresh they claim to be+ for ([_]u16{ 206, 301, 304 }) |status| {+ const result_ = try (TestResponse{ .status = status, .cache_control = "max-age=300" }).run(arena.allocator());+ try testing.expectEqual(null, result_);+ }}test "Cache: tryCache must_revalidate without validators" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();- const no_validators = try tryCache(- arena.allocator(),- 1000,- "https://example.com",- 200,- "text/html",- "no-cache, max-age=300",- null,- null,- null,- null,- false,- false,- );+ const no_validators = try (TestResponse{ .cache_control = "no-cache, max-age=300" }).run(arena.allocator());try testing.expectEqual(null, no_validators);- const with_etag = try tryCache(- arena.allocator(),- 1000,- "https://example.com",- 200,- "text/html",- "no-cache, max-age=300",- null,- null,- "\"abc\"",- null,- false,- false,- );+ const with_etag = try (TestResponse{+ .cache_control = "no-cache, max-age=300",+ .etag = "\"abc\"",+ }).run(arena.allocator());try testing.expectEqual(true, with_etag.?.cache_control.must_revalidate);}-test "Cache: tryCache heuristic when no cache-control with last-modified" {+test "Cache: tryCache vary headers" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();- const result = try tryCache(- arena.allocator(),- 1000,- "https://example.com",- 200,- "text/html",- null,- null,- null,- null,- "Wed, 21 Oct 2015 07:28:00 GMT",- false,- false,- );- try testing.expectEqual(@as(u64, 86400), result.?.cache_control.max_age);- try testing.expectEqual(false, result.?.cache_control.must_revalidate);+ const request_headers = [_]Http.Header{+ .{ .name = "Accept-Encoding", .value = "gzip" },+ .{ .name = "User-Agent", .value = "lightpanda" },+ };++ const stored = try tryCache(arena.allocator(), .{+ .timestamp = test_now,+ .url = "https://example.com",+ .status = 200,+ .content_type = "text/html",+ .headers = &.{+ .{ .name = "Cache-Control", .value = "max-age=300" },+ .{ .name = "Vary", .value = "accept-encoding, accept-language" },+ },+ .request_headers = &request_headers,+ });++ try testing.expectEqual(1, stored.?.vary_headers.len);+ try testing.expectString("Accept-Encoding", stored.?.vary_headers[0].name);+ try testing.expectString("gzip", stored.?.vary_headers[0].value);++ // Vary: * is never stored+ const wildcard = try tryCache(arena.allocator(), .{+ .timestamp = test_now,+ .url = "https://example.com",+ .status = 200,+ .content_type = "text/html",+ .headers = &.{+ .{ .name = "Cache-Control", .value = "max-age=300" },+ .{ .name = "Vary", .value = "*" },+ },+ .request_headers = &request_headers,+ });+ try testing.expectEqual(null, wildcard);}diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zigindex c944fa21a..d33e3eb3c 100644--- a/src/network/cache/SqliteCache.zig+++ b/src/network/cache/SqliteCache.zig@@ -27,8 +27,6 @@ const RenewResponse = Cache.RenewResponse;const CachePutRequest = Cache.CachePutRequest;const CacheGetResult = Cache.CacheGetResult;const CachedResponse = Cache.CachedResponse;-const CacheControl = Cache.CacheControl;-const parseDeltaSeconds = Cache.parseDeltaSeconds;const Http = @import("../http.zig");const Blob = @import("../../storage/sqlite/Sqlite.zig").Blob;@@ -351,25 +349,7 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !voidtry conn.begin(.immediate);defer conn.rollback() catch {};- var age_at_store: u64 = 0;- var content_type: ?[]const u8 = null;- var etag: ?[]const u8 = null;- var last_modified: ?[]const u8 = null;- var cache_control: ?CacheControl = null;-- for (req.headers) |h| {- if (std.ascii.eqlIgnoreCase(h.name, "Age")) {- age_at_store = parseDeltaSeconds(h.value) orelse 0;- } else if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {- cache_control = CacheControl.parse(h.value) orelse continue;- } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {- etag = h.value;- } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {- last_modified = h.value;- } else if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {- content_type = h.value;- }- }+ const directive = req.directive();try conn.exec(\\ update cache@@ -382,13 +362,13 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void\\ last_modified = coalesce($7, last_modified)\\ where url = $8, .{- req.timestamp,- age_at_store,- if (cache_control) |cc| cc.max_age else null,- if (cache_control) |cc| cc.must_revalidate else null,- content_type,- etag,- last_modified,+ directive.timestamp,+ directive.age_at_store,+ directive.max_age,+ directive.must_revalidate,+ directive.content_type,+ directive.etag,+ directive.last_modified,req.url,});