Skip to content

Lightpanda: the headless browser designed for AI and automation

http: improve caching

Karl Seguin321334205562
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 8365edc51..f16b350e3 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1249,15 +1249,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
req.url;
transfer._cache_key = key;
- const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);
- for (transfer.req_headers.items, req_headers) |hdr, *out| {
- out.* = .{ .name = hdr.name, .value = hdr.value };
- }
-
const cache_result = cache.get(arena.allocator(), .{
.url = key,
.timestamp = lp.datetime.timestamp(.real),
- .request_headers = req_headers,
+ .request_headers = try transfer.cacheRequestHeaders(),
}) catch |e| blk: {
log.err(.cache, "failed to get", .{ .url = req.url, .err = e });
break :blk .miss;
@@ -1327,6 +1322,19 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {
return true;
}
+// Everything the cache needs about this response. Names and values are
+// borrowed from the transfer arena, which outlives the put.
+fn cachePutRequest(transfer: *Transfer, rh: *http.ResponseHead) !?Cache.CachePutRequest {
+ return Cache.tryCache(transfer.arena.allocator(), .{
+ .timestamp = lp.datetime.timestamp(.real),
+ .url = transfer._cache_key,
+ .status = rh.status,
+ .content_type = rh.contentType(),
+ .headers = transfer.res.headers,
+ .request_headers = try transfer.cacheRequestHeaders(),
+ });
+}
+
// Store a cacheable response at completion, from the materialized response
// headers and the buffered body. Failures are logged, never fatal — the
// consumer gets its response either way.
@@ -1347,49 +1355,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
// could have been disabled while waiting of the response
const cache = self.cache.active() orelse return;
- const arena = transfer.arena;
const rh = &(transfer.res.header orelse return);
- const headers = transfer.res.headers;
-
- const vary = findHeader(headers, "vary");
- const maybe_req = Cache.tryCache(
- arena.allocator(),
- lp.datetime.timestamp(.real),
- transfer._cache_key,
- rh.status,
- rh.contentType(),
- findHeader(headers, "cache-control"),
- vary,
- findHeader(headers, "age"),
- findHeader(headers, "etag"),
- findHeader(headers, "last-modified"),
- findHeader(headers, "set-cookie") != null,
- findHeader(headers, "authorization") != null,
- ) catch |err| {
+
+ const maybe_req = cachePutRequest(transfer, rh) catch |err| {
log.warn(.http, "cache eligibility", .{ .err = err });
return;
};
- var req = maybe_req orelse return;
-
- var vary_headers: std.ArrayList(http.Header) = .empty;
- if (vary) |vary_str| {
- for (transfer.req_headers.items) |hdr| {
- var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
- while (vary_iter.next()) |part| {
- const name = std.mem.trim(u8, part, &std.ascii.whitespace);
- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
- // name/value already live in transfer.arena
- vary_headers.append(arena.allocator(), .{
- .name = hdr.name,
- .value = hdr.value,
- }) catch return;
- }
- }
- }
- }
-
- req.headers = headers;
- req.vary_headers = vary_headers.items;
+ const req = maybe_req orelse return;
if (comptime lp.IS_DEBUG) {
log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .put = req });
@@ -3646,6 +3618,16 @@ pub const Transfer = struct {
return null;
}
+ // The cache works in plain headers; req_headers carry a `source` alongside
+ // them that it has no use for.
+ fn cacheRequestHeaders(self: *const Transfer) ![]http.Header {
+ const headers = try self.arena.alloc(http.Header, self.req_headers.items.len);
+ for (self.req_headers.items, headers) |hdr, *out| {
+ out.* = .{ .name = hdr.name, .value = hdr.value };
+ }
+ return headers;
+ }
+
fn removeHeader(self: *Transfer, name: []const u8) void {
var i: usize = 0;
while (i < self.req_headers.items.len) {
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index 006642fed..6a21fe4fb 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -131,53 +131,44 @@ pub fn parseDeltaSeconds(value: []const u8) ?u64 {
pub const CacheControl = struct {
max_age: u64,
must_revalidate: bool = false,
+};
- pub fn parse(value: []const u8) ?CacheControl {
- var cc: CacheControl = .{ .max_age = undefined };
+pub const ResponseDirectives = struct {
+ no_store: bool = false,
+ no_cache: bool = false,
+ private: bool = false,
+ public: bool = false,
+ max_age: ?u64 = null,
- var max_age_set = false;
- var max_s_age_set = false;
+ pub fn parse(value: []const u8) ResponseDirectives {
+ var directives: ResponseDirectives = .{};
var iter = std.mem.splitScalar(u8, value, ',');
while (iter.next()) |part| {
const directive = std.mem.trim(u8, part, &std.ascii.whitespace);
- if (std.ascii.eqlIgnoreCase(directive, "no-store")) {
- return null;
- }
- if (std.ascii.eqlIgnoreCase(directive, "no-cache")) {
- if (!max_age_set) {
- cc.max_age = 0;
- max_age_set = true;
- }
-
- cc.must_revalidate = true;
- continue;
- }
- if (std.ascii.eqlIgnoreCase(directive, "private")) {
- return null;
- }
-
- if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) {
- if (!max_s_age_set) {
- if (parseDeltaSeconds(directive[8..])) |max_age| {
- cc.max_age = max_age;
- max_age_set = true;
- }
- }
- } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) {
- if (parseDeltaSeconds(directive[9..])) |max_age| {
- cc.max_age = max_age;
- max_age_set = true;
- max_s_age_set = true;
- }
+ // We only care about argument for max-age. For something like
+ // `no-cache="set-cookie"` we ignore it and just treat it as "no-cache"
+ // which is on the safe side.
+ const name, const argument = if (std.mem.indexOfScalar(u8, directive, '=')) |i|
+ .{ directive[0..i], directive[i + 1 ..] }
+ else
+ .{ directive, "" };
+
+ if (std.ascii.eqlIgnoreCase(name, "no-store")) {
+ directives.no_store = true;
+ } else if (std.ascii.eqlIgnoreCase(name, "no-cache")) {
+ directives.no_cache = true;
+ } else if (std.ascii.eqlIgnoreCase(name, "private")) {
+ directives.private = true;
+ } else if (std.ascii.eqlIgnoreCase(name, "public")) {
+ directives.public = true;
+ } else if (std.ascii.eqlIgnoreCase(name, "max-age")) {
+ directives.max_age = parseDeltaSeconds(argument);
}
}
- if (!max_age_set) return null;
- if (cc.max_age == 0 and !cc.must_revalidate) return null;
-
- return cc;
+ return directives;
}
};
@@ -232,6 +223,33 @@ pub const RenewResponse = struct {
url: []const u8,
timestamp: u64,
headers: []const Http.Header,
+
+ // What a revalidation response tells us about the cache entry (a null
+ // value means: we were told nothing, keep what we have)
+ pub const Directive = struct {
+ timestamp: u64,
+ age_at_store: u64,
+ max_age: ?u64,
+ must_revalidate: ?bool,
+ content_type: ?[]const u8,
+ etag: ?[]const u8,
+ last_modified: ?[]const u8,
+ };
+
+ pub fn directive(self: RenewResponse) Directive {
+ const response: ResponseHeaders = .parse(self.headers);
+ const cache_control = explicitFreshness(self.timestamp, response.directives, response.expires, response.date);
+
+ return .{
+ .timestamp = self.timestamp,
+ .age_at_store = if (response.age) |a| parseDeltaSeconds(a) orelse 0 else 0,
+ .max_age = if (cache_control) |cc| cc.max_age else null,
+ .must_revalidate = if (cache_control) |cc| cc.must_revalidate else null,
+ .content_type = response.content_type,
+ .etag = response.etag,
+ .last_modified = response.last_modified,
+ };
+ }
};
const CachedData = union(enum) {
@@ -281,197 +299,419 @@ pub const CachedResponse = struct {
}
};
-pub fn tryCache(
- arena: std.mem.Allocator,
+// Cache-related headers (for storing a new one and/or renewing)
+const ResponseHeaders = struct {
+ directives: ResponseDirectives = .{},
+ content_type: ?[]const u8 = null,
+ date: ?[]const u8 = null,
+ expires: ?[]const u8 = null,
+ vary: ?[]const u8 = null,
+ age: ?[]const u8 = null,
+ etag: ?[]const u8 = null,
+ last_modified: ?[]const u8 = null,
+ has_set_cookie: bool = false,
+ has_authorization: bool = false,
+
+ fn parse(headers: []const Http.Header) ResponseHeaders {
+ var self: ResponseHeaders = .{};
+
+ for (headers) |h| {
+ switch (h.name.len) {
+ 3 => if (std.ascii.eqlIgnoreCase(h.name, "Age")) {
+ self.age = h.value;
+ },
+ 4 => {
+ if (std.ascii.eqlIgnoreCase(h.name, "Date")) {
+ self.date = h.value;
+ } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {
+ self.etag = h.value;
+ } else if (std.ascii.eqlIgnoreCase(h.name, "Vary")) {
+ self.vary = h.value;
+ }
+ },
+ 7 => if (std.ascii.eqlIgnoreCase(h.name, "Expires")) {
+ self.expires = h.value;
+ },
+ 10 => if (std.ascii.eqlIgnoreCase(h.name, "Set-Cookie")) {
+ self.has_set_cookie = true;
+ },
+ 12 => if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
+ self.content_type = h.value;
+ },
+ 13 => {
+ if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {
+ self.directives = .parse(h.value);
+ } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {
+ self.last_modified = h.value;
+ } else if (std.ascii.eqlIgnoreCase(h.name, "Authorization")) {
+ self.has_authorization = true;
+ }
+ },
+ else => {},
+ }
+ }
+
+ return self;
+ }
+};
+
+const CacheCandidate = struct {
timestamp: u64,
url: [:0]const u8,
status: u16,
- content_type: ?[]const u8,
- cache_control: ?[]const u8,
- vary: ?[]const u8,
- age: ?[]const u8,
- etag: ?[]const u8,
- last_modified: ?[]const u8,
- has_set_cookie: bool,
- has_authorization: bool,
-) !?CachePutRequest {
- if (status != 200) {
+ content_type: ?[]const u8, // from ResponseHead, can be truncated
+ headers: []const Http.Header,
+ request_headers: []const Http.Header, // needed for vary headers
+};
+
+fn explicitFreshness(timestamp: u64, directives: ResponseDirectives, expires_: ?[]const u8, date: ?[]const u8) ?CacheControl {
+ if (directives.no_cache) {
+ // Storable, but every use has to revalidate first.
+ return .{ .max_age = directives.max_age orelse 0, .must_revalidate = true };
+ }
+
+ if (directives.max_age) |max_age| {
+ return .{ .max_age = max_age };
+ }
+
+ const expires = expires_ orelse return null;
+
+ return .{
+ .max_age = blk: {
+ // an unparsable value is considered expired
+ const expires_at = parseHttpDate(expires) orelse break :blk 0;
+ const sent_at = if (date) |d| parseHttpDate(d) orelse @as(i64, @intCast(timestamp)) else @as(i64, @intCast(timestamp));
+ if (expires_at <= sent_at) {
+ break :blk 0;
+ }
+ break :blk @min(@as(u64, @intCast(expires_at - sent_at)), max_delta_seconds);
+ },
+ };
+}
+
+pub fn tryCache(arena: std.mem.Allocator, candidate: CacheCandidate) !?CachePutRequest {
+ const url = candidate.url;
+ const status = candidate.status;
+ const response: ResponseHeaders = .parse(candidate.headers);
+
+ if (status == 206 or (status >= 300 and status < 400)) {
+ // TODO
+ // Could be cached, but HttpClient doesn't handle this correctly, it
+ // just writes the response instead of actually processing it.
log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" });
return null;
}
- if (has_set_cookie) {
+
+ if (response.has_set_cookie) {
log.debug(.cache, "no store", .{ .url = url, .reason = "has_cookies" });
return null;
}
- if (has_authorization) {
+
+ if (response.has_authorization) {
log.debug(.cache, "no store", .{ .url = url, .reason = "has_authorization" });
return null;
}
- if (vary) |v| if (std.mem.eql(u8, v, "*")) {
+
+ if (response.vary) |v| if (std.mem.eql(u8, v, "*")) {
log.debug(.cache, "no store", .{ .url = url, .vary = v, .reason = "vary" });
return null;
};
- const cc = blk: {
- if (cache_control) |c| {
- if (CacheControl.parse(c)) |cc| {
- break :blk cc;
+ const cc: CacheControl = blk: {
+ const directives = response.directives;
+ // "private" bars shared cache, which is how our caching works - tied
+ // to the app while the cookie jar is on the Session. Safer not to cache.
+ if (directives.no_store or directives.private) {
+ break :blk null;
+ }
+ const timestamp = candidate.timestamp;
+ if (explicitFreshness(timestamp, directives, response.expires, response.date)) |cc| {
+ if (cc.max_age == 0 and cc.must_revalidate == false) {
+ // Already stale on arrival, with nothing to revalidate it against.
+ break :blk null;
}
- } else if (last_modified != null) {
- // Requires Last-Modified to be present to heuristically cache.
- log.debug(.cache, "heuristic cache", .{ .url = url, .max_age = 86400 });
- break :blk CacheControl{ .max_age = 86400, .must_revalidate = false };
+ break :blk cc;
}
+ if (heuristicallyCacheable(status) == false and directives.public == false) {
+ // Need a cacheable status and a public cache to use heuristic caching
+ break :blk null;
+ }
+ // also need a last_modified
+ const last_modified = response.last_modified orelse break :blk null;
+ const lifetime = heuristicLifetime(last_modified, timestamp);
+ break :blk if (lifetime == 0) null else CacheControl{ .max_age = lifetime };
+ } orelse {
log.debug(.cache, "no store", .{
.url = url,
- .cache_control = cache_control orelse "null",
- .last_modified = last_modified orelse "null",
+ .reason = "not fresh",
+ .expires = response.expires orelse "null",
+ .last_modified = response.last_modified orelse "null",
});
return null;
};
// get() treats must_revalidate as always-expired, so without validators
// the entry could never be served, only purged.
- if (cc.must_revalidate and etag == null and last_modified == null) {
+ if (cc.must_revalidate and response.etag == null and response.last_modified == null) {
log.debug(.cache, "no store", .{ .url = url, .reason = "must_revalidate without validators" });
return null;
}
return .{
.url = try arena.dupeZ(u8, url),
- .content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",
+ .content_type = if (candidate.content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",
.status = status,
- .stored_at = timestamp,
- .age_at_store = if (age) |a| parseDeltaSeconds(a) orelse 0 else 0,
+ .stored_at = candidate.timestamp,
+ .age_at_store = if (response.age) |a| parseDeltaSeconds(a) orelse 0 else 0,
.cache_control = cc,
- .headers = &.{},
- .vary_headers = &.{},
- .etag = if (etag) |e| try arena.dupe(u8, e) else null,
- .last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null,
+ .headers = candidate.headers,
+ .vary_headers = try varyHeaders(arena, response.vary, candidate.request_headers),
+ .etag = if (response.etag) |e| try arena.dupe(u8, e) else null,
+ .last_modified = if (response.last_modified) |lm| try arena.dupe(u8, lm) else null,
+ };
+}
+
+fn heuristicallyCacheable(status: u16) bool {
+ return switch (status) {
+ // tryCache discards 206 and redirects, so skip them early here
+ 200, 203, 204, 404, 405, 410, 414, 501 => true,
+ else => false,
};
}
-const testing = @import("../../testing.zig");
-test "Cache: CacheControl.parse" {
- try testing.expectEqual(300, CacheControl.parse("max-age=300").?.max_age);
+/// RFC 9111 §4.2.2 suggests 10% of the interval since Last-Modified. The cap
+/// keeps a document that has not changed in years from being held forever.
+fn heuristicLifetime(last_modified: []const u8, now: u64) u64 {
+ const modified_at = parseHttpDate(last_modified) orelse return 0;
+ const now_seconds: i64 = @intCast(now);
+ if (modified_at >= now_seconds) {
+ return 0;
+ }
+ const heuristic_max_age: u64 = 86400;
+ return @min(@as(u64, @intCast(now_seconds - modified_at)) / 10, heuristic_max_age);
+}
- try testing.expectEqual(300, CacheControl.parse("Max-Age=300").?.max_age);
- try testing.expectEqual(300, CacheControl.parse("MAX-AGE=300").?.max_age);
+fn parseHttpDate(value: []const u8) ?i64 {
+ const date_time = lp.datetime.DateTime.parse(value, .rfc822) catch return null;
+ return date_time.unix(.seconds);
+}
- try testing.expectEqual(300, CacheControl.parse("public, max-age=300").?.max_age);
- try testing.expectEqual(300, CacheControl.parse(" max-age=300 ").?.max_age);
+/// The request headers the Vary list names, which a later lookup has to match
+/// against. Names and values stay borrowed from the caller's headers.
+fn varyHeaders(arena: std.mem.Allocator, vary: ?[]const u8, request_headers: []const Http.Header) ![]const Http.Header {
+ const vary_value = vary orelse return &.{};
- try testing.expectEqual(600, CacheControl.parse("max-age=300, s-maxage=600").?.max_age);
- try testing.expectEqual(600, CacheControl.parse("s-maxage=600, max-age=300").?.max_age);
+ var headers: std.ArrayList(Http.Header) = .empty;
+ for (request_headers) |hdr| {
+ var iter = std.mem.splitScalar(u8, vary_value, ',');
+ while (iter.next()) |part| {
+ const name = std.mem.trim(u8, part, &std.ascii.whitespace);
+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+ try headers.append(arena, hdr);
+ }
+ }
+ }
+ return headers.items;
+}
+
+const testing = @import("../../testing.zig");
+test "Cache: ResponseDirectives.parse" {
+ try testing.expectEqual(300, ResponseDirectives.parse("max-age=300").max_age);
+ try testing.expectEqual(300, ResponseDirectives.parse("Max-Age=300").max_age);
+ try testing.expectEqual(300, ResponseDirectives.parse("MAX-AGE=300").max_age);
+ try testing.expectEqual(300, ResponseDirectives.parse("public, max-age=300").max_age);
+ try testing.expectEqual(300, ResponseDirectives.parse(" max-age=300 ").max_age);
+
+ // we are a private cache: s-maxage is for shared caches only
+ try testing.expectEqual(300, ResponseDirectives.parse("max-age=300, s-maxage=600").max_age);
+ try testing.expectEqual(null, ResponseDirectives.parse("s-maxage=600").max_age);
+
+ try testing.expectEqual(true, ResponseDirectives.parse("no-store").no_store);
+ try testing.expectEqual(true, ResponseDirectives.parse("max-age=300, no-store").no_store);
+ try testing.expectEqual(true, ResponseDirectives.parse("no-cache").no_cache);
+ try testing.expectEqual(true, ResponseDirectives.parse("no-cache=\"set-cookie\"").no_cache);
+ try testing.expectEqual(true, ResponseDirectives.parse("public").public);
- try testing.expectEqual(null, CacheControl.parse("no-store"));
- try testing.expectEqual(
- CacheControl{ .max_age = 0, .must_revalidate = true },
- CacheControl.parse("no-cache"),
- );
- try testing.expectEqual(null, CacheControl.parse("private"));
- try testing.expectEqual(null, CacheControl.parse("max-age=300, no-store"));
try testing.expectEqual(
- CacheControl{ .max_age = 300, .must_revalidate = true },
- CacheControl.parse("no-cache, max-age=300"),
+ ResponseDirectives{ .private = true, .max_age = 300 },
+ ResponseDirectives.parse("Private, max-age=300"),
);
- try testing.expectEqual(null, CacheControl.parse("Private, max-age=300"));
-
- try testing.expectEqual(null, CacheControl.parse("max-age=0"));
-
- try testing.expectEqual(null, CacheControl.parse("public"));
- try testing.expectEqual(null, CacheControl.parse(""));
- try testing.expectEqual(null, CacheControl.parse("max-age=abc"));
- try testing.expectEqual(null, CacheControl.parse("max-age="));
+ try testing.expectEqual(ResponseDirectives{}, ResponseDirectives.parse(""));
+ try testing.expectEqual(null, ResponseDirectives.parse("max-age=abc").max_age);
+ try testing.expectEqual(null, ResponseDirectives.parse("max-age=").max_age);
// values longer than 8 digits must not be truncated
- try testing.expectEqual(315360000, CacheControl.parse("max-age=315360000").?.max_age);
+ try testing.expectEqual(315360000, ResponseDirectives.parse("max-age=315360000").max_age);
// delta-seconds too large to represent are capped at 2^31 (RFC 9111 §1.2.2)
- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=2147483649").?.max_age);
- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=9999999999999999999").?.max_age);
- try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=99999999999999999999999").?.max_age);
- try testing.expectEqual(max_delta_seconds, CacheControl.parse("s-maxage=9999999999999999999").?.max_age);
+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=2147483649").max_age);
+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=9999999999999999999").max_age);
+ try testing.expectEqual(max_delta_seconds, ResponseDirectives.parse("max-age=99999999999999999999999").max_age);
}
-test "Cache: tryCache heuristic when no cache-control" {
+// 2026-09-21T07:00:00Z, so that the HTTP-dates below land on either side of it.
+const test_now: u64 = 1789974000;
+
+/// Spells a response out as the header list `tryCache` reads it from.
+const TestResponse = struct {
+ status: u16 = 200,
+ cache_control: ?[]const u8 = null,
+ expires: ?[]const u8 = null,
+ date: ?[]const u8 = null,
+ etag: ?[]const u8 = null,
+ last_modified: ?[]const u8 = null,
+
+ fn run(self: TestResponse, arena: std.mem.Allocator) !?CachePutRequest {
+ var headers: std.ArrayList(Http.Header) = .empty;
+ inline for (.{
+ .{ "Cache-Control", self.cache_control },
+ .{ "Expires", self.expires },
+ .{ "Date", self.date },
+ .{ "ETag", self.etag },
+ .{ "Last-Modified", self.last_modified },
+ }) |field| {
+ if (field[1]) |value| {
+ try headers.append(arena, .{ .name = field[0], .value = value });
+ }
+ }
+
+ return tryCache(arena, .{
+ .timestamp = test_now,
+ .url = "https://example.com",
+ .status = self.status,
+ .content_type = "text/html",
+ .headers = headers.items,
+ .request_headers = &.{},
+ });
+ }
+};
+
+test "Cache: tryCache freshness" {
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
- const result = try tryCache(
- arena.allocator(),
- 1000,
- "https://example.com",
- 200,
- "text/html",
- null,
- null,
- null,
- null,
- null,
- false,
- false,
- );
- try testing.expectEqual(null, result);
+ const cases = [_]struct { max_age: ?u64, response: TestResponse }{
+ // nothing to go on
+ .{ .max_age = null, .response = .{} },
+
+ .{ .max_age = 300, .response = .{ .cache_control = "max-age=300" } },
+ .{ .max_age = null, .response = .{ .cache_control = "max-age=0" } },
+ .{ .max_age = null, .response = .{ .cache_control = "no-store, max-age=300" } },
+
+ // Expires is relative to Date when the response carries one
+ .{ .max_age = 3600, .response = .{
+ .expires = "Mon, 21 Sep 2026 08:00:00 GMT",
+ .date = "Mon, 21 Sep 2026 07:00:00 GMT",
+ } },
+ .{ .max_age = 3600, .response = .{ .expires = "Mon, 21 Sep 2026 08:00:00 GMT" } },
+
+ // a past Expires, and an unparsable one, are both already expired
+ .{ .max_age = null, .response = .{ .expires = "Mon, 21 Sep 2026 06:00:00 GMT" } },
+ .{ .max_age = null, .response = .{ .expires = "0" } },
+
+ // a broken Expires must not fall back to the Last-Modified heuristic
+ .{ .max_age = null, .response = .{
+ .expires = "0",
+ .last_modified = "Mon, 21 Sep 2026 04:13:20 GMT",
+ } },
+
+ // Cache-Control wins over Expires
+ .{ .max_age = 300, .response = .{
+ .cache_control = "max-age=300",
+ .expires = "Mon, 21 Sep 2026 06:00:00 GMT",
+ } },
+ };
+
+ for (cases, 0..) |c, i| {
+ const result = try c.response.run(arena.allocator());
+ testing.expectEqual(c.max_age, if (result) |r| r.cache_control.max_age else null) catch |err| {
+ testing.print("case {d}\n", .{i});
+ return err;
+ };
+ }
+}
+
+test "Cache: tryCache heuristic freshness" {
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ // 10% of the 10000 seconds since it was last modified
+ const modified = "Mon, 21 Sep 2026 04:13:20 GMT";
+
+ const result = try (TestResponse{ .last_modified = modified }).run(arena.allocator());
+ try testing.expectEqual(1000, result.?.cache_control.max_age);
+ try testing.expectEqual(false, result.?.cache_control.must_revalidate);
+
+ // a status with no heuristic of its own, unless the origin said `public`
+ const unknown_status = try (TestResponse{ .status = 299, .last_modified = modified }).run(arena.allocator());
+ try testing.expectEqual(null, unknown_status);
+
+ const public_unknown_status = try (TestResponse{
+ .status = 299,
+ .cache_control = "public",
+ .last_modified = modified,
+ }).run(arena.allocator());
+ try testing.expectEqual(1000, public_unknown_status.?.cache_control.max_age);
+
+ // never stored, however fresh they claim to be
+ for ([_]u16{ 206, 301, 304 }) |status| {
+ const result_ = try (TestResponse{ .status = status, .cache_control = "max-age=300" }).run(arena.allocator());
+ try testing.expectEqual(null, result_);
+ }
}
test "Cache: tryCache must_revalidate without validators" {
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
- const no_validators = try tryCache(
- arena.allocator(),
- 1000,
- "https://example.com",
- 200,
- "text/html",
- "no-cache, max-age=300",
- null,
- null,
- null,
- null,
- false,
- false,
- );
+ const no_validators = try (TestResponse{ .cache_control = "no-cache, max-age=300" }).run(arena.allocator());
try testing.expectEqual(null, no_validators);
- const with_etag = try tryCache(
- arena.allocator(),
- 1000,
- "https://example.com",
- 200,
- "text/html",
- "no-cache, max-age=300",
- null,
- null,
- "\"abc\"",
- null,
- false,
- false,
- );
+ const with_etag = try (TestResponse{
+ .cache_control = "no-cache, max-age=300",
+ .etag = "\"abc\"",
+ }).run(arena.allocator());
try testing.expectEqual(true, with_etag.?.cache_control.must_revalidate);
}
-test "Cache: tryCache heuristic when no cache-control with last-modified" {
+test "Cache: tryCache vary headers" {
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
- const result = try tryCache(
- arena.allocator(),
- 1000,
- "https://example.com",
- 200,
- "text/html",
- null,
- null,
- null,
- null,
- "Wed, 21 Oct 2015 07:28:00 GMT",
- false,
- false,
- );
- try testing.expectEqual(@as(u64, 86400), result.?.cache_control.max_age);
- try testing.expectEqual(false, result.?.cache_control.must_revalidate);
+ const request_headers = [_]Http.Header{
+ .{ .name = "Accept-Encoding", .value = "gzip" },
+ .{ .name = "User-Agent", .value = "lightpanda" },
+ };
+
+ const stored = try tryCache(arena.allocator(), .{
+ .timestamp = test_now,
+ .url = "https://example.com",
+ .status = 200,
+ .content_type = "text/html",
+ .headers = &.{
+ .{ .name = "Cache-Control", .value = "max-age=300" },
+ .{ .name = "Vary", .value = "accept-encoding, accept-language" },
+ },
+ .request_headers = &request_headers,
+ });
+
+ try testing.expectEqual(1, stored.?.vary_headers.len);
+ try testing.expectString("Accept-Encoding", stored.?.vary_headers[0].name);
+ try testing.expectString("gzip", stored.?.vary_headers[0].value);
+
+ // Vary: * is never stored
+ const wildcard = try tryCache(arena.allocator(), .{
+ .timestamp = test_now,
+ .url = "https://example.com",
+ .status = 200,
+ .content_type = "text/html",
+ .headers = &.{
+ .{ .name = "Cache-Control", .value = "max-age=300" },
+ .{ .name = "Vary", .value = "*" },
+ },
+ .request_headers = &request_headers,
+ });
+ try testing.expectEqual(null, wildcard);
}
diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig
index c944fa21a..d33e3eb3c 100644
--- a/src/network/cache/SqliteCache.zig
+++ b/src/network/cache/SqliteCache.zig
@@ -27,8 +27,6 @@ const RenewResponse = Cache.RenewResponse;
const CachePutRequest = Cache.CachePutRequest;
const CacheGetResult = Cache.CacheGetResult;
const CachedResponse = Cache.CachedResponse;
-const CacheControl = Cache.CacheControl;
-const parseDeltaSeconds = Cache.parseDeltaSeconds;
const Http = @import("../http.zig");
const Blob = @import("../../storage/sqlite/Sqlite.zig").Blob;
@@ -351,25 +349,7 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void
try conn.begin(.immediate);
defer conn.rollback() catch {};
- var age_at_store: u64 = 0;
- var content_type: ?[]const u8 = null;
- var etag: ?[]const u8 = null;
- var last_modified: ?[]const u8 = null;
- var cache_control: ?CacheControl = null;
-
- for (req.headers) |h| {
- if (std.ascii.eqlIgnoreCase(h.name, "Age")) {
- age_at_store = parseDeltaSeconds(h.value) orelse 0;
- } else if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {
- cache_control = CacheControl.parse(h.value) orelse continue;
- } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {
- etag = h.value;
- } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {
- last_modified = h.value;
- } else if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
- content_type = h.value;
- }
- }
+ const directive = req.directive();
try conn.exec(
\\ update cache
@@ -382,13 +362,13 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void
\\ last_modified = coalesce($7, last_modified)
\\ where url = $8
, .{
- req.timestamp,
- age_at_store,
- if (cache_control) |cc| cc.max_age else null,
- if (cache_control) |cc| cc.must_revalidate else null,
- content_type,
- etag,
- last_modified,
+ directive.timestamp,
+ directive.age_at_store,
+ directive.max_age,
+ directive.must_revalidate,
+ directive.content_type,
+ directive.etag,
+ directive.last_modified,
req.url,
});