Lightpanda: the headless browser designed for AI and automation
chore: Move cache maintenance to startup and browser.deinit
Karl Seguina3034105aaaf
diff --git a/src/browser/Browser.zig b/src/browser/Browser.zigindex 225af81c5..368b3383b 100644--- a/src/browser/Browser.zig+++ b/src/browser/Browser.zig@@ -154,6 +154,9 @@ pub fn deinit(self: *Browser) void {// fire — only now is it safe to free the pool backing their parameters.self.fc_identity_pool.deinit(allocator);self.page_pool.deinit(allocator);+ if (self.http_client.cache) |cache| {+ cache.maintenance(lp.datetime.timestamp(.real));+ }self.http_client.deinit();self.clearPermissions();self.permissions.deinit(allocator);diff --git a/src/help.zon b/src/help.zonindex 83104a8c3..99dd019fa 100644--- a/src/help.zon+++ b/src/help.zon@@ -346,7 +346,9 @@\\ this disables caching.\\ Defaults to no caching.\\ --http-cache-entry-limit <INT>- \\ Maximum number of entries that can be stored in the HTTP cache. 0 means no limit.+ \\ Maximum number of entries kept in the HTTP cache. The limit is+ \\ soft: it is enforced at startup and when a browser session ends.+ \\ The cache can temporarily exceed it. 0 means no limit.\\ Defaults to 1000.\\ --http-connect-timeout <INT>\\ Time in ms to establish an HTTP connection before timing out. 0 meansdiff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex 1c593c386..71fd819c7 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -67,6 +67,12 @@ pub fn clear(self: *Cache) !void {};}+pub fn maintenance(self: *Cache, now: u64) void {+ return switch (self.kind) {+ inline else => |*c| c.maintenance(now),+ };+}+/// RFC 9111 delta-seconds values larger than this are capped rather than/// rejected (§1.2.2). Capping also keeps the value safely castable to i64/// for freshness arithmetic and storage.@@ -283,6 +289,13 @@ pub fn tryCache(return null;};+ // get() treats must_revalidate as always-expired, so without validators+ // the entry could never be served, only purged.+ if (cc.must_revalidate and etag == null and last_modified == null) {+ log.debug(.cache, "no store", .{ .url = url, .reason = "must_revalidate without validators" });+ return null;+ }+return .{.url = try arena.dupeZ(u8, url),.content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",@@ -362,6 +375,43 @@ test "Cache: tryCache heuristic when no cache-control" {try testing.expectEqual(null, result);}+test "Cache: tryCache must_revalidate without validators" {+ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const no_validators = try tryCache(+ arena.allocator(),+ 1000,+ "https://example.com",+ 200,+ "text/html",+ "no-cache, max-age=300",+ null,+ null,+ null,+ null,+ false,+ false,+ );+ try testing.expectEqual(null, no_validators);++ const with_etag = try tryCache(+ arena.allocator(),+ 1000,+ "https://example.com",+ 200,+ "text/html",+ "no-cache, max-age=300",+ null,+ null,+ "\"abc\"",+ null,+ false,+ false,+ );+ try testing.expectEqual(true, with_etag.?.cache_control.must_revalidate);+}+test "Cache: tryCache heuristic when no cache-control with last-modified" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zigindex 45322e45e..a9c2c394f 100644--- a/src/network/cache/SqliteCache.zig+++ b/src/network/cache/SqliteCache.zig@@ -69,8 +69,19 @@ const cache_migrations: []const Migration = &.{},.{ .sql = "create index header_url on header(url)" },.{ .sql = "create index cache_stored_at on cache(stored_at)" },+ .{ .sql =+ \\ create index cache_expiry on cache(stored_at + max_age - age_at_store)+ \\ where etag is null and last_modified is null+ },};+// Don't change this without also updating the cache_expiry index.+const purge_expired_sql =+ \\ delete from cache+ \\ where etag is null and last_modified is null+ \\ and stored_at + max_age - age_at_store <= $1+;+pub const SqliteCachePath = union(enum) {path: []const u8,memory,@@ -126,26 +137,46 @@ pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath, entry_limit: u3try conn.exec("pragma foreign_keys=on", .{});}+ var cache: SqliteCache = .{ .allocator = allocator, .pool = pool, .entry_limit = entry_limit };+ cache.maintenance(lp.datetime.timestamp(.real));+log.info(.cache, "sqlite cache initialized", .{ .path = path, .entry_limit = entry_limit, .version = version });- return .{ .allocator = allocator, .pool = pool, .entry_limit = entry_limit };+ return cache;}pub fn deinit(self: *SqliteCache) void {self.pool.deinit(self.allocator);}-fn evictOverflow(self: *SqliteCache, conn: Conn) !void {- const limit = self.entry_limit;- if (limit == 0) return;+/// Removes expired entries and enforces size limit.+pub fn maintenance(self: *SqliteCache, now: u64) void {+ const conn = self.pool.acquire() catch |err| {+ log.err(.cache, "sqlite acquire", .{ .err = err });+ return;+ };+ defer self.pool.release(conn);++ conn.exec(purge_expired_sql, .{@as(i64, @intCast(now))}) catch |err| {+ log.err(.cache, "purge expired", .{ .err = err });+ };+ const purged = conn.changes();- try conn.exec(+ if (self.entry_limit == 0) {+ return;+ }++ conn.exec(\\ delete from cache\\ where rowid in (\\ select rowid from cache\\ order by stored_at desc\\ limit -1 offset $1\\ )- , .{@as(i64, @intCast(limit))});+ , .{@as(i64, @intCast(self.entry_limit))}) catch |err| {+ log.err(.cache, "evict overflow", .{ .err = err });+ };++ log.debug(.cache, "maintenance", .{ .purged = purged, .evicted = conn.changes() });}pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {@@ -286,7 +317,6 @@ pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void {);}- try self.evictOverflow(conn);try conn.commit();log.debug(.cache, "put", .{ .url = req.url, .body_len = body.len });@@ -918,6 +948,89 @@ test "SqliteCache: renew preserves body" {try testing.expectEqualStrings("original body", result.hit.data.buffer);}+test "SqliteCache: maintenance purges expired entries without validators" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ // Expires at 6000, no validators: purged.+ try cache.put(.{+ .url = "https://example.com/expired",+ .content_type = "text/html",+ .status = 200,+ .stored_at = 5000,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 1000 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "expired");++ // Expires at 6000 but has an etag: kept for revalidation.+ try cache.put(.{+ .url = "https://example.com/revalidate",+ .content_type = "text/html",+ .status = 200,+ .stored_at = 5000,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 1000 },+ .etag = "ABC",+ .headers = &.{},+ .vary_headers = &.{},+ }, "revalidate");++ // Expires at 7900: kept.+ try cache.put(.{+ .url = "https://example.com/fresh",+ .content_type = "text/html",+ .status = 200,+ .stored_at = 6900,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 1000 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "fresh");++ cache.maintenance(7000);++ try testing.expect((try cache.get(arena.allocator(), .{+ .url = "https://example.com/expired",+ .timestamp = 7000,+ .request_headers = &.{},+ })) == .miss);+ try testing.expect((try cache.get(arena.allocator(), .{+ .url = "https://example.com/revalidate",+ .timestamp = 7000,+ .request_headers = &.{},+ })) == .revalidate);+ try testing.expect((try cache.get(arena.allocator(), .{+ .url = "https://example.com/fresh",+ .timestamp = 7000,+ .request_headers = &.{},+ })) == .hit);+}++test "SqliteCache: expired purge uses the cache_expiry index" {+ var cache = try SqliteCache.init(testing.allocator, .memory, 0);+ defer cache.deinit();++ const conn = try cache.pool.acquire();+ defer cache.pool.release(conn);++ var plan = (try conn.row(+ "explain query plan " ++ purge_expired_sql,+ .{@as(i64, 0)},+ )) orelse return error.NoQueryPlan;+ defer plan.deinit();++ const detail = plan.get([]const u8, 3);+ if (std.mem.indexOf(u8, detail, "cache_expiry") == null) {+ std.debug.print("query plan: {s}\n", .{detail});+ return error.FullTableScan;+ }+}+test "SqliteCache: evicts oldest entries over the limit" {var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .memory, 3) } };defer cache.deinit();@@ -947,6 +1060,8 @@ test "SqliteCache: evicts oldest entries over the limit" {}, url);}+ cache.maintenance(now + urls.len);+const evicted = try cache.get(arena.allocator(), .{.url = "https://example.com/a",.timestamp = now,