Lightpanda: the headless browser designed for AI and automation
add Cache.revalidate + supporting logic
Muki Kiboigoa6c8a28977ea
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex 5a551a98d..b4a1039a6 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -55,6 +55,12 @@ pub fn evict(self: *Cache, url: []const u8) void {};}+pub fn revalidate(self: *Cache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {+ return switch (self.kind) {+ inline else => |*c| c.revalidate(arena, url, timestamp),+ };+}+pub fn clear(self: *Cache) !void {return switch (self.kind) {inline else => |*c| c.clear(),@@ -122,10 +128,13 @@ pub const CachedMetadata = struct {cache_control: CacheControl,/// Response Headersheaders: []const Http.Header,-/// These are Request Headers used by Vary.vary_headers: []const Http.Header,+ // Validators for conditional requests.+ etag: ?[]const u8 = null,+ last_modified: ?[]const u8 = null,+pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void {try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | vary=[", .{self.url,@@ -145,6 +154,15 @@ pub const CachedMetadata = struct {}try writer.print("]", .{});}++ pub fn isStale(self: CachedMetadata, timestamp: i64) bool {+ const age = (timestamp - self.stored_at) + @as(i64, @intCast(self.age_at_store));+ return age >= @as(i64, @intCast(self.cache_control.max_age));+ }++ pub fn hasValidators(self: CachedMetadata) bool {+ return self.etag != null or self.last_modified != null;+ }};pub const CacheRequest = struct {@@ -172,8 +190,10 @@ pub const CachedData = union(enum) {pub const CachedResponse = struct {metadata: CachedMetadata,data: CachedData,+ expired: bool,pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void {+ try writer.print("expired={}, ", .{self.expired});try writer.print("metadata=(", .{});try self.metadata.format(writer);try writer.print("), data=", .{});diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zigindex 3880b6ee9..4d4919ae5 100644--- a/src/network/cache/FsCache.zig+++ b/src/network/cache/FsCache.zig@@ -72,6 +72,54 @@ fn cacheTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {return path;}+fn writeCacheFile(+ self: *FsCache,+ hashed_key: *const [HASHED_KEY_LEN]u8,+ body_reader: *std.io.Reader,+ body_len: u64,+ meta: CachedMetadata,+) !void {+ const cache_p = cachePath(hashed_key);+ const cache_tmp_p = cacheTmpPath(hashed_key);++ const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| {+ log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e });+ return e;+ };+ errdefer self.dir.deleteFile(&cache_tmp_p) catch {};+ defer file.close();++ var writer_buf: [1024]u8 = undefined;+ var file_writer = file.writer(&writer_buf);+ const w = &file_writer.interface;++ var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;+ std.mem.writeInt(u64, &len_buf, body_len, .little);+ try w.writeAll(&len_buf);++ var copy_buf: [4096]u8 = undefined;+ var remaining = body_len;+ while (remaining > 0) {+ const to_read = @min(copy_buf.len, remaining);+ const n = try body_reader.readSliceShort(copy_buf[0..to_read]);+ if (n == 0) break;+ try w.writeAll(copy_buf[0..n]);+ remaining -= n;+ }++ try std.json.Stringify.value(+ CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta },+ .{ .whitespace = .minified },+ w,+ );+ try w.flush();++ self.dir.rename(&cache_tmp_p, &cache_p) catch |e| {+ log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e });+ return e;+ };+}+pub fn init(path: []const u8) !FsCache {const cwd = std.fs.cwd();@@ -162,15 +210,6 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedRconst metadata = cache_file.metadata;- // Check entry expiration.- const now = req.timestamp;- const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store));- if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) {- log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired" });- cleanup = true;- return null;- }-// If we have Vary headers, ensure they are present & matching.for (metadata.vary_headers) |vary_hdr| {const name = vary_hdr.name;@@ -199,7 +238,9 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedRreturn null;}- log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key });+ // Check entry expiration.+ const expired = metadata.isStale(req.timestamp);+ log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key, .expired = expired });return .{.metadata = metadata,@@ -210,56 +251,19 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedR.len = body_len,},},+ .expired = expired,};}pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void {const hashed_key = hashKey(meta.url);- const cache_p = cachePath(&hashed_key);- const cache_tmp_p = cacheTmpPath(&hashed_key);const lock = self.getLockPtr(&hashed_key);lock.lock();defer lock.unlock();- const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| {- log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e });- return e;- };- errdefer self.dir.deleteFile(&cache_tmp_p) catch {};- defer file.close();-- var writer_buf: [1024]u8 = undefined;- var file_writer = file.writer(&writer_buf);- var file_writer_iface = &file_writer.interface;-- var len_buf: [8]u8 = undefined;- std.mem.writeInt(u64, &len_buf, body.len, .little);-- file_writer_iface.writeAll(&len_buf) catch |e| {- log.err(.cache, "write body len", .{ .url = meta.url, .err = e });- return e;- };- file_writer_iface.writeAll(body) catch |e| {- log.err(.cache, "write body", .{ .url = meta.url, .err = e });- return e;- };- std.json.Stringify.value(- CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta },- .{ .whitespace = .minified },- file_writer_iface,- ) catch |e| {- log.err(.cache, "write metadata", .{ .url = meta.url, .err = e });- return e;- };- file_writer_iface.flush() catch |e| {- log.err(.cache, "flush", .{ .url = meta.url, .err = e });- return e;- };- self.dir.rename(&cache_tmp_p, &cache_p) catch |e| {- log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e });- return e;- };+ var body_reader = std.io.Reader.fixed(body);+ try self.writeCacheFile(&hashed_key, &body_reader, body.len, meta);log.debug(.cache, "put", .{ .url = meta.url, .hash = &hashed_key, .body_len = body.len });}@@ -294,6 +298,57 @@ pub fn evict(self: *FsCache, url: []const u8) void {};}+pub fn revalidate(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {+ const hashed_key = hashKey(url);+ const cache_p = cachePath(&hashed_key);++ const lock = self.getLockPtr(&hashed_key);+ lock.lock();+ defer lock.unlock();++ const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| {+ log.warn(.cache, "revalidate open failed", .{ .url = url, .err = e });+ return e;+ };+ defer file.close();++ var file_buf: [1024]u8 = undefined;+ var file_reader = file.reader(&file_buf);+ const r = &file_reader.interface;++ var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;+ r.readSliceAll(&len_buf) catch |e| {+ log.warn(.cache, "revalidate read len", .{ .url = url, .err = e });+ return e;+ };+ const body_len = std.mem.readInt(u64, &len_buf, .little);++ try file_reader.seekTo(BODY_LEN_HEADER_LEN + body_len);++ var json_reader = std.json.Reader.init(arena, r);+ var parsed = std.json.parseFromTokenSourceLeaky(+ CacheMetadataJson,+ arena,+ &json_reader,+ .{ .allocate = .alloc_always },+ ) catch |e| {+ log.warn(.cache, "revalidate parse", .{ .url = url, .err = e });+ return e;+ };++ parsed.metadata.stored_at = timestamp;+ parsed.metadata.age_at_store = 0;++ try file_reader.seekTo(BODY_LEN_HEADER_LEN);++ self.writeCacheFile(&hashed_key, r, body_len, parsed.metadata) catch |e| {+ log.warn(.cache, "revalidate write", .{ .url = url, .err = e });+ return e;+ };++ log.debug(.cache, "revalidated", .{ .url = url });+}+const testing = std.testing;fn setupCache() !struct { tmp: testing.TmpDir, cache: Cache } {@@ -396,23 +451,17 @@ test "FsCache: get expiration" {) orelse return error.CacheMiss;result.data.file.file.close();- try testing.expectEqual(null, cache.get(+ // Expired: age = 200 + 900 = 1100 >= 1000+ const stale = cache.get(arena.allocator(),.{.url = "https://example.com",.timestamp = now + 200,.request_headers = &.{},},- ));-- try testing.expectEqual(null, cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now,- .request_headers = &.{},- },- ));+ ) orelse return error.CacheMiss;+ defer stale.data.file.file.close();+ try testing.expectEqual(true, stale.expired);}test "FsCache: put override" {@@ -837,3 +886,105 @@ test "FsCache: evict removes entry" {},));}++test "FsCache: revalidate refreshes expiry" {+ var setup = try setupCache();+ defer {+ setup.cache.deinit();+ setup.tmp.cleanup();+ }++ const cache = &setup.cache;++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: i64 = 5000;+ const max_age: u64 = 1000;++ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = max_age },+ .headers = &.{},+ .vary_headers = &.{},+ };++ try cache.put(meta, "hello world");++ // Revalidate while still fresh at now+500+ try cache.revalidate(arena.allocator(), "https://example.com", now + 500);++ // Without revalidation would expire at now+1000, but clock reset to now+500+ // so still fresh at now+1200+ const r1 = cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 1200,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ r1.data.file.file.close();++ // Expires at now+500+1000 = now+1500+ const stale1 = cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 1500,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ stale1.data.file.file.close();+ try testing.expectEqual(true, stale1.expired);+}++test "FsCache: revalidate preserves body" {+ var setup = try setupCache();+ defer {+ setup.cache.deinit();+ setup.tmp.cleanup();+ }++ const cache = &setup.cache;++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now = std.time.timestamp();+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ };++ const body = "original body";+ try cache.put(meta, body);++ try cache.revalidate(arena.allocator(), "https://example.com", now + 100);++ const result = cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now + 100,+ .request_headers = &.{},+ }) orelse return error.CacheMiss;++ const f = result.data.file;+ defer f.file.close();++ var buf: [64]u8 = undefined;+ var file_reader = f.file.reader(&buf);+ try file_reader.seekTo(f.offset);+ const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len);+ defer testing.allocator.free(read_buf);+ try testing.expectEqualStrings(body, read_buf);+}diff --git a/src/network/layer/CacheLayer.zig b/src/network/layer/CacheLayer.zigindex 4164102e4..fca3e192f 100644--- a/src/network/layer/CacheLayer.zig+++ b/src/network/layer/CacheLayer.zig@@ -65,37 +65,42 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {.timestamp = std.time.timestamp(),.request_headers = req_header_list.items,})) |cached| {- // Dispatch that the Request was served from the Cache.- transfer.req.notification.dispatch(- .http_request_served_from_cache,- &.{ .transfer = transfer },- );-- const ctx = try arena.create(CachedResponse);- ctx.* = cached;-- try transfer.client.runNextTick(transfer, ctx, .{- .run = struct {- fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {- defer t.deinit();-- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));- serveFromCache(&t.req, c) catch |err| {- t.req.error_callback(t.req.ctx, err);- };- }- }.run,- .abort = struct {- fn abort(ctx_ptr: ?*anyopaque) void {- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));- switch (c.data) {- .buffer => |_| {},- .file => |f| f.file.close(),+ if (cached.expired) {+ // If it is expired, evict from Cache.+ transfer.client.network.cache.?.evict(req.url);+ } else {+ // Dispatch that the Request was served from the Cache.+ transfer.req.notification.dispatch(+ .http_request_served_from_cache,+ &.{ .transfer = transfer },+ );++ const ctx = try arena.create(CachedResponse);+ ctx.* = cached;++ try transfer.client.runNextTick(transfer, ctx, .{+ .run = struct {+ fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {+ defer t.deinit();++ const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));+ serveFromCache(&t.req, c) catch |err| {+ t.req.error_callback(t.req.ctx, err);+ };}- }- }.abort,- });- return;+ }.run,+ .abort = struct {+ fn abort(ctx_ptr: ?*anyopaque) void {+ const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));+ switch (c.data) {+ .buffer => |_| {},+ .file => |f| f.file.close(),+ }+ }+ }.abort,+ });+ return;+ }}// Cache miss: install wrappers so we can inspect the response and decide