Skip to content

Lightpanda: the headless browser designed for AI and automation

add Cache.revalidate + supporting logic

Muki Kiboigoa6c8a28977ea
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index 5a551a98d..b4a1039a6 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -55,6 +55,12 @@ pub fn evict(self: *Cache, url: []const u8) void {
};
}
+pub fn revalidate(self: *Cache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {
+ return switch (self.kind) {
+ inline else => |*c| c.revalidate(arena, url, timestamp),
+ };
+}
+
pub fn clear(self: *Cache) !void {
return switch (self.kind) {
inline else => |*c| c.clear(),
@@ -122,10 +128,13 @@ pub const CachedMetadata = struct {
cache_control: CacheControl,
/// Response Headers
headers: []const Http.Header,
-
/// These are Request Headers used by Vary.
vary_headers: []const Http.Header,
+ // Validators for conditional requests.
+ etag: ?[]const u8 = null,
+ last_modified: ?[]const u8 = null,
+
pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void {
try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | vary=[", .{
self.url,
@@ -145,6 +154,15 @@ pub const CachedMetadata = struct {
}
try writer.print("]", .{});
}
+
+ pub fn isStale(self: CachedMetadata, timestamp: i64) bool {
+ const age = (timestamp - self.stored_at) + @as(i64, @intCast(self.age_at_store));
+ return age >= @as(i64, @intCast(self.cache_control.max_age));
+ }
+
+ pub fn hasValidators(self: CachedMetadata) bool {
+ return self.etag != null or self.last_modified != null;
+ }
};
pub const CacheRequest = struct {
@@ -172,8 +190,10 @@ pub const CachedData = union(enum) {
pub const CachedResponse = struct {
metadata: CachedMetadata,
data: CachedData,
+ expired: bool,
pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void {
+ try writer.print("expired={}, ", .{self.expired});
try writer.print("metadata=(", .{});
try self.metadata.format(writer);
try writer.print("), data=", .{});
diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig
index 3880b6ee9..4d4919ae5 100644
--- a/src/network/cache/FsCache.zig
+++ b/src/network/cache/FsCache.zig
@@ -72,6 +72,54 @@ fn cacheTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
return path;
}
+fn writeCacheFile(
+ self: *FsCache,
+ hashed_key: *const [HASHED_KEY_LEN]u8,
+ body_reader: *std.io.Reader,
+ body_len: u64,
+ meta: CachedMetadata,
+) !void {
+ const cache_p = cachePath(hashed_key);
+ const cache_tmp_p = cacheTmpPath(hashed_key);
+
+ const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| {
+ log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e });
+ return e;
+ };
+ errdefer self.dir.deleteFile(&cache_tmp_p) catch {};
+ defer file.close();
+
+ var writer_buf: [1024]u8 = undefined;
+ var file_writer = file.writer(&writer_buf);
+ const w = &file_writer.interface;
+
+ var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;
+ std.mem.writeInt(u64, &len_buf, body_len, .little);
+ try w.writeAll(&len_buf);
+
+ var copy_buf: [4096]u8 = undefined;
+ var remaining = body_len;
+ while (remaining > 0) {
+ const to_read = @min(copy_buf.len, remaining);
+ const n = try body_reader.readSliceShort(copy_buf[0..to_read]);
+ if (n == 0) break;
+ try w.writeAll(copy_buf[0..n]);
+ remaining -= n;
+ }
+
+ try std.json.Stringify.value(
+ CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta },
+ .{ .whitespace = .minified },
+ w,
+ );
+ try w.flush();
+
+ self.dir.rename(&cache_tmp_p, &cache_p) catch |e| {
+ log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e });
+ return e;
+ };
+}
+
pub fn init(path: []const u8) !FsCache {
const cwd = std.fs.cwd();
@@ -162,15 +210,6 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedR
const metadata = cache_file.metadata;
- // Check entry expiration.
- const now = req.timestamp;
- const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store));
- if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) {
- log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired" });
- cleanup = true;
- return null;
- }
-
// If we have Vary headers, ensure they are present & matching.
for (metadata.vary_headers) |vary_hdr| {
const name = vary_hdr.name;
@@ -199,7 +238,9 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedR
return null;
}
- log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key });
+ // Check entry expiration.
+ const expired = metadata.isStale(req.timestamp);
+ log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key, .expired = expired });
return .{
.metadata = metadata,
@@ -210,56 +251,19 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?CachedR
.len = body_len,
},
},
+ .expired = expired,
};
}
pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void {
const hashed_key = hashKey(meta.url);
- const cache_p = cachePath(&hashed_key);
- const cache_tmp_p = cacheTmpPath(&hashed_key);
const lock = self.getLockPtr(&hashed_key);
lock.lock();
defer lock.unlock();
- const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| {
- log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e });
- return e;
- };
- errdefer self.dir.deleteFile(&cache_tmp_p) catch {};
- defer file.close();
-
- var writer_buf: [1024]u8 = undefined;
- var file_writer = file.writer(&writer_buf);
- var file_writer_iface = &file_writer.interface;
-
- var len_buf: [8]u8 = undefined;
- std.mem.writeInt(u64, &len_buf, body.len, .little);
-
- file_writer_iface.writeAll(&len_buf) catch |e| {
- log.err(.cache, "write body len", .{ .url = meta.url, .err = e });
- return e;
- };
- file_writer_iface.writeAll(body) catch |e| {
- log.err(.cache, "write body", .{ .url = meta.url, .err = e });
- return e;
- };
- std.json.Stringify.value(
- CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta },
- .{ .whitespace = .minified },
- file_writer_iface,
- ) catch |e| {
- log.err(.cache, "write metadata", .{ .url = meta.url, .err = e });
- return e;
- };
- file_writer_iface.flush() catch |e| {
- log.err(.cache, "flush", .{ .url = meta.url, .err = e });
- return e;
- };
- self.dir.rename(&cache_tmp_p, &cache_p) catch |e| {
- log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e });
- return e;
- };
+ var body_reader = std.io.Reader.fixed(body);
+ try self.writeCacheFile(&hashed_key, &body_reader, body.len, meta);
log.debug(.cache, "put", .{ .url = meta.url, .hash = &hashed_key, .body_len = body.len });
}
@@ -294,6 +298,57 @@ pub fn evict(self: *FsCache, url: []const u8) void {
};
}
+pub fn revalidate(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {
+ const hashed_key = hashKey(url);
+ const cache_p = cachePath(&hashed_key);
+
+ const lock = self.getLockPtr(&hashed_key);
+ lock.lock();
+ defer lock.unlock();
+
+ const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| {
+ log.warn(.cache, "revalidate open failed", .{ .url = url, .err = e });
+ return e;
+ };
+ defer file.close();
+
+ var file_buf: [1024]u8 = undefined;
+ var file_reader = file.reader(&file_buf);
+ const r = &file_reader.interface;
+
+ var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;
+ r.readSliceAll(&len_buf) catch |e| {
+ log.warn(.cache, "revalidate read len", .{ .url = url, .err = e });
+ return e;
+ };
+ const body_len = std.mem.readInt(u64, &len_buf, .little);
+
+ try file_reader.seekTo(BODY_LEN_HEADER_LEN + body_len);
+
+ var json_reader = std.json.Reader.init(arena, r);
+ var parsed = std.json.parseFromTokenSourceLeaky(
+ CacheMetadataJson,
+ arena,
+ &json_reader,
+ .{ .allocate = .alloc_always },
+ ) catch |e| {
+ log.warn(.cache, "revalidate parse", .{ .url = url, .err = e });
+ return e;
+ };
+
+ parsed.metadata.stored_at = timestamp;
+ parsed.metadata.age_at_store = 0;
+
+ try file_reader.seekTo(BODY_LEN_HEADER_LEN);
+
+ self.writeCacheFile(&hashed_key, r, body_len, parsed.metadata) catch |e| {
+ log.warn(.cache, "revalidate write", .{ .url = url, .err = e });
+ return e;
+ };
+
+ log.debug(.cache, "revalidated", .{ .url = url });
+}
+
const testing = std.testing;
fn setupCache() !struct { tmp: testing.TmpDir, cache: Cache } {
@@ -396,23 +451,17 @@ test "FsCache: get expiration" {
) orelse return error.CacheMiss;
result.data.file.file.close();
- try testing.expectEqual(null, cache.get(
+ // Expired: age = 200 + 900 = 1100 >= 1000
+ const stale = cache.get(
arena.allocator(),
.{
.url = "https://example.com",
.timestamp = now + 200,
.request_headers = &.{},
},
- ));
-
- try testing.expectEqual(null, cache.get(
- arena.allocator(),
- .{
- .url = "https://example.com",
- .timestamp = now,
- .request_headers = &.{},
- },
- ));
+ ) orelse return error.CacheMiss;
+ defer stale.data.file.file.close();
+ try testing.expectEqual(true, stale.expired);
}
test "FsCache: put override" {
@@ -837,3 +886,105 @@ test "FsCache: evict removes entry" {
},
));
}
+
+test "FsCache: revalidate refreshes expiry" {
+ var setup = try setupCache();
+ defer {
+ setup.cache.deinit();
+ setup.tmp.cleanup();
+ }
+
+ const cache = &setup.cache;
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: i64 = 5000;
+ const max_age: u64 = 1000;
+
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = max_age },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+
+ try cache.put(meta, "hello world");
+
+ // Revalidate while still fresh at now+500
+ try cache.revalidate(arena.allocator(), "https://example.com", now + 500);
+
+ // Without revalidation would expire at now+1000, but clock reset to now+500
+ // so still fresh at now+1200
+ const r1 = cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 1200,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ r1.data.file.file.close();
+
+ // Expires at now+500+1000 = now+1500
+ const stale1 = cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 1500,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ stale1.data.file.file.close();
+ try testing.expectEqual(true, stale1.expired);
+}
+
+test "FsCache: revalidate preserves body" {
+ var setup = try setupCache();
+ defer {
+ setup.cache.deinit();
+ setup.tmp.cleanup();
+ }
+
+ const cache = &setup.cache;
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now = std.time.timestamp();
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+
+ const body = "original body";
+ try cache.put(meta, body);
+
+ try cache.revalidate(arena.allocator(), "https://example.com", now + 100);
+
+ const result = cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now + 100,
+ .request_headers = &.{},
+ }) orelse return error.CacheMiss;
+
+ const f = result.data.file;
+ defer f.file.close();
+
+ var buf: [64]u8 = undefined;
+ var file_reader = f.file.reader(&buf);
+ try file_reader.seekTo(f.offset);
+ const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len);
+ defer testing.allocator.free(read_buf);
+ try testing.expectEqualStrings(body, read_buf);
+}
diff --git a/src/network/layer/CacheLayer.zig b/src/network/layer/CacheLayer.zig
index 4164102e4..fca3e192f 100644
--- a/src/network/layer/CacheLayer.zig
+++ b/src/network/layer/CacheLayer.zig
@@ -65,37 +65,42 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
.timestamp = std.time.timestamp(),
.request_headers = req_header_list.items,
})) |cached| {
- // Dispatch that the Request was served from the Cache.
- transfer.req.notification.dispatch(
- .http_request_served_from_cache,
- &.{ .transfer = transfer },
- );
-
- const ctx = try arena.create(CachedResponse);
- ctx.* = cached;
-
- try transfer.client.runNextTick(transfer, ctx, .{
- .run = struct {
- fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {
- defer t.deinit();
-
- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
- serveFromCache(&t.req, c) catch |err| {
- t.req.error_callback(t.req.ctx, err);
- };
- }
- }.run,
- .abort = struct {
- fn abort(ctx_ptr: ?*anyopaque) void {
- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
- switch (c.data) {
- .buffer => |_| {},
- .file => |f| f.file.close(),
+ if (cached.expired) {
+ // If it is expired, evict from Cache.
+ transfer.client.network.cache.?.evict(req.url);
+ } else {
+ // Dispatch that the Request was served from the Cache.
+ transfer.req.notification.dispatch(
+ .http_request_served_from_cache,
+ &.{ .transfer = transfer },
+ );
+
+ const ctx = try arena.create(CachedResponse);
+ ctx.* = cached;
+
+ try transfer.client.runNextTick(transfer, ctx, .{
+ .run = struct {
+ fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {
+ defer t.deinit();
+
+ const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
+ serveFromCache(&t.req, c) catch |err| {
+ t.req.error_callback(t.req.ctx, err);
+ };
}
- }
- }.abort,
- });
- return;
+ }.run,
+ .abort = struct {
+ fn abort(ctx_ptr: ?*anyopaque) void {
+ const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
+ switch (c.data) {
+ .buffer => |_| {},
+ .file => |f| f.file.close(),
+ }
+ }
+ }.abort,
+ });
+ return;
+ }
}
// Cache miss: install wrappers so we can inspect the response and decide