Skip to content

Lightpanda: the headless browser designed for AI and automation

update more fields on renew

Muki Kiboigo47483fd9923e
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index 77e62393c..aea890f25 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -55,9 +55,9 @@ pub fn evict(self: *Cache, url: []const u8) void {
};
}
-pub fn renew(self: *Cache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {
+pub fn renew(self: *Cache, arena: std.mem.Allocator, req: RenewRequest) !void {
return switch (self.kind) {
- inline else => |*c| c.renew(arena, url, timestamp),
+ inline else => |*c| c.renew(arena, req),
};
}
@@ -173,6 +173,26 @@ pub const CachedMetadata = struct {
pub fn hasValidators(self: CachedMetadata) bool {
return self.etag != null or self.last_modified != null;
}
+
+ pub fn renew(self: *CachedMetadata, req: RenewRequest) void {
+ self.stored_at = req.timestamp;
+ self.age_at_store = 0;
+
+ for (req.headers) |h| {
+ const name = h.name;
+ const value = h.value;
+
+ if (std.ascii.eqlIgnoreCase("Age", name)) {
+ self.age_at_store = std.fmt.parseInt(u64, value, 10) catch 0;
+ } else if (std.ascii.eqlIgnoreCase("Cache-Control", name)) {
+ self.cache_control = CacheControl.parse(value) orelse continue;
+ } else if (std.ascii.eqlIgnoreCase("ETag", name)) {
+ self.etag = value;
+ } else if (std.ascii.eqlIgnoreCase("Last-Modified", name)) {
+ self.last_modified = value;
+ }
+ }
+ }
};
pub const CacheRequest = struct {
@@ -181,6 +201,12 @@ pub const CacheRequest = struct {
request_headers: []const Http.Header,
};
+pub const RenewRequest = struct {
+ url: []const u8,
+ timestamp: i64,
+ headers: []const Http.Header,
+};
+
pub const CachedData = union(enum) {
buffer: []const u8,
file: struct {
diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig
index 69a495b82..ade474624 100644
--- a/src/network/cache/FsCache.zig
+++ b/src/network/cache/FsCache.zig
@@ -23,6 +23,7 @@ const Cache = @import("Cache.zig");
const log = lp.log;
const CacheRequest = Cache.CacheRequest;
+const RenewRequest = Cache.RenewRequest;
const CachedMetadata = Cache.CachedMetadata;
const CachedResponse = Cache.CachedResponse;
@@ -298,8 +299,8 @@ pub fn evict(self: *FsCache, url: []const u8) void {
};
}
-pub fn renew(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestamp: i64) !void {
- const hashed_key = hashKey(url);
+pub fn renew(self: *FsCache, arena: std.mem.Allocator, req: RenewRequest) !void {
+ const hashed_key = hashKey(req.url);
const cache_p = cachePath(&hashed_key);
const lock = self.getLockPtr(&hashed_key);
@@ -307,7 +308,7 @@ pub fn renew(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestam
defer lock.unlock();
const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| {
- log.warn(.cache, "renew open failed", .{ .url = url, .err = e });
+ log.warn(.cache, "renew open failed", .{ .url = req.url, .err = e });
return e;
};
defer file.close();
@@ -318,7 +319,7 @@ pub fn renew(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestam
var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;
r.readSliceAll(&len_buf) catch |e| {
- log.warn(.cache, "renew read len", .{ .url = url, .err = e });
+ log.warn(.cache, "renew read len", .{ .url = req.url, .err = e });
return e;
};
const body_len = std.mem.readInt(u64, &len_buf, .little);
@@ -326,27 +327,25 @@ pub fn renew(self: *FsCache, arena: std.mem.Allocator, url: []const u8, timestam
try file_reader.seekTo(BODY_LEN_HEADER_LEN + body_len);
var json_reader = std.json.Reader.init(arena, r);
- var parsed = std.json.parseFromTokenSourceLeaky(
+ var parsed: CacheMetadataJson = std.json.parseFromTokenSourceLeaky(
CacheMetadataJson,
arena,
&json_reader,
.{ .allocate = .alloc_always },
) catch |e| {
- log.warn(.cache, "renew parse", .{ .url = url, .err = e });
+ log.warn(.cache, "renew parse", .{ .url = req.url, .err = e });
return e;
};
- parsed.metadata.stored_at = timestamp;
- parsed.metadata.age_at_store = 0;
-
+ parsed.metadata.renew(req);
try file_reader.seekTo(BODY_LEN_HEADER_LEN);
self.writeCacheFile(&hashed_key, r, body_len, parsed.metadata) catch |e| {
- log.warn(.cache, "renew write", .{ .url = url, .err = e });
+ log.warn(.cache, "renew write", .{ .url = req.url, .err = e });
return e;
};
- log.debug(.cache, "renewed", .{ .url = url });
+ log.debug(.cache, "renewed", .{ .url = req.url });
}
const testing = std.testing;
@@ -916,7 +915,14 @@ test "FsCache: renew refreshes expiry" {
try cache.put(meta, "hello world");
// renew while still fresh at now+500
- try cache.renew(arena.allocator(), "https://example.com", now + 500);
+ try cache.renew(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 500,
+ .headers = &.{},
+ },
+ );
// Without revalidation would expire at now+1000, but clock reset to now+500
// so still fresh at now+1200
@@ -970,7 +976,14 @@ test "FsCache: renew preserves body" {
const body = "original body";
try cache.put(meta, body);
- try cache.renew(arena.allocator(), "https://example.com", now + 100);
+ try cache.renew(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 100,
+ .headers = &.{},
+ },
+ );
const result = cache.get(arena.allocator(), .{
.url = "https://example.com",
diff --git a/src/network/layer/CacheLayer.zig b/src/network/layer/CacheLayer.zig
index 71173dc99..b674780d7 100644
--- a/src/network/layer/CacheLayer.zig
+++ b/src/network/layer/CacheLayer.zig
@@ -277,10 +277,16 @@ const CacheContext = struct {
const stale = self.stale_entry.?;
self.stale_entry = null;
+ var iter = self.req_headers.iterator();
+ const headers = try iter.collect(arena);
+
transfer.client.network.cache.?.renew(
arena,
- self.req_url,
- std.time.timestamp(),
+ .{
+ .url = self.req_url,
+ .timestamp = std.time.timestamp(),
+ .headers = headers.items,
+ },
) catch |err| {
log.warn(.cache, "renew failed", .{ .err = err });
};