Lightpanda: the headless browser designed for AI and automation
add SqliteCache as default local impl
Muki Kiboigo0f2957cda2a3
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 57da9a0f6..89d020e7b 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -905,7 +905,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {out.* = .{ .name = hdr.name, .value = hdr.value };}- const cached = cache.get(arena.allocator(), .{+ const cached = try cache.get(arena.allocator(), .{.url = req.url,.timestamp = lp.datetime.timestamp(.real),.request_headers = req_headers,diff --git a/src/network/Network.zig b/src/network/Network.zigindex 153011b9c..ceaf93f0d 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig");const CurlDebugAllocator = @import("CurlDebugAllocator.zig");const Cache = @import("cache/Cache.zig");-const FsCache = @import("cache/FsCache.zig");+const SqliteCache = @import("cache/SqliteCache.zig");const log = lp.log;const posix = std.posix;@@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {const cache = if (config.httpCacheDir()) |cache_dir_path|Cache{.kind = .{- .fs = FsCache.init(cache_dir_path) catch |e| {+ .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| {log.err(.cache, "failed to init", .{- .kind = "FsCache",+ .kind = "SqliteCache",.path = cache_dir_path,.err = e,});diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex ebe95ca19..159984f08 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -19,7 +19,7 @@const std = @import("std");const lp = @import("lightpanda");const Http = @import("../http.zig");-const FsCache = @import("FsCache.zig");+const SqliteCache = @import("SqliteCache.zig");const log = lp.log;@@ -28,7 +28,7 @@ const log = lp.log;pub const Cache = @This();kind: union(enum) {- fs: FsCache,+ sqlite: SqliteCache,},pub fn deinit(self: *Cache) void {@@ -37,7 +37,7 @@ pub fn deinit(self: *Cache) void {};}-pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {+pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {return switch (self.kind) {inline else => |*c| c.get(arena, req),};diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zignew file mode 100644index 000000000..c9cfbc9e5--- /dev/null+++ b/src/network/cache/SqliteCache.zig@@ -0,0 +1,833 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");+const lp = @import("lightpanda");++const Cache = @import("Cache.zig");++const log = lp.log;+const CacheRequest = Cache.CacheRequest;+const RenewResponse = Cache.RenewResponse;+const CachedMetadata = Cache.CachedMetadata;+const CachedResponse = Cache.CachedResponse;++const Http = @import("../http.zig");+const Pool = @import("../../storage/sqlite/Pool.zig");+const Conn = @import("../../storage/sqlite/Sqlite.zig").Conn;+const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration;+const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations;++pub const SqliteCache = @This();++allocator: std.mem.Allocator,+pool: Pool,++const cache_migrations: []const Migration = &.{+ .{ .sql =+ \\ create table metadata (+ \\ url text not null primary key,+ \\ status integer not null,+ \\ stored_at integer not null,+ \\ age_at_store integer not null,+ \\ max_age integer not null,+ \\ must_revalidate bool not null,+ \\ etag text,+ \\ last_modified text+ \\ )+ },+ .{ .sql =+ \\ create table body (+ \\ url text not null primary key,+ \\ data blob not null,+ \\ foreign key (url) references metadata(url) on delete cascade+ \\ )+ },+ .{ .sql =+ \\ create table header (+ \\ url text not null,+ \\ name text not null,+ \\ value text not null,+ \\ vary bool not null,+ \\ primary key (url, name),+ \\ foreign key (url) references metadata(url) on delete cascade+ \\ )+ },+ .{ .sql = "create index header_url on header(url)" },+};++pub const SqliteCachePath = union(enum) { path: []const u8, memory };++pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath) !SqliteCache {+ var pool = switch (path) {+ .memory => try Pool.init(allocator, ":memory:"),+ .path => |cache_dir| blk: {+ std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| {+ log.err(+ .cache,+ "failed to make path",+ .{ .kind = "httpCacheSqlitePath", .path = cache_dir, .err = e },+ );+ return e;+ };++ const full_path = try std.fmt.allocPrintSentinel(+ allocator,+ "{s}/cache.db",+ .{std.mem.trimEnd(u8, cache_dir, &.{'/'})},+ 0,+ );+ defer allocator.free(full_path);+ break :blk try Pool.init(allocator, full_path);+ },+ };+ errdefer pool.deinit(allocator);++ var version: usize = 0;++ {+ const conn = try pool.acquire();+ defer pool.release(conn);++ try conn.exec("pragma journal_mode=wal", .{});+ version = try Migrations.run(conn, cache_migrations);+ }++ for (pool.conns) |conn| {+ try conn.exec("pragma foreign_keys=on", .{});+ }++ log.info(.cache, "sqlite cache initialized", .{ .path = path, .version = version });+ return .{ .allocator = allocator, .pool = pool };+}++pub fn deinit(self: *SqliteCache) void {+ self.pool.deinit(self.allocator);+}++fn loadMetadata(conn: Conn, arena: std.mem.Allocator, url: []const u8) !?CachedMetadata {+ var entry = try conn.row(+ \\ select status, stored_at, age_at_store,+ \\ max_age, must_revalidate, etag, last_modified+ \\ from metadata+ \\ where url = $1+ , .{url}) orelse return null;+ defer entry.deinit();++ const status: u16 = @intCast(entry.get(i64, 0));+ const stored_at: u64 = @intCast(entry.get(i64, 1));+ const age_at_store = entry.get(i64, 2);+ const max_age: u64 = @intCast(entry.get(i64, 3));+ const must_revalidate = entry.get(bool, 4);+ const etag = if (entry.get(?[]const u8, 5)) |opt| try arena.dupe(u8, opt) else null;+ const last_modified = if (entry.get(?[]const u8, 6)) |opt| try arena.dupe(u8, opt) else null;++ var header_rows = try conn.rows(+ "select name, value, vary from header where url = $1",+ .{url},+ );+ defer header_rows.deinit();++ var headers: std.ArrayList(Http.Header) = .empty;+ var vary_headers: std.ArrayList(Http.Header) = .empty;+ var content_type: []const u8 = "application/octet-stream";++ while (try header_rows.next()) |row| {+ const name = try arena.dupe(u8, row.get([]const u8, 0));+ const value = try arena.dupe(u8, row.get([]const u8, 1));+ const vary = row.get(bool, 2);++ if (std.ascii.eqlIgnoreCase(name, "content-type")) {+ content_type = value;+ }++ const h = Http.Header{ .name = name, .value = value };+ if (vary) {+ try vary_headers.append(arena, h);+ } else {+ try headers.append(arena, h);+ }+ }++ return CachedMetadata{+ .url = try arena.dupeZ(u8, url),+ .content_type = content_type,+ .status = status,+ .stored_at = stored_at,+ .age_at_store = @intCast(age_at_store),+ .cache_control = .{+ .max_age = max_age,+ .must_revalidate = must_revalidate,+ },+ .headers = headers.items,+ .vary_headers = vary_headers.items,+ .etag = etag,+ .last_modified = last_modified,+ };+}++fn loadBody(conn: Conn, arena: std.mem.Allocator, url: []const u8) ![]const u8 {+ const body_entry = try conn.row(+ "select data from body where url = $1",+ .{url},+ ) orelse @panic("valid metadata must have a body");+ defer body_entry.deinit();++ return try arena.dupe(u8, body_entry.get([]const u8, 0));+}++fn insertMetadata(conn: Conn, meta: CachedMetadata, body: []const u8) !void {+ try conn.exec(+ \\ insert or replace into metadata+ \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified)+ \\ values ($1, $2, $3, $4, $5, $6, $7, $8)+ , .{+ meta.url,+ @as(i64, @intCast(meta.status)),+ meta.stored_at,+ @as(i64, @intCast(meta.age_at_store)),+ @as(i64, @intCast(meta.cache_control.max_age)),+ meta.cache_control.must_revalidate,+ meta.etag,+ meta.last_modified,+ });++ try conn.exec(+ "insert into body (url, data) values ($1, $2)",+ .{ meta.url, body },+ );++ var lower_name: [256]u8 = undefined;+ for (meta.headers) |h| {+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);+ try conn.exec(+ "insert into header (url, name, value, vary) values ($1, $2, $3, false)",+ .{ meta.url, name, h.value },+ );+ }+ for (meta.vary_headers) |h| {+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);+ try conn.exec(+ "insert into header (url, name, value, vary) values ($1, $2, $3, true)",+ .{ meta.url, name, h.value },+ );+ }+}++fn updateMetadata(conn: Conn, meta: CachedMetadata) !void {+ try conn.exec(+ \\ update metadata+ \\ set status = $1, stored_at = $2, age_at_store = $3, max_age = $4, must_revalidate = $5, etag = $6, last_modified = $7+ \\ where url = $8+ , .{+ @as(i64, @intCast(meta.status)),+ meta.stored_at,+ @as(i64, @intCast(meta.age_at_store)),+ @as(i64, @intCast(meta.cache_control.max_age)),+ meta.cache_control.must_revalidate,+ meta.etag,+ meta.last_modified,+ meta.url,+ });++ try conn.exec("delete from header where url = $1 and vary = false", .{meta.url});++ var lower_name: [256]u8 = undefined;+ for (meta.headers) |h| {+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);+ try conn.exec(+ "insert into header (url, name, value, vary) values ($1, $2, $3, false)",+ .{ meta.url, name, h.value },+ );+ }+}++pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {+ const conn = try self.pool.acquire();+ defer self.pool.release(conn);++ try conn.begin();+ defer conn.rollback() catch {};++ const metadata = try loadMetadata(conn, arena, req.url) orelse {+ log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });+ return null;+ };++ // Vary matching.+ for (metadata.vary_headers) |vary_hdr| {+ const incoming = for (req.request_headers) |h| {+ if (std.ascii.eqlIgnoreCase(h.name, vary_hdr.name)) break h.value;+ } else "";++ if (!std.ascii.eqlIgnoreCase(vary_hdr.value, incoming)) {+ log.debug(.cache, "miss", .{+ .url = req.url,+ .reason = "vary mismatch",+ .header = vary_hdr.name,+ .expected = vary_hdr.value,+ .got = incoming,+ });+ return null;+ }+ }++ const body = try loadBody(conn, arena, req.url);+ const expired = metadata.isStale(req.timestamp);+ log.debug(.cache, "hit", .{ .url = req.url, .expired = expired });++ return .{+ .metadata = metadata,+ .data = .{ .buffer = body },+ .expired = expired,+ };+}++pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void {+ const conn = try self.pool.acquire();+ defer self.pool.release(conn);++ try conn.begin();+ errdefer conn.rollback() catch {};++ try insertMetadata(conn, meta, body);+ try conn.commit();++ log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len });+}++pub fn clear(self: *SqliteCache) !void {+ const conn = try self.pool.acquire();+ defer self.pool.release(conn);++ try conn.exec("delete from metadata", .{});+ log.debug(.cache, "clear", .{});+}++pub fn evict(self: *SqliteCache, url: []const u8) void {+ const conn = self.pool.acquire() catch |err| {+ log.err(.cache, "sqlite acquire", .{ .url = url, .err = err });+ return;+ };+ defer self.pool.release(conn);++ conn.exec("delete from metadata where url = $1", .{url}) catch |err| {+ log.err(.cache, "delete from cache", .{ .url = url, .err = err });+ return;+ };++ log.debug(.cache, "evict", .{ .url = url });+}++pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void {+ const conn = try self.pool.acquire();+ defer self.pool.release(conn);++ try conn.begin();+ errdefer conn.rollback() catch {};++ var metadata = try loadMetadata(conn, arena, req.url) orelse {+ log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });+ return error.CacheEntryNotFound;+ };++ metadata.renew(req);++ try updateMetadata(conn, metadata);+ try conn.commit();++ log.debug(.cache, "renewed", .{+ .url = req.url,+ .timestamp = req.timestamp,+ });+}++const testing = std.testing;++fn setupCache(allocator: std.mem.Allocator) !Cache {+ return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory) } };+}++test "SqliteCache: Migrations" {+ const allocator = testing.allocator;+ var pool = try Pool.init(allocator, ":memory:");+ defer pool.deinit(allocator);++ const conn = try pool.acquire();+ defer pool.release(conn);++ _ = try Migrations.run(conn, cache_migrations);+}++test "SqliteCache: basic put and get" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600, .must_revalidate = false },+ .headers = &.{.{ .name = "Content-Type", .value = "text/html" }},+ .vary_headers = &.{},+ };++ try cache.put(meta, "hello world");++ const result = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;++ try testing.expectEqualStrings("hello world", result.data.buffer);+ try testing.expectEqual(@as(u16, 200), result.metadata.status);+ try testing.expectEqual(false, result.expired);+ try testing.expectEqualStrings("text/html", result.metadata.content_type);+}++test "SqliteCache: get expiration" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now = 5000;+ const max_age = 1000;++ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 900,+ .cache_control = .{ .max_age = max_age },+ .headers = &.{},+ .vary_headers = &.{},+ };++ try cache.put(meta, "hello world");++ // age = 50 + 900 = 950 < 1000: fresh+ const fresh = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 50,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqual(false, fresh.expired);++ // age = 200 + 900 = 1100 >= 1000: stale+ const stale = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 200,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqual(true, stale.expired);+}++test "SqliteCache: put override" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ {+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = 5000,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 1000 },+ .headers = &.{},+ .vary_headers = &.{},+ };+ try cache.put(meta, "hello world");++ const result = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = 5000,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqualStrings("hello world", result.data.buffer);+ }++ {+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = 10000,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 2000 },+ .headers = &.{},+ .vary_headers = &.{},+ };+ try cache.put(meta, "goodbye world");++ const result = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = 10000,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqualStrings("goodbye world", result.data.buffer);+ }+}++test "SqliteCache: vary hit and miss" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{+ .{ .name = "Accept-Encoding", .value = "gzip" },+ },+ };++ try cache.put(meta, "hello world");++ const hit = try cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},+ }) orelse return error.CacheMiss;+ try testing.expectEqualStrings("hello world", hit.data.buffer);++ try testing.expectEqual(null, try cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }},+ }));++ try testing.expectEqual(null, try cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ }));+}++test "SqliteCache: vary multiple headers" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{+ .{ .name = "Accept-Encoding", .value = "gzip" },+ .{ .name = "Accept-Language", .value = "en" },+ },+ };++ try cache.put(meta, "hello world");++ const hit = try cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{+ .{ .name = "Accept-Encoding", .value = "gzip" },+ .{ .name = "Accept-Language", .value = "en" },+ },+ }) orelse return error.CacheMiss;+ try testing.expectEqualStrings("hello world", hit.data.buffer);++ try testing.expectEqual(null, try cache.get(arena.allocator(), .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{+ .{ .name = "Accept-Encoding", .value = "gzip" },+ .{ .name = "Accept-Language", .value = "fr" },+ },+ }));+}++test "SqliteCache: clear removes all entries" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ try cache.put(.{+ .url = "https://example.com/a",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "body a");++ try cache.put(.{+ .url = "https://example.com/b",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "body b");++ try testing.expect(null != try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com/a",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));+ try testing.expect(null != try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com/b",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));++ try cache.clear();++ try testing.expectEqual(null, try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com/a",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));+ try testing.expectEqual(null, try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com/b",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));+}++test "SqliteCache: put after clear works" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ };++ try cache.put(meta, "before clear");+ try cache.clear();++ try testing.expectEqual(null, try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));++ try cache.put(meta, "after clear");+ const result = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqualStrings("after clear", result.data.buffer);+}++test "SqliteCache: evict removes entry" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ const meta = CachedMetadata{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ };++ try cache.put(meta, "hello world");++ _ = try cache.get(+ arena.allocator(),+ .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} },+ ) orelse return error.CacheMiss;++ cache.evict("https://example.com");++ try testing.expectEqual(null, try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ },+ ));+}++test "SqliteCache: renew refreshes expiry" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: i64 = 5000;+ try cache.put(.{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 1000 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "hello world");++ try cache.renew(+ arena.allocator(),+ .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} },+ );++ // Clock reset to now+500, so still fresh at now+1200+ const fresh = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 1200,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqual(false, fresh.expired);++ // Expires at now+500+1000 = now+1500+ const stale = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 1500,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqual(true, stale.expired);+}++test "SqliteCache: renew preserves body" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ var arena = std.heap.ArenaAllocator.init(testing.allocator);+ defer arena.deinit();++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ try cache.put(.{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{},+ .vary_headers = &.{},+ }, "original body");++ try cache.renew(+ arena.allocator(),+ .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} },+ );++ const result = try cache.get(+ arena.allocator(),+ .{+ .url = "https://example.com",+ .timestamp = now + 100,+ .request_headers = &.{},+ },+ ) orelse return error.CacheMiss;+ try testing.expectEqualStrings("original body", result.data.buffer);+}diff --git a/src/storage/Storage.zig b/src/storage/Storage.zigindex 54cefdaf7..54ac21f3d 100644--- a/src/storage/Storage.zig+++ b/src/storage/Storage.zig@@ -28,12 +28,10 @@ const Storage = @This();pub const EngineType = enum {none,- sqlite,};const Engine = union(EngineType) {none: Blackhole,- sqlite: Sqlite,};engine: Engine,@@ -50,13 +48,9 @@ pub fn init(allocator: Allocator, config: *const Config) !Storage {};}-fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine {+fn initEngine(_: Allocator, engine_type: EngineType, _: *const Config) !Engine {switch (engine_type) {.none => return .{ .none = Blackhole{} },- .sqlite => {- const sqlite_path = config.storageSqlitePath();- return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) };- },}}diff --git a/src/storage/sqlite/Sqlite.zig b/src/storage/sqlite/Sqlite.zigindex 6a8098fac..f06d5f63b 100644--- a/src/storage/sqlite/Sqlite.zig+++ b/src/storage/sqlite/Sqlite.zig@@ -27,32 +27,60 @@ const Allocator = std.mem.Allocator;const Sqlite = @This();-pool: Pool,+pub const Migration = union(enum) {+ sql: [:0]const u8,+ func: struct {+ ctx: *anyopaque,+ func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void,+ },+};-pub fn init(allocator: Allocator, path_: ?[:0]const u8) !Sqlite {- const path = path_ orelse ":memory:";- var pool = try Pool.init(allocator, path);- errdefer pool.deinit(allocator);+pub const Migrations = struct {+ pub fn run(conn: Conn, migrations: []const Migration) !usize {+ try conn.exec(+ \\create table if not exists migrations (+ \\ id integer primary key,+ \\ applied_at integer not null+ \\) strict+ , .{});++ const current = (try conn.scalar(+ i64,+ "select max(id) from migrations",+ .{},+ )) orelse 0;+ const start: usize = @intCast(current);++ if (start > migrations.len) {+ log.err(.storage, "migrations removed", .{+ .applied = start,+ .defined = migrations.len,+ });+ return error.MigrationsRemoved;+ }- {- // copy by value warning! The connection HAS to be returned to the- // pool in this scope. If we didn't have this scope, we'd assign the- // pool to the return value (copy A) and then release the original- const conn = try pool.acquire();- defer pool.release(conn);-- const version = try @import("migrations.zig").run(conn);- log.info(.storage, "storage initialized", .{ .engine = "sqlite", .version = version, .path = path });- }+ if (start == migrations.len) {+ return start;+ }- return .{- .pool = pool,- };-}+ try conn.begin();+ errdefer conn.rollback() catch {};-pub fn deinit(self: *Sqlite, allocator: Allocator) void {- self.pool.deinit(allocator);-}+ for (migrations[start..], start..) |migration, i| {+ switch (migration) {+ .sql => |sql| try conn.exec(sql, .{}),+ .func => |f| try f.func(conn, f.ctx),+ }+ try conn.exec(+ "insert into migrations (id, applied_at) values ($1, $2)",+ .{ @as(i64, @intCast(i + 1)), std.Io.Timestamp.now(lp.io, .boot).toMilliseconds() },+ );+ }++ try conn.commit();+ return migrations.len;+ }+};pub const Conn = struct {conn: *c.sqlite3,@@ -138,6 +166,18 @@ pub const Conn = struct {return .{ .stmt = stmt.?, .conn = self.conn };}+ pub fn begin(self: Conn) !void {+ try self.exec("begin", .{});+ }++ pub fn commit(self: Conn) !void {+ try self.exec("commit", .{});+ }++ pub fn rollback(self: Conn) !void {+ try self.exec("rollback", .{});+ }+pub fn busyTimeout(self: Conn, ms: c_int) !void {const rc = c.sqlite3_busy_timeout(self.conn, ms);if (rc != c.SQLITE_OK) {@@ -568,12 +608,42 @@ test "Sqlite: exec, row and scalar" {}}-test "Sqlite: Migration" {- var sqlite = try Sqlite.init(testing.allocator, ":memory:");- defer sqlite.deinit(testing.allocator);+test "Sqlite: Migrations - basic" {+ var conn = try Sqlite.Conn.open(":memory:");+ defer conn.close();++ const migrations: []const Migration = &.{+ .{ .sql = "create table test (id integer primary key, name text)" },+ .{ .sql = "alter table test add column email text" },+ };++ const v1 = try Migrations.run(conn, migrations);+ try testing.expectEqual(@as(usize, 2), v1);++ // idempotent - running again should return same version+ const v2 = try Migrations.run(conn, migrations);+ try testing.expectEqual(@as(usize, 2), v2);++ // verify migrations table has correct entries+ try testing.expectEqual(+ @as(i64, 2),+ (try conn.scalar(i64, "select count(*) from migrations", .{})).?,+ );+}++test "Sqlite: Migrations - removed migration" {+ var conn = try Sqlite.Conn.open(":memory:");+ defer conn.close();- const conn = try sqlite.pool.acquire();- defer sqlite.pool.release(conn);+ const m1: []const Migration = &.{+ .{ .sql = "create table test (id integer primary key, name text)" },+ .{ .sql = "alter table test add column email text" },+ };+ _ = try Migrations.run(conn, m1);- try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?);+ // fewer migrations than were applied+ const m2: []const Migration = &.{+ .{ .sql = "create table test (id integer primary key, name text)" },+ };+ try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2));}diff --git a/src/storage/sqlite/migrations.zig b/src/storage/sqlite/migrations.zigdeleted file mode 100644index e431c443f..000000000--- a/src/storage/sqlite/migrations.zig+++ /dev/null@@ -1,62 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const lp = @import("lightpanda");--const Sqlite = @import("Sqlite.zig");--const log = lp.log;--pub fn run(conn: Sqlite.Conn) !i64 {- const version = try getVersion(conn);- return version;-}--fn getVersion(conn: Sqlite.Conn) !i64 {- const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')";- if (try conn.scalar(bool, exists_sql, .{}) orelse false) {- if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| {- return version;- }-- log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" });- return error.CorruptDatabase;- }-- // this pragma is one of the the few (if not only) one that's persisted, so- // we only have to do it the first time.- conn.exec("pragma journal_mode=wal", .{}) catch |err| {- log.fatal(.storage, "migrate", .{- .err = err,- .step = "journal_mode",- .sqlite = conn.lastError(),- });- return err;- };-- const create_sql =- \\ create table migrations as- \\ select 1 as id, current_timestamp as created_at- ;- conn.exec(create_sql, .{}) catch |err| {- log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" });- return err;- };-- return 1;-}