Skip to content

Lightpanda: the headless browser designed for AI and automation

add SqliteCache as default local impl

Muki Kiboigo0f2957cda2a3
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 57da9a0f6..89d020e7b 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -905,7 +905,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
out.* = .{ .name = hdr.name, .value = hdr.value };
}
- const cached = cache.get(arena.allocator(), .{
+ const cached = try cache.get(arena.allocator(), .{
.url = req.url,
.timestamp = lp.datetime.timestamp(.real),
.request_headers = req_headers,
diff --git a/src/network/Network.zig b/src/network/Network.zig
index 153011b9c..ceaf93f0d 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig");
const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
const Cache = @import("cache/Cache.zig");
-const FsCache = @import("cache/FsCache.zig");
+const SqliteCache = @import("cache/SqliteCache.zig");
const log = lp.log;
const posix = std.posix;
@@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
const cache = if (config.httpCacheDir()) |cache_dir_path|
Cache{
.kind = .{
- .fs = FsCache.init(cache_dir_path) catch |e| {
+ .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| {
log.err(.cache, "failed to init", .{
- .kind = "FsCache",
+ .kind = "SqliteCache",
.path = cache_dir_path,
.err = e,
});
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index ebe95ca19..159984f08 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -19,7 +19,7 @@
const std = @import("std");
const lp = @import("lightpanda");
const Http = @import("../http.zig");
-const FsCache = @import("FsCache.zig");
+const SqliteCache = @import("SqliteCache.zig");
const log = lp.log;
@@ -28,7 +28,7 @@ const log = lp.log;
pub const Cache = @This();
kind: union(enum) {
- fs: FsCache,
+ sqlite: SqliteCache,
},
pub fn deinit(self: *Cache) void {
@@ -37,7 +37,7 @@ pub fn deinit(self: *Cache) void {
};
}
-pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {
+pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {
return switch (self.kind) {
inline else => |*c| c.get(arena, req),
};
diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig
new file mode 100644
index 000000000..c9cfbc9e5
--- /dev/null
+++ b/src/network/cache/SqliteCache.zig
@@ -0,0 +1,833 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const Cache = @import("Cache.zig");
+
+const log = lp.log;
+const CacheRequest = Cache.CacheRequest;
+const RenewResponse = Cache.RenewResponse;
+const CachedMetadata = Cache.CachedMetadata;
+const CachedResponse = Cache.CachedResponse;
+
+const Http = @import("../http.zig");
+const Pool = @import("../../storage/sqlite/Pool.zig");
+const Conn = @import("../../storage/sqlite/Sqlite.zig").Conn;
+const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration;
+const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations;
+
+pub const SqliteCache = @This();
+
+allocator: std.mem.Allocator,
+pool: Pool,
+
+const cache_migrations: []const Migration = &.{
+ .{ .sql =
+ \\ create table metadata (
+ \\ url text not null primary key,
+ \\ status integer not null,
+ \\ stored_at integer not null,
+ \\ age_at_store integer not null,
+ \\ max_age integer not null,
+ \\ must_revalidate bool not null,
+ \\ etag text,
+ \\ last_modified text
+ \\ )
+ },
+ .{ .sql =
+ \\ create table body (
+ \\ url text not null primary key,
+ \\ data blob not null,
+ \\ foreign key (url) references metadata(url) on delete cascade
+ \\ )
+ },
+ .{ .sql =
+ \\ create table header (
+ \\ url text not null,
+ \\ name text not null,
+ \\ value text not null,
+ \\ vary bool not null,
+ \\ primary key (url, name),
+ \\ foreign key (url) references metadata(url) on delete cascade
+ \\ )
+ },
+ .{ .sql = "create index header_url on header(url)" },
+};
+
+pub const SqliteCachePath = union(enum) { path: []const u8, memory };
+
+pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath) !SqliteCache {
+ var pool = switch (path) {
+ .memory => try Pool.init(allocator, ":memory:"),
+ .path => |cache_dir| blk: {
+ std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| {
+ log.err(
+ .cache,
+ "failed to make path",
+ .{ .kind = "httpCacheSqlitePath", .path = cache_dir, .err = e },
+ );
+ return e;
+ };
+
+ const full_path = try std.fmt.allocPrintSentinel(
+ allocator,
+ "{s}/cache.db",
+ .{std.mem.trimEnd(u8, cache_dir, &.{'/'})},
+ 0,
+ );
+ defer allocator.free(full_path);
+ break :blk try Pool.init(allocator, full_path);
+ },
+ };
+ errdefer pool.deinit(allocator);
+
+ var version: usize = 0;
+
+ {
+ const conn = try pool.acquire();
+ defer pool.release(conn);
+
+ try conn.exec("pragma journal_mode=wal", .{});
+ version = try Migrations.run(conn, cache_migrations);
+ }
+
+ for (pool.conns) |conn| {
+ try conn.exec("pragma foreign_keys=on", .{});
+ }
+
+ log.info(.cache, "sqlite cache initialized", .{ .path = path, .version = version });
+ return .{ .allocator = allocator, .pool = pool };
+}
+
+pub fn deinit(self: *SqliteCache) void {
+ self.pool.deinit(self.allocator);
+}
+
+fn loadMetadata(conn: Conn, arena: std.mem.Allocator, url: []const u8) !?CachedMetadata {
+ var entry = try conn.row(
+ \\ select status, stored_at, age_at_store,
+ \\ max_age, must_revalidate, etag, last_modified
+ \\ from metadata
+ \\ where url = $1
+ , .{url}) orelse return null;
+ defer entry.deinit();
+
+ const status: u16 = @intCast(entry.get(i64, 0));
+ const stored_at: u64 = @intCast(entry.get(i64, 1));
+ const age_at_store = entry.get(i64, 2);
+ const max_age: u64 = @intCast(entry.get(i64, 3));
+ const must_revalidate = entry.get(bool, 4);
+ const etag = if (entry.get(?[]const u8, 5)) |opt| try arena.dupe(u8, opt) else null;
+ const last_modified = if (entry.get(?[]const u8, 6)) |opt| try arena.dupe(u8, opt) else null;
+
+ var header_rows = try conn.rows(
+ "select name, value, vary from header where url = $1",
+ .{url},
+ );
+ defer header_rows.deinit();
+
+ var headers: std.ArrayList(Http.Header) = .empty;
+ var vary_headers: std.ArrayList(Http.Header) = .empty;
+ var content_type: []const u8 = "application/octet-stream";
+
+ while (try header_rows.next()) |row| {
+ const name = try arena.dupe(u8, row.get([]const u8, 0));
+ const value = try arena.dupe(u8, row.get([]const u8, 1));
+ const vary = row.get(bool, 2);
+
+ if (std.ascii.eqlIgnoreCase(name, "content-type")) {
+ content_type = value;
+ }
+
+ const h = Http.Header{ .name = name, .value = value };
+ if (vary) {
+ try vary_headers.append(arena, h);
+ } else {
+ try headers.append(arena, h);
+ }
+ }
+
+ return CachedMetadata{
+ .url = try arena.dupeZ(u8, url),
+ .content_type = content_type,
+ .status = status,
+ .stored_at = stored_at,
+ .age_at_store = @intCast(age_at_store),
+ .cache_control = .{
+ .max_age = max_age,
+ .must_revalidate = must_revalidate,
+ },
+ .headers = headers.items,
+ .vary_headers = vary_headers.items,
+ .etag = etag,
+ .last_modified = last_modified,
+ };
+}
+
+fn loadBody(conn: Conn, arena: std.mem.Allocator, url: []const u8) ![]const u8 {
+ const body_entry = try conn.row(
+ "select data from body where url = $1",
+ .{url},
+ ) orelse @panic("valid metadata must have a body");
+ defer body_entry.deinit();
+
+ return try arena.dupe(u8, body_entry.get([]const u8, 0));
+}
+
+fn insertMetadata(conn: Conn, meta: CachedMetadata, body: []const u8) !void {
+ try conn.exec(
+ \\ insert or replace into metadata
+ \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified)
+ \\ values ($1, $2, $3, $4, $5, $6, $7, $8)
+ , .{
+ meta.url,
+ @as(i64, @intCast(meta.status)),
+ meta.stored_at,
+ @as(i64, @intCast(meta.age_at_store)),
+ @as(i64, @intCast(meta.cache_control.max_age)),
+ meta.cache_control.must_revalidate,
+ meta.etag,
+ meta.last_modified,
+ });
+
+ try conn.exec(
+ "insert into body (url, data) values ($1, $2)",
+ .{ meta.url, body },
+ );
+
+ var lower_name: [256]u8 = undefined;
+ for (meta.headers) |h| {
+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+ try conn.exec(
+ "insert into header (url, name, value, vary) values ($1, $2, $3, false)",
+ .{ meta.url, name, h.value },
+ );
+ }
+ for (meta.vary_headers) |h| {
+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+ try conn.exec(
+ "insert into header (url, name, value, vary) values ($1, $2, $3, true)",
+ .{ meta.url, name, h.value },
+ );
+ }
+}
+
+fn updateMetadata(conn: Conn, meta: CachedMetadata) !void {
+ try conn.exec(
+ \\ update metadata
+ \\ set status = $1, stored_at = $2, age_at_store = $3, max_age = $4, must_revalidate = $5, etag = $6, last_modified = $7
+ \\ where url = $8
+ , .{
+ @as(i64, @intCast(meta.status)),
+ meta.stored_at,
+ @as(i64, @intCast(meta.age_at_store)),
+ @as(i64, @intCast(meta.cache_control.max_age)),
+ meta.cache_control.must_revalidate,
+ meta.etag,
+ meta.last_modified,
+ meta.url,
+ });
+
+ try conn.exec("delete from header where url = $1 and vary = false", .{meta.url});
+
+ var lower_name: [256]u8 = undefined;
+ for (meta.headers) |h| {
+ if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+ const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+ try conn.exec(
+ "insert into header (url, name, value, vary) values ($1, $2, $3, false)",
+ .{ meta.url, name, h.value },
+ );
+ }
+}
+
+pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {
+ const conn = try self.pool.acquire();
+ defer self.pool.release(conn);
+
+ try conn.begin();
+ defer conn.rollback() catch {};
+
+ const metadata = try loadMetadata(conn, arena, req.url) orelse {
+ log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
+ return null;
+ };
+
+ // Vary matching.
+ for (metadata.vary_headers) |vary_hdr| {
+ const incoming = for (req.request_headers) |h| {
+ if (std.ascii.eqlIgnoreCase(h.name, vary_hdr.name)) break h.value;
+ } else "";
+
+ if (!std.ascii.eqlIgnoreCase(vary_hdr.value, incoming)) {
+ log.debug(.cache, "miss", .{
+ .url = req.url,
+ .reason = "vary mismatch",
+ .header = vary_hdr.name,
+ .expected = vary_hdr.value,
+ .got = incoming,
+ });
+ return null;
+ }
+ }
+
+ const body = try loadBody(conn, arena, req.url);
+ const expired = metadata.isStale(req.timestamp);
+ log.debug(.cache, "hit", .{ .url = req.url, .expired = expired });
+
+ return .{
+ .metadata = metadata,
+ .data = .{ .buffer = body },
+ .expired = expired,
+ };
+}
+
+pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void {
+ const conn = try self.pool.acquire();
+ defer self.pool.release(conn);
+
+ try conn.begin();
+ errdefer conn.rollback() catch {};
+
+ try insertMetadata(conn, meta, body);
+ try conn.commit();
+
+ log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len });
+}
+
+pub fn clear(self: *SqliteCache) !void {
+ const conn = try self.pool.acquire();
+ defer self.pool.release(conn);
+
+ try conn.exec("delete from metadata", .{});
+ log.debug(.cache, "clear", .{});
+}
+
+pub fn evict(self: *SqliteCache, url: []const u8) void {
+ const conn = self.pool.acquire() catch |err| {
+ log.err(.cache, "sqlite acquire", .{ .url = url, .err = err });
+ return;
+ };
+ defer self.pool.release(conn);
+
+ conn.exec("delete from metadata where url = $1", .{url}) catch |err| {
+ log.err(.cache, "delete from cache", .{ .url = url, .err = err });
+ return;
+ };
+
+ log.debug(.cache, "evict", .{ .url = url });
+}
+
+pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void {
+ const conn = try self.pool.acquire();
+ defer self.pool.release(conn);
+
+ try conn.begin();
+ errdefer conn.rollback() catch {};
+
+ var metadata = try loadMetadata(conn, arena, req.url) orelse {
+ log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
+ return error.CacheEntryNotFound;
+ };
+
+ metadata.renew(req);
+
+ try updateMetadata(conn, metadata);
+ try conn.commit();
+
+ log.debug(.cache, "renewed", .{
+ .url = req.url,
+ .timestamp = req.timestamp,
+ });
+}
+
+const testing = std.testing;
+
+fn setupCache(allocator: std.mem.Allocator) !Cache {
+ return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory) } };
+}
+
+test "SqliteCache: Migrations" {
+ const allocator = testing.allocator;
+ var pool = try Pool.init(allocator, ":memory:");
+ defer pool.deinit(allocator);
+
+ const conn = try pool.acquire();
+ defer pool.release(conn);
+
+ _ = try Migrations.run(conn, cache_migrations);
+}
+
+test "SqliteCache: basic put and get" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600, .must_revalidate = false },
+ .headers = &.{.{ .name = "Content-Type", .value = "text/html" }},
+ .vary_headers = &.{},
+ };
+
+ try cache.put(meta, "hello world");
+
+ const result = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+
+ try testing.expectEqualStrings("hello world", result.data.buffer);
+ try testing.expectEqual(@as(u16, 200), result.metadata.status);
+ try testing.expectEqual(false, result.expired);
+ try testing.expectEqualStrings("text/html", result.metadata.content_type);
+}
+
+test "SqliteCache: get expiration" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now = 5000;
+ const max_age = 1000;
+
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 900,
+ .cache_control = .{ .max_age = max_age },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+
+ try cache.put(meta, "hello world");
+
+ // age = 50 + 900 = 950 < 1000: fresh
+ const fresh = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 50,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqual(false, fresh.expired);
+
+ // age = 200 + 900 = 1100 >= 1000: stale
+ const stale = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 200,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqual(true, stale.expired);
+}
+
+test "SqliteCache: put override" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ {
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = 5000,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 1000 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+ try cache.put(meta, "hello world");
+
+ const result = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = 5000,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("hello world", result.data.buffer);
+ }
+
+ {
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = 10000,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 2000 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+ try cache.put(meta, "goodbye world");
+
+ const result = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = 10000,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("goodbye world", result.data.buffer);
+ }
+}
+
+test "SqliteCache: vary hit and miss" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{
+ .{ .name = "Accept-Encoding", .value = "gzip" },
+ },
+ };
+
+ try cache.put(meta, "hello world");
+
+ const hit = try cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},
+ }) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("hello world", hit.data.buffer);
+
+ try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }},
+ }));
+
+ try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{},
+ }));
+}
+
+test "SqliteCache: vary multiple headers" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{
+ .{ .name = "Accept-Encoding", .value = "gzip" },
+ .{ .name = "Accept-Language", .value = "en" },
+ },
+ };
+
+ try cache.put(meta, "hello world");
+
+ const hit = try cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{
+ .{ .name = "Accept-Encoding", .value = "gzip" },
+ .{ .name = "Accept-Language", .value = "en" },
+ },
+ }) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("hello world", hit.data.buffer);
+
+ try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{
+ .{ .name = "Accept-Encoding", .value = "gzip" },
+ .{ .name = "Accept-Language", .value = "fr" },
+ },
+ }));
+}
+
+test "SqliteCache: clear removes all entries" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ try cache.put(.{
+ .url = "https://example.com/a",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ }, "body a");
+
+ try cache.put(.{
+ .url = "https://example.com/b",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ }, "body b");
+
+ try testing.expect(null != try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com/a",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+ try testing.expect(null != try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com/b",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+
+ try cache.clear();
+
+ try testing.expectEqual(null, try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com/a",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+ try testing.expectEqual(null, try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com/b",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+}
+
+test "SqliteCache: put after clear works" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+
+ try cache.put(meta, "before clear");
+ try cache.clear();
+
+ try testing.expectEqual(null, try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+
+ try cache.put(meta, "after clear");
+ const result = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("after clear", result.data.buffer);
+}
+
+test "SqliteCache: evict removes entry" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ const meta = CachedMetadata{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ };
+
+ try cache.put(meta, "hello world");
+
+ _ = try cache.get(
+ arena.allocator(),
+ .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} },
+ ) orelse return error.CacheMiss;
+
+ cache.evict("https://example.com");
+
+ try testing.expectEqual(null, try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now,
+ .request_headers = &.{},
+ },
+ ));
+}
+
+test "SqliteCache: renew refreshes expiry" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: i64 = 5000;
+ try cache.put(.{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 1000 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ }, "hello world");
+
+ try cache.renew(
+ arena.allocator(),
+ .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} },
+ );
+
+ // Clock reset to now+500, so still fresh at now+1200
+ const fresh = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 1200,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqual(false, fresh.expired);
+
+ // Expires at now+500+1000 = now+1500
+ const stale = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 1500,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqual(true, stale.expired);
+}
+
+test "SqliteCache: renew preserves body" {
+ var cache = try setupCache(testing.allocator);
+ defer cache.deinit();
+
+ var arena = std.heap.ArenaAllocator.init(testing.allocator);
+ defer arena.deinit();
+
+ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+ try cache.put(.{
+ .url = "https://example.com",
+ .content_type = "text/html",
+ .status = 200,
+ .stored_at = now,
+ .age_at_store = 0,
+ .cache_control = .{ .max_age = 600 },
+ .headers = &.{},
+ .vary_headers = &.{},
+ }, "original body");
+
+ try cache.renew(
+ arena.allocator(),
+ .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} },
+ );
+
+ const result = try cache.get(
+ arena.allocator(),
+ .{
+ .url = "https://example.com",
+ .timestamp = now + 100,
+ .request_headers = &.{},
+ },
+ ) orelse return error.CacheMiss;
+ try testing.expectEqualStrings("original body", result.data.buffer);
+}
diff --git a/src/storage/Storage.zig b/src/storage/Storage.zig
index 54cefdaf7..54ac21f3d 100644
--- a/src/storage/Storage.zig
+++ b/src/storage/Storage.zig
@@ -28,12 +28,10 @@ const Storage = @This();
pub const EngineType = enum {
none,
- sqlite,
};
const Engine = union(EngineType) {
none: Blackhole,
- sqlite: Sqlite,
};
engine: Engine,
@@ -50,13 +48,9 @@ pub fn init(allocator: Allocator, config: *const Config) !Storage {
};
}
-fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine {
+fn initEngine(_: Allocator, engine_type: EngineType, _: *const Config) !Engine {
switch (engine_type) {
.none => return .{ .none = Blackhole{} },
- .sqlite => {
- const sqlite_path = config.storageSqlitePath();
- return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) };
- },
}
}
diff --git a/src/storage/sqlite/Sqlite.zig b/src/storage/sqlite/Sqlite.zig
index 6a8098fac..f06d5f63b 100644
--- a/src/storage/sqlite/Sqlite.zig
+++ b/src/storage/sqlite/Sqlite.zig
@@ -27,32 +27,60 @@ const Allocator = std.mem.Allocator;
const Sqlite = @This();
-pool: Pool,
+pub const Migration = union(enum) {
+ sql: [:0]const u8,
+ func: struct {
+ ctx: *anyopaque,
+ func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void,
+ },
+};
-pub fn init(allocator: Allocator, path_: ?[:0]const u8) !Sqlite {
- const path = path_ orelse ":memory:";
- var pool = try Pool.init(allocator, path);
- errdefer pool.deinit(allocator);
+pub const Migrations = struct {
+ pub fn run(conn: Conn, migrations: []const Migration) !usize {
+ try conn.exec(
+ \\create table if not exists migrations (
+ \\ id integer primary key,
+ \\ applied_at integer not null
+ \\) strict
+ , .{});
+
+ const current = (try conn.scalar(
+ i64,
+ "select max(id) from migrations",
+ .{},
+ )) orelse 0;
+ const start: usize = @intCast(current);
+
+ if (start > migrations.len) {
+ log.err(.storage, "migrations removed", .{
+ .applied = start,
+ .defined = migrations.len,
+ });
+ return error.MigrationsRemoved;
+ }
- {
- // copy by value warning! The connection HAS to be returned to the
- // pool in this scope. If we didn't have this scope, we'd assign the
- // pool to the return value (copy A) and then release the original
- const conn = try pool.acquire();
- defer pool.release(conn);
-
- const version = try @import("migrations.zig").run(conn);
- log.info(.storage, "storage initialized", .{ .engine = "sqlite", .version = version, .path = path });
- }
+ if (start == migrations.len) {
+ return start;
+ }
- return .{
- .pool = pool,
- };
-}
+ try conn.begin();
+ errdefer conn.rollback() catch {};
-pub fn deinit(self: *Sqlite, allocator: Allocator) void {
- self.pool.deinit(allocator);
-}
+ for (migrations[start..], start..) |migration, i| {
+ switch (migration) {
+ .sql => |sql| try conn.exec(sql, .{}),
+ .func => |f| try f.func(conn, f.ctx),
+ }
+ try conn.exec(
+ "insert into migrations (id, applied_at) values ($1, $2)",
+ .{ @as(i64, @intCast(i + 1)), std.Io.Timestamp.now(lp.io, .boot).toMilliseconds() },
+ );
+ }
+
+ try conn.commit();
+ return migrations.len;
+ }
+};
pub const Conn = struct {
conn: *c.sqlite3,
@@ -138,6 +166,18 @@ pub const Conn = struct {
return .{ .stmt = stmt.?, .conn = self.conn };
}
+ pub fn begin(self: Conn) !void {
+ try self.exec("begin", .{});
+ }
+
+ pub fn commit(self: Conn) !void {
+ try self.exec("commit", .{});
+ }
+
+ pub fn rollback(self: Conn) !void {
+ try self.exec("rollback", .{});
+ }
+
pub fn busyTimeout(self: Conn, ms: c_int) !void {
const rc = c.sqlite3_busy_timeout(self.conn, ms);
if (rc != c.SQLITE_OK) {
@@ -568,12 +608,42 @@ test "Sqlite: exec, row and scalar" {
}
}
-test "Sqlite: Migration" {
- var sqlite = try Sqlite.init(testing.allocator, ":memory:");
- defer sqlite.deinit(testing.allocator);
+test "Sqlite: Migrations - basic" {
+ var conn = try Sqlite.Conn.open(":memory:");
+ defer conn.close();
+
+ const migrations: []const Migration = &.{
+ .{ .sql = "create table test (id integer primary key, name text)" },
+ .{ .sql = "alter table test add column email text" },
+ };
+
+ const v1 = try Migrations.run(conn, migrations);
+ try testing.expectEqual(@as(usize, 2), v1);
+
+ // idempotent - running again should return same version
+ const v2 = try Migrations.run(conn, migrations);
+ try testing.expectEqual(@as(usize, 2), v2);
+
+ // verify migrations table has correct entries
+ try testing.expectEqual(
+ @as(i64, 2),
+ (try conn.scalar(i64, "select count(*) from migrations", .{})).?,
+ );
+}
+
+test "Sqlite: Migrations - removed migration" {
+ var conn = try Sqlite.Conn.open(":memory:");
+ defer conn.close();
- const conn = try sqlite.pool.acquire();
- defer sqlite.pool.release(conn);
+ const m1: []const Migration = &.{
+ .{ .sql = "create table test (id integer primary key, name text)" },
+ .{ .sql = "alter table test add column email text" },
+ };
+ _ = try Migrations.run(conn, m1);
- try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?);
+ // fewer migrations than were applied
+ const m2: []const Migration = &.{
+ .{ .sql = "create table test (id integer primary key, name text)" },
+ };
+ try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2));
}
diff --git a/src/storage/sqlite/migrations.zig b/src/storage/sqlite/migrations.zig
deleted file mode 100644
index e431c443f..000000000
--- a/src/storage/sqlite/migrations.zig
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const lp = @import("lightpanda");
-
-const Sqlite = @import("Sqlite.zig");
-
-const log = lp.log;
-
-pub fn run(conn: Sqlite.Conn) !i64 {
- const version = try getVersion(conn);
- return version;
-}
-
-fn getVersion(conn: Sqlite.Conn) !i64 {
- const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')";
- if (try conn.scalar(bool, exists_sql, .{}) orelse false) {
- if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| {
- return version;
- }
-
- log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" });
- return error.CorruptDatabase;
- }
-
- // this pragma is one of the the few (if not only) one that's persisted, so
- // we only have to do it the first time.
- conn.exec("pragma journal_mode=wal", .{}) catch |err| {
- log.fatal(.storage, "migrate", .{
- .err = err,
- .step = "journal_mode",
- .sqlite = conn.lastError(),
- });
- return err;
- };
-
- const create_sql =
- \\ create table migrations as
- \\ select 1 as id, current_timestamp as created_at
- ;
- conn.exec(create_sql, .{}) catch |err| {
- log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" });
- return err;
- };
-
- return 1;
-}