Lightpanda: the headless browser designed for AI and automation
chore: rework sqlite cache
Karl Seguinff97f1ba5cb9
diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zigindex aa865fc31..66c424ea6 100644--- a/src/network/cache/SqliteCache.zig+++ b/src/network/cache/SqliteCache.zig@@ -19,29 +19,33 @@const std = @import("std");const lp = @import("lightpanda");+const Pool = @import("../../storage/sqlite/Pool.zig");+const Sqlite = @import("../../storage/sqlite/Sqlite.zig");++const Http = @import("../http.zig");const Cache = @import("Cache.zig");const log = lp.log;+const Blob = Sqlite.Blob;const CacheGetRequest = Cache.CacheGetRequest;const RenewResponse = Cache.RenewResponse;const CachePutRequest = Cache.CachePutRequest;const CacheGetResult = Cache.CacheGetResult;const CachedResponse = Cache.CachedResponse;-const Http = @import("../http.zig");-const Blob = @import("../../storage/sqlite/Sqlite.zig").Blob;-const Pool = @import("../../storage/sqlite/Pool.zig");-const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration;-const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations;-pub const SqliteCache = @This();-allocator: std.mem.Allocator,pool: Pool,+allocator: std.mem.Allocator,entry_limit: u32,-const cache_migrations: []const Migration = &.{- .{ .sql =+// When the schema changes, change this value, and the DB will be deleted and+// re-created with the schema+const schema_version = 1;++// body stays last so that reading columns before it hopefully doesn't visit+// overflow pages.+const schema = [_][:0]const u8{\\ create table cache (\\ url text not null primary key,\\ status integer not null,@@ -52,24 +56,15 @@ const cache_migrations: []const Migration = &.{\\ content_type text not null,\\ etag text,\\ last_modified text,+ \\ vary_headers blob not null,+ \\ headers blob not null,\\ body blob not null\\ ) strict- },- .{ .sql =- \\ create table header (- \\ url text not null,- \\ name text not null,- \\ value blob not null,- \\ vary integer not null,- \\ foreign key (url) references cache(url) on delete cascade- \\ ) strict- },- .{ .sql = "create index header_url on header(url)" },- .{ .sql = "create index cache_stored_at on cache(stored_at)" },- .{ .sql =+ ,+ "create index cache_stored_at on cache(stored_at)",\\ create index cache_expiry on cache(stored_at + max_age - age_at_store)\\ where etag is null and last_modified is null- },+ ,};// Don't change this without also updating the cache_expiry index.@@ -105,11 +100,11 @@ pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath, entry_limit: u3.memory => return err,};switch (err) {- error.NotADB, error.Corrupt => {},+ error.NotADB, error.Corrupt, error.SchemaVersion => {},else => return err,}// It's only a cache: start over rather than fail to start.- log.warn(.cache, "discarding unreadable cache", .{ .kind = "SqliteCache", .path = cache_dir, .err = err });+ log.warn(.cache, "discarding cache", .{ .kind = "SqliteCache", .path = cache_dir, .err = err });var dir = try std.Io.Dir.cwd().openDir(lp.io, cache_dir, .{});defer dir.close(lp.io);for (db_files) |name| {@@ -146,27 +141,45 @@ fn open(allocator: std.mem.Allocator, path: SqliteCachePath, entry_limit: u32) !};errdefer pool.deinit(allocator);- var version: usize = 0;-{const conn = try pool.acquire();defer pool.release(conn);try conn.exec("pragma journal_mode=wal", .{});- version = try Migrations.run(conn, cache_migrations);- }-- for (pool.conns) |conn| {- try conn.exec("pragma foreign_keys=on", .{});+ try createSchema(conn);}var cache: SqliteCache = .{ .allocator = allocator, .pool = pool, .entry_limit = entry_limit };cache.maintenance(lp.datetime.timestamp(.real));- log.info(.cache, "sqlite cache initialized", .{ .path = path, .entry_limit = entry_limit, .version = version });+ log.info(.cache, "sqlite cache initialized", .{ .path = path, .entry_limit = entry_limit, .version = schema_version });return cache;}+fn createSchema(conn: Sqlite.Conn) !void {+ // immediate: another process opening the same new file waits, then sees+ // the schema this one created.+ try conn.begin(.immediate);+ defer conn.rollback() catch {};++ const version = try conn.scalar(i64, "pragma user_version", .{}) orelse 0;+ if (version == schema_version) {+ return;+ }++ const objects = try conn.scalar(i64, "select count(*) from sqlite_schema", .{}) orelse 0;+ if (objects != 0) {+ // Another version's cache (user_version 0 = the old migrations schema).+ return error.SchemaVersion;+ }++ for (schema) |sql| {+ try conn.exec(sql, .{});+ }+ try conn.exec(std.fmt.comptimePrint("pragma user_version = {d}", .{schema_version}), .{});+ try conn.commit();+}+pub fn deinit(self: *SqliteCache) void {self.pool.deinit(self.allocator);}@@ -206,12 +219,19 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !const conn = try self.pool.acquire();defer self.pool.release(conn);- try conn.begin(.deferred);- defer conn.rollback() catch {};+ return readEntry(conn, arena, req) catch |err| switch (err) {+ error.CorruptEntry => {+ discardCorrupt(conn, req.url);+ return .miss;+ },+ else => return err,+ };+}+fn readEntry(conn: Sqlite.Conn, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {var entry = try conn.row(- \\ select status, stored_at, age_at_store,- \\ max_age, must_revalidate, content_type, etag, last_modified, body+ \\ select status, stored_at, age_at_store, max_age, must_revalidate,+ \\ content_type, etag, last_modified, vary_headers, headers, body\\ from cache where url = $1, .{req.url}) orelse {log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });@@ -219,19 +239,19 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !};defer entry.deinit();- const status: u16 = @intCast(entry.get(i64, 0));- const stored_at: u64 = @intCast(entry.get(i64, 1));- const age_at_store: u64 = @intCast(entry.get(i64, 2));- const max_age: u64 = @intCast(entry.get(i64, 3));+ const status = std.math.cast(u16, entry.get(i64, 0)) orelse return error.CorruptEntry;+ const stored_at = std.math.cast(u64, entry.get(i64, 1)) orelse return error.CorruptEntry;+ const age_at_store = std.math.cast(u64, entry.get(i64, 2)) orelse return error.CorruptEntry;+ const max_age = std.math.cast(u64, entry.get(i64, 3)) orelse return error.CorruptEntry;const must_revalidate = entry.get(bool, 4);const raw_content_type = entry.get([]const u8, 5);const raw_etag = entry.get(?[]const u8, 6);const raw_last_modified = entry.get(?[]const u8, 7);- const raw_body = entry.get(Blob, 8);const expired = must_revalidate or blk: {- const age = (req.timestamp - stored_at) + age_at_store;- break :blk age >= @as(i64, @intCast(max_age));+ // Saturates if the clock went back since the entry was stored.+ const age = (req.timestamp -| stored_at) +| age_at_store;+ break :blk age >= max_age;};const has_validators = raw_etag != null or raw_last_modified != null;@@ -242,73 +262,68 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !return .stale;}- var vary_rows = try conn.rows(- "select name, value from header where url = $1 and vary = true",- .{req.url},- );- defer vary_rows.deinit();-- while (try vary_rows.next()) |row| {- const name = row.get([]const u8, 0);- const value = row.get(Blob, 1).data;-+ var vary_it: HeaderIterator = try .init(entry.get(Blob, 8).data);+ while (try vary_it.next()) |vary| {const incoming = for (req.request_headers) |rh| {- if (std.ascii.eqlIgnoreCase(rh.name, name)) break rh.value;+ if (std.ascii.eqlIgnoreCase(rh.name, vary.name)) {+ break rh.value;+ }} else "";- if (!std.ascii.eqlIgnoreCase(value, incoming)) {+ if (!std.ascii.eqlIgnoreCase(vary.value, incoming)) {log.debug(.cache, "miss", .{.url = req.url,.reason = "vary mismatch",- .header = name,- .expected = value,+ .header = vary.name,+ .expected = vary.value,.got = incoming,});return .miss;}}- var header_rows = try conn.rows(- "select name, value from header where url = $1 and vary = false",- .{req.url},- );- defer header_rows.deinit();-- var headers: std.ArrayList(Http.Header) = .empty;-- while (try header_rows.next()) |row| {- const name = try arena.dupe(u8, row.get([]const u8, 0));- const value = try arena.dupe(u8, row.get(Blob, 1).data);- try headers.append(arena, .{ .name = name, .value = value });+ var header_it: HeaderIterator = try .init(try arena.dupe(u8, entry.get(Blob, 9).data));+ const headers = try arena.alloc(Http.Header, header_it.remaining);+ for (headers) |*h| {+ h.* = try header_it.pair();}+ try header_it.end();log.debug(.cache, "hit", .{ .url = req.url, .expired = expired });const resp: CachedResponse = .{.status = status,+ .headers = headers,.content_type = try arena.dupe(u8, raw_content_type),- .headers = headers.items,.etag = if (raw_etag) |v| try arena.dupe(u8, v) else null,.last_modified = if (raw_last_modified) |v| try arena.dupe(u8, v) else null,- .data = .{ .buffer = try arena.dupe(u8, raw_body.data) },+ .data = .{ .buffer = try arena.dupe(u8, entry.get(Blob, 10).data) },};return if (expired) .{ .revalidate = resp } else .{ .hit = resp };}pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void {- const conn = try self.pool.acquire();- defer self.pool.release(conn);+ assertLowercaseNames(req.headers);- try conn.begin(.immediate);- defer conn.rollback() catch {};+ // * 50 is just a guess+ var encoded: std.Io.Writer.Allocating = try .initCapacity(self.allocator, req.vary_headers.len + req.headers.len * 50);+ defer encoded.deinit();++ try HeaderEncoder.all(&encoded.writer, req.vary_headers);+ const vary_len = encoded.written().len;+ try HeaderEncoder.all(&encoded.writer, req.headers);- try conn.exec("delete from cache where url = $1", .{req.url});+ const vary_headers = encoded.written()[0..vary_len];+ const headers = encoded.written()[vary_len..];++ const conn = try self.pool.acquire();+ defer self.pool.release(conn);try conn.exec(- \\ insert into cache+ \\ insert or replace into cache\\ (url, status, stored_at, age_at_store, max_age, must_revalidate,- \\ content_type, etag, last_modified, body)- \\ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)+ \\ content_type, etag, last_modified, vary_headers, headers, body)+ \\ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12), .{req.url,@as(i64, @intCast(req.status)),@@ -319,29 +334,11 @@ pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void {req.content_type,req.etag,req.last_modified,+ Blob{ .data = vary_headers },+ Blob{ .data = headers },Blob{ .data = body },});- var lower_name: [256]u8 = undefined;- for (req.headers) |h| {- if (h.name.len > lower_name.len) return error.HeaderNameTooLong;- const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);- try conn.exec(- "insert into header (url, name, value, vary) values ($1, $2, $3, false)",- .{ req.url, name, Blob{ .data = h.value } },- );- }- for (req.vary_headers) |h| {- if (h.name.len > lower_name.len) return error.HeaderNameTooLong;- const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);- try conn.exec(- "insert into header (url, name, value, vary) values ($1, $2, $3, true)",- .{ req.url, name, Blob{ .data = h.value } },- );- }-- try conn.commit();-log.debug(.cache, "put", .{ .url = req.url, .body_len = body.len });}@@ -360,21 +357,49 @@ pub fn evict(self: *SqliteCache, url: []const u8) void {};defer self.pool.release(conn);+ deleteEntry(conn, url);+ log.debug(.cache, "evict", .{ .url = url });+}++fn deleteEntry(conn: Sqlite.Conn, url: []const u8) void {conn.exec("delete from cache where url = $1", .{url}) catch |err| {log.err(.cache, "delete from cache", .{ .url = url, .err = err });- return;};+}- log.debug(.cache, "evict", .{ .url = url });+fn discardCorrupt(conn: Sqlite.Conn, url: []const u8) void {+ log.warn(.cache, "discarding corrupt entry", .{ .url = url });+ deleteEntry(conn, url);}-pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void {+pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void {const conn = try self.pool.acquire();defer self.pool.release(conn);try conn.begin(.immediate);defer conn.rollback() catch {};+ assertLowercaseNames(req.headers);++ // The 304's headers replace stored ones with the same name, except+ // content-length, which describes the 304 itself.+ var encoded: std.Io.Writer.Allocating = .init(arena);+ var encoder: HeaderEncoder = .{ .writer = &encoded.writer };+ keepStoredHeaders(conn, &encoder, req) catch |err| switch (err) {+ error.CorruptEntry => {+ discardCorrupt(conn, req.url);+ try conn.commit();+ return;+ },+ else => return err,+ };+ for (req.headers) |h| {+ if (isContentLength(h.name) == false) {+ try encoder.add(h);+ }+ }+ try encoder.finish();+const directive = req.directive();try conn.exec(@@ -385,8 +410,9 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void\\ must_revalidate = coalesce($4, must_revalidate),\\ content_type = coalesce($5, content_type),\\ etag = coalesce($6, etag),- \\ last_modified = coalesce($7, last_modified)- \\ where url = $8+ \\ last_modified = coalesce($7, last_modified),+ \\ headers = $8+ \\ where url = $9, .{directive.timestamp,directive.age_at_store,@@ -395,51 +421,151 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !voiddirective.content_type,directive.etag,directive.last_modified,+ Blob{ .data = encoded.written() },req.url,});- const affected = conn.changes();- if (affected == 0) {+ try conn.commit();++ log.debug(.cache, "renewed", .{+ .url = req.url,+ .timestamp = req.timestamp,+ });+}++// Encodes the stored headers the 304 doesn't replace.+fn keepStoredHeaders(conn: Sqlite.Conn, encoder: *HeaderEncoder, req: RenewResponse) !void {+ var current = try conn.row("select headers from cache where url = $1", .{req.url}) orelse {log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });return error.CacheEntryNotFound;+ };+ defer current.deinit();++ var it: HeaderIterator = try .init(current.get(Blob, 0).data);+ while (try it.next()) |stored| {+ const replaced = for (req.headers) |h| {+ if (std.mem.eql(u8, h.name, stored.name) and isContentLength(h.name) == false) {+ break true;+ }+ } else false;+ if (replaced == false) {+ try encoder.add(stored);+ }}+}- // replace headers with the same name.- var lower_name: [256]u8 = undefined;- for (req.headers) |h| {- if (h.name.len > lower_name.len) {- return error.HeaderNameTooLong;+fn isContentLength(name: []const u8) bool {+ return std.mem.eql(u8, name, "content-length");+}++// We depend on the response header names being lowercase. HttpClient does this.+// Assert it in debug mode.+fn assertLowercaseNames(headers: []const Http.Header) void {+ if (comptime lp.IS_DEBUG) {+ for (headers) |h| {+ for (h.name) |b| {+ std.debug.assert(std.ascii.isUpper(b) == false);+ }}- const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);- if (std.mem.eql(u8, name, "content-length")) {- // content-length describes the 304 response itself- continue;+ }+}++// We encode headers using length-prefixed name, length-prefixed value+// and end with a count of the total # of headers. Putting the count at the+// end is a bit unorthodox, but it makes writing simpler AND, we aren't+// streaming: we have the full header blob in memory, reading the last 2 bytes+// is as easy as reading the first two.+const HeaderEncoder = struct {+ count: u16 = 0,+ writer: *std.Io.Writer,++ fn all(writer: *std.Io.Writer, headers: []const Http.Header) !void {+ var encoder: HeaderEncoder = .{ .writer = writer };+ for (headers) |h| {+ try encoder.add(h);}+ try encoder.finish();+ }- try conn.exec(- "delete from header where url = $1 and name = $2 and vary = false",- .{ req.url, name },- );+ fn add(self: *HeaderEncoder, h: Http.Header) !void {+ if (self.count == std.math.maxInt(u16)) {+ return error.TooManyHeaders;+ }+ if (h.name.len > std.math.maxInt(u16)) {+ return error.InvalidHeaderName;+ }+ if (h.value.len > std.math.maxInt(u16)) {+ return error.InvalidHeaderValue;+ }+ try self.writer.writeInt(u16, @intCast(h.name.len), .little);+ try self.writer.writeAll(h.name);+ try self.writer.writeInt(u16, @intCast(h.value.len), .little);+ try self.writer.writeAll(h.value);+ self.count += 1;}- for (req.headers) |h| {- const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);- if (std.mem.eql(u8, name, "content-length")) {- continue;+ fn finish(self: *HeaderEncoder) !void {+ try self.writer.writeInt(u16, self.count, .little);+ }+};++const HeaderIterator = struct {+ data: []const u8,+ remaining: u16,++ fn init(blob: []const u8) !HeaderIterator {+ if (blob.len < 2) {+ return error.CorruptEntry;+ }+ const pairs_len = blob.len - 2;+ const count = std.mem.readInt(u16, blob[pairs_len..][0..2], .little);+ if (@as(usize, count) * 4 > pairs_len) {+ // every name=>value pair is at least 4 bytes (the two u16 lengths)+ return error.CorruptEntry;}- try conn.exec(- "insert into header (url, name, value, vary) values ($1, $2, $3, false)",- .{ req.url, name, Blob{ .data = h.value } },- );+ return .{+ .remaining = count,+ .data = blob[0..pairs_len],+ };}- try conn.commit();+ fn next(self: *HeaderIterator) !?Http.Header {+ if (self.remaining == 0) {+ try self.end();+ return null;+ }+ return try self.pair();+ }- log.debug(.cache, "renewed", .{- .url = req.url,- .timestamp = req.timestamp,- });-}+ fn pair(self: *HeaderIterator) !Http.Header {+ if (self.remaining == 0) {+ return error.CorruptEntry;+ }+ self.remaining -= 1;+ const name = try self.take();+ return .{ .name = name, .value = try self.take() };+ }++ // The count and the pairs must agree.+ fn end(self: *const HeaderIterator) !void {+ if (self.remaining != 0 or self.data.len != 0) {+ return error.CorruptEntry;+ }+ }++ fn take(self: *HeaderIterator) ![]const u8 {+ if (self.data.len < 2) {+ return error.CorruptEntry;+ }+ const len = std.mem.readInt(u16, self.data[0..2], .little);+ const rest = self.data[2..];+ if (rest.len < len) {+ return error.CorruptEntry;+ }+ self.data = rest[len..];+ return rest[0..len];+ }+};const testing = @import("../../testing.zig");@@ -447,24 +573,10 @@ fn setupCache(allocator: std.mem.Allocator) !Cache {return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory, 0) } };}-test "SqliteCache: Migrations" {- const allocator = testing.allocator;- var pool = try Pool.init(allocator, ":memory:");- defer pool.deinit(allocator);-- const conn = try pool.acquire();- defer pool.release(conn);-- _ = try Migrations.run(conn, cache_migrations);-}-test "SqliteCache: basic put and get" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const req = CachePutRequest{.url = "https://example.com",@@ -473,20 +585,17 @@ test "SqliteCache: basic put and get" {.stored_at = now,.age_at_store = 0,.cache_control = .{ .max_age = 600, .must_revalidate = false },- .headers = &.{.{ .name = "Content-Type", .value = "text/html" }},+ .headers = &.{.{ .name = "content-type", .value = "text/html" }},.vary_headers = &.{},};try cache.put(req, "hello world");- const result = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now,- .request_headers = &.{},- },- );+ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ });try testing.expect(result == .hit);try testing.expectEqual("hello world", result.hit.data.buffer);@@ -498,9 +607,6 @@ test "SqliteCache: get expiration" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now = 5000;const max_age = 1000;@@ -519,24 +625,18 @@ test "SqliteCache: get expiration" {try cache.put(req, "hello world");// age = 50 + 900 = 950 < 1000: fresh- const fresh = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 50,- .request_headers = &.{},- },- );+ const fresh = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 50,+ .request_headers = &.{},+ });try testing.expect(fresh == .hit);- const stale = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 200,- .request_headers = &.{},- },- );+ const stale = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 200,+ .request_headers = &.{},+ });try testing.expect(stale == .revalidate);try testing.expectEqual("hello world", stale.revalidate.data.buffer);}@@ -545,9 +645,6 @@ test "SqliteCache: get expiration (without validators)" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now = 5000;const max_age = 1000;@@ -565,24 +662,18 @@ test "SqliteCache: get expiration (without validators)" {try cache.put(req, "hello world");// age = 50 + 900 = 950 < 1000: fresh- const fresh = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 50,- .request_headers = &.{},- },- );+ const fresh = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 50,+ .request_headers = &.{},+ });try testing.expect(fresh == .hit);- const stale = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 200,- .request_headers = &.{},- },- );+ const stale = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 200,+ .request_headers = &.{},+ });try testing.expect(stale == .stale);}@@ -590,9 +681,6 @@ test "SqliteCache: put override" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-{const req = CachePutRequest{.url = "https://example.com",@@ -606,14 +694,11 @@ test "SqliteCache: put override" {};try cache.put(req, "hello world");- const result = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = 5000,- .request_headers = &.{},- },- );+ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = 5000,+ .request_headers = &.{},+ });try testing.expect(result == .hit);try testing.expectEqual("hello world", result.hit.data.buffer);}@@ -631,14 +716,11 @@ test "SqliteCache: put override" {};try cache.put(req, "goodbye world");- const result = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = 10000,- .request_headers = &.{},- },- );+ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = 10000,+ .request_headers = &.{},+ });try testing.expect(result == .hit);try testing.expectEqual("goodbye world", result.hit.data.buffer);}@@ -648,9 +730,6 @@ test "SqliteCache: vary hit and miss" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const req = CachePutRequest{.url = "https://example.com",@@ -667,7 +746,7 @@ test "SqliteCache: vary hit and miss" {try cache.put(req, "hello world");- const hit = try cache.get(arena.allocator(), .{+ const hit = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},@@ -675,14 +754,14 @@ test "SqliteCache: vary hit and miss" {try testing.expect(hit == .hit);try testing.expectEqual("hello world", hit.hit.data.buffer);- const mismatch = try cache.get(arena.allocator(), .{+ const mismatch = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }},});try testing.expect(mismatch == .miss);- const missing_header = try cache.get(arena.allocator(), .{+ const missing_header = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{},@@ -694,9 +773,6 @@ test "SqliteCache: vary multiple headers" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const req = CachePutRequest{.url = "https://example.com",@@ -714,7 +790,7 @@ test "SqliteCache: vary multiple headers" {try cache.put(req, "hello world");- const hit = try cache.get(arena.allocator(), .{+ const hit = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{@@ -725,7 +801,7 @@ test "SqliteCache: vary multiple headers" {try testing.expect(hit == .hit);try testing.expectEqual("hello world", hit.hit.data.buffer);- const mismatch = try cache.get(arena.allocator(), .{+ const mismatch = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{@@ -740,9 +816,6 @@ test "SqliteCache: clear removes all entries" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());try cache.put(.{.url = "https://example.com/a",@@ -767,7 +840,7 @@ test "SqliteCache: clear removes all entries" {}, "body b");try testing.expect((try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com/a",.timestamp = now,@@ -775,7 +848,7 @@ test "SqliteCache: clear removes all entries" {},)) == .hit);try testing.expect((try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com/b",.timestamp = now,@@ -786,7 +859,7 @@ test "SqliteCache: clear removes all entries" {try cache.clear();try testing.expect((try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com/a",.timestamp = now,@@ -794,7 +867,7 @@ test "SqliteCache: clear removes all entries" {},)) == .miss);try testing.expect((try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com/b",.timestamp = now,@@ -807,9 +880,6 @@ test "SqliteCache: put after clear works" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const req = CachePutRequest{.url = "https://example.com",@@ -825,24 +895,18 @@ test "SqliteCache: put after clear works" {try cache.put(req, "before clear");try cache.clear();- try testing.expect((try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now,- .request_headers = &.{},- },- )) == .miss);+ try testing.expect((try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ })) == .miss);try cache.put(req, "after clear");- const result = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now,- .request_headers = &.{},- },- );+ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ });try testing.expect(result == .hit);try testing.expectEqual("after clear", result.hit.data.buffer);}@@ -851,9 +915,6 @@ test "SqliteCache: evict removes entry" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const req = CachePutRequest{.url = "https://example.com",@@ -868,16 +929,13 @@ test "SqliteCache: evict removes entry" {try cache.put(req, "hello world");- const before = try cache.get(- arena.allocator(),- .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} },- );+ const before = try cache.get(testing.arena_allocator, .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} });try testing.expect(before == .hit);cache.evict("https://example.com");try testing.expect((try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com",.timestamp = now,@@ -890,9 +948,6 @@ test "SqliteCache: renew refreshes expiry" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: i64 = 5000;try cache.put(.{.url = "https://example.com",@@ -906,30 +961,25 @@ test "SqliteCache: renew refreshes expiry" {.vary_headers = &.{},}, "hello world");- try cache.renew(- arena.allocator(),- .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} },- );+ try cache.renew(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 500,+ .headers = &.{},+ });// Clock reset to now+500, so still fresh at now+1200- const fresh = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 1200,- .request_headers = &.{},- },- );+ const fresh = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 1200,+ .request_headers = &.{},+ });try testing.expect(fresh == .hit);- const stale = try cache.get(- arena.allocator(),- .{- .url = "https://example.com",- .timestamp = now + 1500,- .request_headers = &.{},- },- );+ const stale = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 1500,+ .request_headers = &.{},+ });try testing.expect(stale == .revalidate);}@@ -937,9 +987,6 @@ test "SqliteCache: renew preserves body" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());try cache.put(.{.url = "https://example.com",@@ -952,13 +999,14 @@ test "SqliteCache: renew preserves body" {.vary_headers = &.{},}, "original body");- try cache.renew(- arena.allocator(),- .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} },- );+ try cache.renew(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now + 100,+ .headers = &.{},+ });const result = try cache.get(- arena.allocator(),+ testing.arena_allocator,.{.url = "https://example.com",.timestamp = now + 100,@@ -973,9 +1021,6 @@ test "SqliteCache: renew merges the 304's headers into the stored ones" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());try cache.put(.{.url = "https://example.com",@@ -986,15 +1031,15 @@ test "SqliteCache: renew merges the 304's headers into the stored ones" {.cache_control = .{ .max_age = 0 },.etag = "\"v1\"",.headers = &.{- .{ .name = "Content-Type", .value = "application/json" },- .{ .name = "Content-Length", .value = "11" },- .{ .name = "ETag", .value = "\"v1\"" },- .{ .name = "Link", .value = "</old>" },+ .{ .name = "content-type", .value = "application/json" },+ .{ .name = "content-length", .value = "11" },+ .{ .name = "etag", .value = "\"v1\"" },+ .{ .name = "link", .value = "</old>" },},.vary_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},}, "hello world");- try cache.renew(arena.allocator(), .{+ try cache.renew(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now + 100,.headers = &.{@@ -1006,7 +1051,7 @@ test "SqliteCache: renew merges the 304's headers into the stored ones" {},});- const result = try cache.get(arena.allocator(), .{+ const result = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now + 100,.request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},@@ -1033,9 +1078,6 @@ test "SqliteCache: maintenance purges expired entries without validators" {var cache = try setupCache(testing.allocator);defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-// Expires at 6000, no validators: purged.try cache.put(.{.url = "https://example.com/expired",@@ -1075,17 +1117,17 @@ test "SqliteCache: maintenance purges expired entries without validators" {cache.maintenance(7000);- try testing.expect((try cache.get(arena.allocator(), .{+ try testing.expect((try cache.get(testing.arena_allocator, .{.url = "https://example.com/expired",.timestamp = 7000,.request_headers = &.{},})) == .miss);- try testing.expect((try cache.get(arena.allocator(), .{+ try testing.expect((try cache.get(testing.arena_allocator, .{.url = "https://example.com/revalidate",.timestamp = 7000,.request_headers = &.{},})) == .revalidate);- try testing.expect((try cache.get(arena.allocator(), .{+ try testing.expect((try cache.get(testing.arena_allocator, .{.url = "https://example.com/fresh",.timestamp = 7000,.request_headers = &.{},@@ -1116,9 +1158,6 @@ test "SqliteCache: evicts oldest entries over the limit" {var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .memory, 3) } };defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());const urls = [_][:0]const u8{@@ -1143,7 +1182,7 @@ test "SqliteCache: evicts oldest entries over the limit" {cache.maintenance(now + urls.len);- const evicted = try cache.get(arena.allocator(), .{+ const evicted = try cache.get(testing.arena_allocator, .{.url = "https://example.com/a",.timestamp = now,.request_headers = &.{},@@ -1151,7 +1190,7 @@ test "SqliteCache: evicts oldest entries over the limit" {try testing.expect(evicted == .miss);for (urls[1..]) |url| {- const hit = try cache.get(arena.allocator(), .{+ const hit = try cache.get(testing.arena_allocator, .{.url = url,.timestamp = now + urls.len,.request_headers = &.{},@@ -1174,9 +1213,6 @@ test "SqliteCache: starts over from an unreadable database" {var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .{ .path = cache_dir }, 0) } };defer cache.deinit();- var arena = std.heap.ArenaAllocator.init(testing.allocator);- defer arena.deinit();-const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());try cache.put(.{.url = "https://example.com",@@ -1189,7 +1225,7 @@ test "SqliteCache: starts over from an unreadable database" {.vary_headers = &.{},}, "hello world");- const result = try cache.get(arena.allocator(), .{+ const result = try cache.get(testing.arena_allocator, .{.url = "https://example.com",.timestamp = now,.request_headers = &.{},@@ -1197,3 +1233,143 @@ test "SqliteCache: starts over from an unreadable database" {try testing.expect(result == .hit);try testing.expectEqual("hello world", result.hit.data.buffer);}++test "SqliteCache: starts over from another schema version" {+ testing.expectLog(&.{.cache});+ var tmp = std.testing.tmpDir(.{});+ defer tmp.cleanup();++ const cache_dir = try std.Io.Dir.path.join(testing.allocator, &.{ ".zig-cache", "tmp", &tmp.sub_path });+ defer testing.allocator.free(cache_dir);++ {+ // The schema from before user_version was used.+ const db_path = try testing.allocator.printSentinel("{s}/" ++ db_name, .{cache_dir}, 0);+ defer testing.allocator.free(db_path);+ const conn = try Sqlite.Conn.open(db_path);+ defer conn.close();+ try conn.exec("create table migrations (id integer primary key, applied_at integer not null) strict", .{});+ try conn.exec("create table cache (url text not null primary key, body blob not null) strict", .{});+ }++ var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .{ .path = cache_dir }, 0) } };+ defer cache.deinit();++ {+ const conn = try cache.kind.sqlite.pool.acquire();+ defer cache.kind.sqlite.pool.release(conn);+ try testing.expectEqual(schema_version, (try conn.scalar(i64, "pragma user_version", .{})).?);+ try testing.expectEqual(0, (try conn.scalar(i64, "select count(*) from sqlite_schema where name = 'migrations'", .{})).?);+ }++ const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());+ try cache.put(.{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .headers = &.{.{ .name = "x-test", .value = "1" }},+ .vary_headers = &.{},+ }, "hello world");++ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ });+ try testing.expect(result == .hit);+ try testing.expectEqual("hello world", result.hit.data.buffer);+ try testing.expectEqual(1, result.hit.headers.len);+ try testing.expectEqual("x-test", result.hit.headers[0].name);+}++fn putCorruptible(cache: *Cache, now: u64) !void {+ try cache.put(.{+ .url = "https://example.com",+ .content_type = "text/html",+ .status = 200,+ .stored_at = now,+ .age_at_store = 0,+ .cache_control = .{ .max_age = 600 },+ .etag = "ABC",+ .headers = &.{.{ .name = "x-test", .value = "1" }},+ .vary_headers = &.{},+ }, "hello world");+}++fn corrupt(cache: *Cache, sql: [:0]const u8) !void {+ const conn = try cache.kind.sqlite.pool.acquire();+ defer cache.kind.sqlite.pool.release(conn);+ try conn.exec(sql, .{});+}++fn entryCount(cache: *Cache) !i64 {+ const conn = try cache.kind.sqlite.pool.acquire();+ defer cache.kind.sqlite.pool.release(conn);+ return (try conn.scalar(i64, "select count(*) from cache", .{})).?;+}++test "SqliteCache: get discards a corrupt entry" {+ const corruptions = [_][:0]const u8{+ // count says 2, one pair (aaa: b) present+ "update cache set headers = x'03006161610100620200'",+ // count says 0, a pair present+ "update cache set headers = x'0100610100620000'",+ // value length past the end+ "update cache set headers = x'010061ff00620100'",+ // count far larger than the pairs could hold+ "update cache set headers = x'ffff'",+ // too short for the count+ "update cache set vary_headers = x'00'",+ "update cache set status = -1",+ "update cache set status = 70000",+ "update cache set max_age = -5",+ };++ const now: u64 = 5000;+ for (corruptions) |sql| {+ testing.expectLog(&.{.cache});+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ try putCorruptible(&cache, now);+ try corrupt(&cache, sql);++ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = now,+ .request_headers = &.{},+ });+ try testing.expect(result == .miss);+ try testing.expectEqual(0, try entryCount(&cache));+ }+}++test "SqliteCache: renew discards a corrupt entry" {+ testing.expectLog(&.{.cache});++ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ try putCorruptible(&cache, 5000);+ try corrupt(&cache, "update cache set headers = x'01'");++ try cache.renew(testing.arena_allocator, .{ .url = "https://example.com", .timestamp = 5100, .headers = &.{} });+ try testing.expectEqual(0, try entryCount(&cache));+}++test "SqliteCache: clock gone backwards" {+ var cache = try setupCache(testing.allocator);+ defer cache.deinit();++ try putCorruptible(&cache, 5000);+ const result = try cache.get(testing.arena_allocator, .{+ .url = "https://example.com",+ .timestamp = 4000,+ .request_headers = &.{},+ });+ try testing.expect(result == .hit);+ try testing.expectEqual(1, result.hit.headers.len);+}diff --git a/src/storage/sqlite/Sqlite.zig b/src/storage/sqlite/Sqlite.zigindex 3ca321ec3..9d2afcf97 100644--- a/src/storage/sqlite/Sqlite.zig+++ b/src/storage/sqlite/Sqlite.zig@@ -27,61 +27,6 @@ const Sqlite = @This();pub const Blob = struct { data: []const u8 };-pub const Migration = union(enum) {- sql: [:0]const u8,- func: struct {- ctx: *anyopaque,- func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void,- },-};--pub const Migrations = struct {- pub fn run(conn: Conn, migrations: []const Migration) !usize {- try conn.exec(- \\create table if not exists migrations (- \\ id integer primary key,- \\ applied_at integer not null- \\) strict- , .{});-- const current = (try conn.scalar(- i64,- "select max(id) from migrations",- .{},- )) orelse 0;- const start: usize = @intCast(current);-- if (start > migrations.len) {- log.err(.storage, "migrations removed", .{- .applied = start,- .defined = migrations.len,- });- return error.MigrationsRemoved;- }-- if (start == migrations.len) {- return start;- }-- try conn.begin(.immediate);- errdefer conn.rollback() catch {};-- for (migrations[start..], start..) |migration, i| {- switch (migration) {- .sql => |sql| try conn.exec(sql, .{}),- .func => |f| try f.func(conn, f.ctx),- }- try conn.exec(- "insert into migrations (id, applied_at) values ($1, $2)",- .{ @as(i64, @intCast(i + 1)), std.Io.Clock.now(.real, lp.io).toSeconds() },- );- }-- try conn.commit();- return migrations.len;- }-};-pub const Conn = struct {conn: *c.sqlite3,@@ -631,45 +576,3 @@ test "Sqlite: exec, row and scalar" {try testing.expectEqual(1, (try conn.scalar(i64, "select count(*) from test where id = 1", .{})).?);}}--test "Sqlite: Migrations - basic" {- var conn = try Sqlite.Conn.open(":memory:");- defer conn.close();-- const migrations: []const Migration = &.{- .{ .sql = "create table test (id integer primary key, name text)" },- .{ .sql = "alter table test add column email text" },- };-- const v1 = try Migrations.run(conn, migrations);- try testing.expectEqual(@as(usize, 2), v1);-- // idempotent - running again should return same version- const v2 = try Migrations.run(conn, migrations);- try testing.expectEqual(@as(usize, 2), v2);-- // verify migrations table has correct entries- try testing.expectEqual(- @as(i64, 2),- (try conn.scalar(i64, "select count(*) from migrations", .{})).?,- );-}--test "Sqlite: Migrations - removed migration" {- testing.expectLog(&.{.storage});-- var conn = try Sqlite.Conn.open(":memory:");- defer conn.close();-- const m1: []const Migration = &.{- .{ .sql = "create table test (id integer primary key, name text)" },- .{ .sql = "alter table test add column email text" },- };- _ = try Migrations.run(conn, m1);-- // fewer migrations than were applied- const m2: []const Migration = &.{- .{ .sql = "create table test (id integer primary key, name text)" },- };- try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2));-}