Skip to content

Lightpanda: the headless browser designed for AI and automation

46.3 KB
Raw Blame
// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
//
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

const std = @import("std");
const lp = @import("lightpanda");

const Pool = @import("../../storage/sqlite/Pool.zig");
const Sqlite = @import("../../storage/sqlite/Sqlite.zig");

const Http = @import("../http.zig");
const Cache = @import("Cache.zig");

const log = lp.log;
const Blob = Sqlite.Blob;
const CacheGetRequest = Cache.CacheGetRequest;
const RenewResponse = Cache.RenewResponse;
const CachePutRequest = Cache.CachePutRequest;
const CacheGetResult = Cache.CacheGetResult;
const CachedResponse = Cache.CachedResponse;

pub const SqliteCache = @This();

pool: Pool,
allocator: std.mem.Allocator,
entry_limit: u32,

// When the schema changes, change this value, and the DB will be deleted and
// re-created with the schema
const schema_version = 2;

// body stays last so that reading columns before it hopefully doesn't visit
// overflow pages.
const schema = [_][:0]const u8{
    \\ create table cache (
    \\      url               text not null primary key,
    \\      status            integer not null,
    \\      stored_at         integer not null,
    \\      age_at_store      integer not null,
    \\      max_age           integer not null,
    \\      must_revalidate   integer not null,
    \\      content_type      text not null,
    \\      etag              text,
    \\      last_modified     text,
    \\      vary_headers      blob not null,
    \\      headers           blob not null,
    \\      script_cache      blob,
    \\      body              blob not null
    \\ ) strict
    ,
    "create index cache_stored_at on cache(stored_at)",
    \\ create index cache_expiry on cache(stored_at + max_age - age_at_store)
    \\     where etag is null and last_modified is null
    ,
};

// Don't change this without also updating the cache_expiry index.
const purge_expired_sql =
    \\ delete from cache
    \\ where etag is null and last_modified is null
    \\     and stored_at + max_age - age_at_store <= $1
;

const SqliteCachePath = union(enum) {
    path: []const u8,
    memory,

    pub fn format(
        self: SqliteCachePath,
        writer: *std.Io.Writer,
    ) std.Io.Writer.Error!void {
        const name = switch (self) {
            .memory => "memory",
            .path => |p| p,
        };
        try writer.writeAll(name);
    }
};

const db_name = "cache.db";
const db_files = [_][]const u8{ db_name, db_name ++ "-wal", db_name ++ "-shm" };

pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath, entry_limit: u32) !SqliteCache {
    return open(allocator, path, entry_limit) catch |err| {
        const cache_dir = switch (path) {
            .path => |p| p,
            .memory => return err,
        };
        switch (err) {
            error.NotADB, error.Corrupt, error.SchemaVersion => {},
            else => return err,
        }
        // It's only a cache: start over rather than fail to start.
        log.warn(.cache, "discarding cache", .{ .kind = "SqliteCache", .path = cache_dir, .err = err });
        var dir = try std.Io.Dir.cwd().openDir(lp.io, cache_dir, .{});
        defer dir.close(lp.io);
        for (db_files) |name| {
            dir.deleteFile(lp.io, name) catch |delete_err| switch (delete_err) {
                error.FileNotFound => {},
                else => return delete_err,
            };
        }
        return open(allocator, path, entry_limit);
    };
}

fn open(allocator: std.mem.Allocator, path: SqliteCachePath, entry_limit: u32) !SqliteCache {
    var pool = switch (path) {
        .memory => try Pool.init(allocator, ":memory:"),
        .path => |cache_dir| blk: {
            std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| {
                log.err(
                    .cache,
                    "failed to make path",
                    .{ .kind = "SqliteCache", .path = cache_dir, .err = e },
                );
                return e;
            };

            const full_path = try allocator.printSentinel(
                "{s}/" ++ db_name,
                .{std.mem.trimEnd(u8, cache_dir, &.{'/'})},
                0,
            );
            defer allocator.free(full_path);
            break :blk try Pool.init(allocator, full_path);
        },
    };
    errdefer pool.deinit(allocator);

    {
        const conn = try pool.acquire();
        defer pool.release(conn);

        try conn.exec("pragma journal_mode=wal", .{});
        try createSchema(conn);
    }

    var cache: SqliteCache = .{ .allocator = allocator, .pool = pool, .entry_limit = entry_limit };
    cache.maintenance(lp.datetime.timestamp(.real));

    log.info(.cache, "sqlite cache initialized", .{ .path = path, .entry_limit = entry_limit, .version = schema_version });
    return cache;
}

fn createSchema(conn: Sqlite.Conn) !void {
    // immediate: another process opening the same new file waits, then sees
    // the schema this one created.
    try conn.begin(.immediate);
    defer conn.rollback() catch {};

    const version = try conn.scalar(i64, "pragma user_version", .{}) orelse 0;
    if (version == schema_version) {
        return;
    }

    const objects = try conn.scalar(i64, "select count(*) from sqlite_schema", .{}) orelse 0;
    if (objects != 0) {
        // Another version's cache (user_version 0 = the old migrations schema).
        return error.SchemaVersion;
    }

    for (schema) |sql| {
        try conn.exec(sql, .{});
    }
    try conn.exec(std.fmt.comptimePrint("pragma user_version = {d}", .{schema_version}), .{});
    try conn.commit();
}

pub fn deinit(self: *SqliteCache) void {
    self.pool.deinit(self.allocator);
}

/// Removes expired entries and enforces size limit.
pub fn maintenance(self: *SqliteCache, now: u64) void {
    const conn = self.pool.acquire() catch |err| {
        log.err(.cache, "sqlite acquire", .{ .err = err });
        return;
    };
    defer self.pool.release(conn);

    conn.exec(purge_expired_sql, .{@as(i64, @intCast(now))}) catch |err| {
        log.err(.cache, "purge expired", .{ .err = err });
    };
    const purged = conn.changes();

    if (self.entry_limit == 0) {
        return;
    }

    conn.exec(
        \\ delete from cache
        \\ where rowid in (
        \\     select rowid from cache
        \\     order by stored_at desc
        \\     limit -1 offset $1
        \\ )
    , .{@as(i64, @intCast(self.entry_limit))}) catch |err| {
        log.err(.cache, "evict overflow", .{ .err = err });
    };

    log.debug(.cache, "maintenance", .{ .purged = purged, .evicted = conn.changes() });
}

pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {
    const conn = try self.pool.acquire();
    defer self.pool.release(conn);

    return readEntry(conn, arena, req) catch |err| switch (err) {
        error.CorruptEntry => {
            discardCorrupt(conn, req.url);
            return .miss;
        },
        else => return err,
    };
}

fn readEntry(conn: Sqlite.Conn, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {
    var entry = try conn.row(
        \\ select status, stored_at, age_at_store, max_age, must_revalidate,
        \\     content_type, etag, last_modified, vary_headers, headers,
        \\     script_cache, body
        \\ from cache where url = $1
    , .{req.url}) orelse {
        log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
        return .miss;
    };
    defer entry.deinit();

    const status = std.math.cast(u16, entry.get(i64, 0)) orelse return error.CorruptEntry;
    const stored_at = std.math.cast(u64, entry.get(i64, 1)) orelse return error.CorruptEntry;
    const age_at_store = std.math.cast(u64, entry.get(i64, 2)) orelse return error.CorruptEntry;
    const max_age = std.math.cast(u64, entry.get(i64, 3)) orelse return error.CorruptEntry;
    const must_revalidate = entry.get(bool, 4);
    const raw_content_type = entry.get([]const u8, 5);
    const raw_etag = entry.get(?[]const u8, 6);
    const raw_last_modified = entry.get(?[]const u8, 7);

    const expired = must_revalidate or blk: {
        // Saturates if the clock went back since the entry was stored.
        const age = (req.timestamp -| stored_at) +| age_at_store;
        break :blk age >= max_age;
    };
    const has_validators = raw_etag != null or raw_last_modified != null;

    // If it is expired without validators,
    // we are going to have to make a network request for this resource.
    if (expired and !has_validators) {
        log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired with no validators" });
        return .stale;
    }

    var vary_it: HeaderIterator = try .init(entry.get(Blob, 8).data);
    while (try vary_it.next()) |vary| {
        const incoming = for (req.request_headers) |rh| {
            if (std.ascii.eqlIgnoreCase(rh.name, vary.name)) {
                break rh.value;
            }
        } else "";

        if (!std.ascii.eqlIgnoreCase(vary.value, incoming)) {
            log.debug(.cache, "miss", .{
                .url = req.url,
                .reason = "vary mismatch",
                .header = vary.name,
                .expected = vary.value,
                .got = incoming,
            });
            return .miss;
        }
    }

    var header_it: HeaderIterator = try .init(try arena.dupe(u8, entry.get(Blob, 9).data));
    const headers = try arena.alloc(Http.Header, header_it.remaining);
    for (headers) |*h| {
        h.* = try header_it.pair();
    }
    try header_it.end();

    log.debug(.cache, "hit", .{ .url = req.url, .expired = expired });

    const resp: CachedResponse = .{
        .status = status,
        .headers = headers,
        .content_type = try arena.dupe(u8, raw_content_type),
        .etag = if (raw_etag) |v| try arena.dupe(u8, v) else null,
        .last_modified = if (raw_last_modified) |v| try arena.dupe(u8, v) else null,
        .script_cache = if (entry.get(?Blob, 10)) |b| try arena.dupe(u8, b.data) else null,
        .data = .{ .buffer = try arena.dupe(u8, entry.get(Blob, 11).data) },
    };
    return if (expired) .{ .revalidate = resp } else .{ .hit = resp };
}

pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void {
    assertLowercaseNames(req.headers);

    // * 50 is just a guess
    var encoded: std.Io.Writer.Allocating = try .initCapacity(self.allocator, req.vary_headers.len + req.headers.len * 50);
    defer encoded.deinit();

    try HeaderEncoder.all(&encoded.writer, req.vary_headers);
    const vary_len = encoded.written().len;
    try HeaderEncoder.all(&encoded.writer, req.headers);

    const vary_headers = encoded.written()[0..vary_len];
    const headers = encoded.written()[vary_len..];

    const conn = try self.pool.acquire();
    defer self.pool.release(conn);

    try conn.exec(
        \\ insert or replace into cache
        \\     (url, status, stored_at, age_at_store, max_age, must_revalidate,
        \\     content_type, etag, last_modified, vary_headers, headers, body)
        \\ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
    , .{
        req.url,
        @as(i64, @intCast(req.status)),
        req.stored_at,
        @as(i64, @intCast(req.age_at_store)),
        @as(i64, @intCast(req.cache_control.max_age)),
        req.cache_control.must_revalidate,
        req.content_type,
        req.etag,
        req.last_modified,
        Blob{ .data = vary_headers },
        Blob{ .data = headers },
        Blob{ .data = body },
    });

    log.debug(.cache, "put", .{ .url = req.url, .body_len = body.len });
}

pub fn putScriptCache(self: *SqliteCache, url: []const u8, body: []const u8, bytes: []const u8) !void {
    const conn = try self.pool.acquire();
    defer self.pool.release(conn);

    // We match on the body to make sure we're setting the script_cache for
    // the right body. It is possible, though unlikely, that this row has been
    // updated since we read it with a new body. If so, our cached script would
    // no longer be valid.
    try conn.exec(
        "update cache set script_cache = $1 where url = $2 and body = $3",
        .{ Blob{ .data = bytes }, url, Blob{ .data = body } },
    );
}

pub fn clear(self: *SqliteCache) !void {
    const conn = try self.pool.acquire();
    defer self.pool.release(conn);

    try conn.exec("delete from cache", .{});
    log.debug(.cache, "clear", .{});
}

pub fn evict(self: *SqliteCache, url: []const u8) void {
    const conn = self.pool.acquire() catch |err| {
        log.err(.cache, "sqlite acquire", .{ .url = url, .err = err });
        return;
    };
    defer self.pool.release(conn);

    deleteEntry(conn, url);
    log.debug(.cache, "evict", .{ .url = url });
}

fn deleteEntry(conn: Sqlite.Conn, url: []const u8) void {
    conn.exec("delete from cache where url = $1", .{url}) catch |err| {
        log.err(.cache, "delete from cache", .{ .url = url, .err = err });
    };
}

fn discardCorrupt(conn: Sqlite.Conn, url: []const u8) void {
    log.warn(.cache, "discarding corrupt entry", .{ .url = url });
    deleteEntry(conn, url);
}

pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void {
    const conn = try self.pool.acquire();
    defer self.pool.release(conn);

    try conn.begin(.immediate);
    defer conn.rollback() catch {};

    assertLowercaseNames(req.headers);

    // The 304's headers replace stored ones with the same name, except
    // content-length, which describes the 304 itself.
    var encoded: std.Io.Writer.Allocating = .init(arena);
    var encoder: HeaderEncoder = .{ .writer = &encoded.writer };
    keepStoredHeaders(conn, &encoder, req) catch |err| switch (err) {
        error.CorruptEntry => {
            discardCorrupt(conn, req.url);
            try conn.commit();
            return;
        },
        else => return err,
    };
    for (req.headers) |h| {
        if (isContentLength(h.name) == false) {
            try encoder.add(h);
        }
    }
    try encoder.finish();

    const directive = req.directive();

    try conn.exec(
        \\ update cache
        \\ set stored_at = $1,
        \\     age_at_store = $2,
        \\     max_age = coalesce($3, max_age),
        \\     must_revalidate = coalesce($4, must_revalidate),
        \\     content_type = coalesce($5, content_type),
        \\     etag = coalesce($6, etag),
        \\     last_modified = coalesce($7, last_modified),
        \\     headers = $8
        \\ where url = $9
    , .{
        directive.timestamp,
        directive.age_at_store,
        directive.max_age,
        directive.must_revalidate,
        directive.content_type,
        directive.etag,
        directive.last_modified,
        Blob{ .data = encoded.written() },
        req.url,
    });

    try conn.commit();

    log.debug(.cache, "renewed", .{
        .url = req.url,
        .timestamp = req.timestamp,
    });
}

// Encodes the stored headers the 304 doesn't replace.
fn keepStoredHeaders(conn: Sqlite.Conn, encoder: *HeaderEncoder, req: RenewResponse) !void {
    var current = try conn.row("select headers from cache where url = $1", .{req.url}) orelse {
        log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
        return error.CacheEntryNotFound;
    };
    defer current.deinit();

    var it: HeaderIterator = try .init(current.get(Blob, 0).data);
    while (try it.next()) |stored| {
        const replaced = for (req.headers) |h| {
            if (std.mem.eql(u8, h.name, stored.name) and isContentLength(h.name) == false) {
                break true;
            }
        } else false;
        if (replaced == false) {
            try encoder.add(stored);
        }
    }
}

fn isContentLength(name: []const u8) bool {
    return std.mem.eql(u8, name, "content-length");
}

// We depend on the response header names being lowercase. HttpClient does this.
// Assert it in debug mode.
fn assertLowercaseNames(headers: []const Http.Header) void {
    if (comptime lp.IS_DEBUG) {
        for (headers) |h| {
            for (h.name) |b| {
                std.debug.assert(std.ascii.isUpper(b) == false);
            }
        }
    }
}

// We encode headers using length-prefixed name, length-prefixed value
// and end with a count of the total # of headers. Putting the count at the
// end is a bit unorthodox, but it makes writing simpler AND, we aren't
// streaming: we have the full header blob in memory, reading the last 2 bytes
// is as easy as reading the first two.
const HeaderEncoder = struct {
    count: u16 = 0,
    writer: *std.Io.Writer,

    fn all(writer: *std.Io.Writer, headers: []const Http.Header) !void {
        var encoder: HeaderEncoder = .{ .writer = writer };
        for (headers) |h| {
            try encoder.add(h);
        }
        try encoder.finish();
    }

    fn add(self: *HeaderEncoder, h: Http.Header) !void {
        if (self.count == std.math.maxInt(u16)) {
            return error.TooManyHeaders;
        }
        if (h.name.len > std.math.maxInt(u16)) {
            return error.InvalidHeaderName;
        }
        if (h.value.len > std.math.maxInt(u16)) {
            return error.InvalidHeaderValue;
        }
        try self.writer.writeInt(u16, @intCast(h.name.len), .little);
        try self.writer.writeAll(h.name);
        try self.writer.writeInt(u16, @intCast(h.value.len), .little);
        try self.writer.writeAll(h.value);
        self.count += 1;
    }

    fn finish(self: *HeaderEncoder) !void {
        try self.writer.writeInt(u16, self.count, .little);
    }
};

const HeaderIterator = struct {
    data: []const u8,
    remaining: u16,

    fn init(blob: []const u8) !HeaderIterator {
        if (blob.len < 2) {
            return error.CorruptEntry;
        }
        const pairs_len = blob.len - 2;
        const count = std.mem.readInt(u16, blob[pairs_len..][0..2], .little);
        if (@as(usize, count) * 4 > pairs_len) {
            // every name=>value pair is at least 4 bytes (the two u16 lengths)
            return error.CorruptEntry;
        }
        return .{
            .remaining = count,
            .data = blob[0..pairs_len],
        };
    }

    fn next(self: *HeaderIterator) !?Http.Header {
        if (self.remaining == 0) {
            try self.end();
            return null;
        }
        return try self.pair();
    }

    fn pair(self: *HeaderIterator) !Http.Header {
        if (self.remaining == 0) {
            return error.CorruptEntry;
        }
        self.remaining -= 1;
        const name = try self.take();
        return .{ .name = name, .value = try self.take() };
    }

    // The count and the pairs must agree.
    fn end(self: *const HeaderIterator) !void {
        if (self.remaining != 0 or self.data.len != 0) {
            return error.CorruptEntry;
        }
    }

    fn take(self: *HeaderIterator) ![]const u8 {
        if (self.data.len < 2) {
            return error.CorruptEntry;
        }
        const len = std.mem.readInt(u16, self.data[0..2], .little);
        const rest = self.data[2..];
        if (rest.len < len) {
            return error.CorruptEntry;
        }
        self.data = rest[len..];
        return rest[0..len];
    }
};

const testing = @import("../../testing.zig");

fn setupCache(allocator: std.mem.Allocator) !Cache {
    return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory, 0) } };
}

test "SqliteCache: basic put and get" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600, .must_revalidate = false },
        .headers = &.{.{ .name = "content-type", .value = "text/html" }},
        .vary_headers = &.{},
    };

    try cache.put(req, "hello world");

    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    });

    try testing.expect(result == .hit);
    try testing.expectEqual("hello world", result.hit.data.buffer);
    try testing.expectEqual(@as(u16, 200), result.hit.status);
    try testing.expectEqual("text/html", result.hit.content_type);
}

test "SqliteCache: get expiration" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now = 5000;
    const max_age = 1000;

    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 900,
        .cache_control = .{ .max_age = max_age },
        .etag = "ABC",
        .headers = &.{},
        .vary_headers = &.{},
    };

    try cache.put(req, "hello world");

    // age = 50 + 900 = 950 < 1000: fresh
    const fresh = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 50,
        .request_headers = &.{},
    });
    try testing.expect(fresh == .hit);

    const stale = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 200,
        .request_headers = &.{},
    });
    try testing.expect(stale == .revalidate);
    try testing.expectEqual("hello world", stale.revalidate.data.buffer);
}

test "SqliteCache: get expiration (without validators)" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now = 5000;
    const max_age = 1000;

    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 900,
        .cache_control = .{ .max_age = max_age },
        .headers = &.{},
        .vary_headers = &.{},
    };

    try cache.put(req, "hello world");

    // age = 50 + 900 = 950 < 1000: fresh
    const fresh = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 50,
        .request_headers = &.{},
    });
    try testing.expect(fresh == .hit);

    const stale = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 200,
        .request_headers = &.{},
    });
    try testing.expect(stale == .stale);
}

test "SqliteCache: put override" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    {
        const req = CachePutRequest{
            .url = "https://example.com",
            .content_type = "text/html",
            .status = 200,
            .stored_at = 5000,
            .age_at_store = 0,
            .cache_control = .{ .max_age = 1000 },
            .headers = &.{},
            .vary_headers = &.{},
        };
        try cache.put(req, "hello world");

        const result = try cache.get(testing.arena_allocator, .{
            .url = "https://example.com",
            .timestamp = 5000,
            .request_headers = &.{},
        });
        try testing.expect(result == .hit);
        try testing.expectEqual("hello world", result.hit.data.buffer);
    }

    {
        const req = CachePutRequest{
            .url = "https://example.com",
            .content_type = "text/html",
            .status = 200,
            .stored_at = 10000,
            .age_at_store = 0,
            .cache_control = .{ .max_age = 2000 },
            .headers = &.{},
            .vary_headers = &.{},
        };
        try cache.put(req, "goodbye world");

        const result = try cache.get(testing.arena_allocator, .{
            .url = "https://example.com",
            .timestamp = 10000,
            .request_headers = &.{},
        });
        try testing.expect(result == .hit);
        try testing.expectEqual("goodbye world", result.hit.data.buffer);
    }
}

test "SqliteCache: vary hit and miss" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{
            .{ .name = "Accept-Encoding", .value = "gzip" },
        },
    };

    try cache.put(req, "hello world");

    const hit = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},
    });
    try testing.expect(hit == .hit);
    try testing.expectEqual("hello world", hit.hit.data.buffer);

    const mismatch = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }},
    });
    try testing.expect(mismatch == .miss);

    const missing_header = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    });
    try testing.expect(missing_header == .miss);
}

test "SqliteCache: vary multiple headers" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{
            .{ .name = "Accept-Encoding", .value = "gzip" },
            .{ .name = "Accept-Language", .value = "en" },
        },
    };

    try cache.put(req, "hello world");

    const hit = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{
            .{ .name = "Accept-Encoding", .value = "gzip" },
            .{ .name = "Accept-Language", .value = "en" },
        },
    });
    try testing.expect(hit == .hit);
    try testing.expectEqual("hello world", hit.hit.data.buffer);

    const mismatch = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{
            .{ .name = "Accept-Encoding", .value = "gzip" },
            .{ .name = "Accept-Language", .value = "fr" },
        },
    });
    try testing.expect(mismatch == .miss);
}

test "SqliteCache: clear removes all entries" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    try cache.put(.{
        .url = "https://example.com/a",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "body a");

    try cache.put(.{
        .url = "https://example.com/b",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "body b");

    try testing.expect((try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com/a",
            .timestamp = now,
            .request_headers = &.{},
        },
    )) == .hit);
    try testing.expect((try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com/b",
            .timestamp = now,
            .request_headers = &.{},
        },
    )) == .hit);

    try cache.clear();

    try testing.expect((try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com/a",
            .timestamp = now,
            .request_headers = &.{},
        },
    )) == .miss);
    try testing.expect((try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com/b",
            .timestamp = now,
            .request_headers = &.{},
        },
    )) == .miss);
}

test "SqliteCache: put after clear works" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    };

    try cache.put(req, "before clear");
    try cache.clear();

    try testing.expect((try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    })) == .miss);

    try cache.put(req, "after clear");
    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    });
    try testing.expect(result == .hit);
    try testing.expectEqual("after clear", result.hit.data.buffer);
}

test "SqliteCache: evict removes entry" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req = CachePutRequest{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    };

    try cache.put(req, "hello world");

    const before = try cache.get(testing.arena_allocator, .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} });
    try testing.expect(before == .hit);

    cache.evict("https://example.com");

    try testing.expect((try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com",
            .timestamp = now,
            .request_headers = &.{},
        },
    )) == .miss);
}

test "SqliteCache: renew refreshes expiry" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: i64 = 5000;
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 1000 },
        .etag = "ABC",
        .headers = &.{},
        .vary_headers = &.{},
    }, "hello world");

    try cache.renew(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 500,
        .headers = &.{},
    });

    // Clock reset to now+500, so still fresh at now+1200
    const fresh = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 1200,
        .request_headers = &.{},
    });
    try testing.expect(fresh == .hit);

    const stale = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 1500,
        .request_headers = &.{},
    });
    try testing.expect(stale == .revalidate);
}

test "SqliteCache: renew preserves body" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "original body");

    try cache.renew(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 100,
        .headers = &.{},
    });

    const result = try cache.get(
        testing.arena_allocator,
        .{
            .url = "https://example.com",
            .timestamp = now + 100,
            .request_headers = &.{},
        },
    );
    try testing.expect(result == .hit);
    try testing.expectEqual("original body", result.hit.data.buffer);
}

test "SqliteCache: script cache is kept by renew and cleared by put" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    const req: CachePutRequest = .{
        .url = "https://example.com/a.js",
        .content_type = "text/javascript",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    };
    try cache.put(req, "var a = 1;");
    try testing.expectEqual(null, try getScriptCache(&cache, now));

    // Not written when the stored body isn't the one the bytes came from.
    try cache.putScriptCache(req.url, "var a = 2;", "stale");
    try testing.expectEqual(null, try getScriptCache(&cache, now));

    try cache.putScriptCache(req.url, "var a = 1;", "compiled");
    try testing.expectEqual("compiled", (try getScriptCache(&cache, now)).?);

    try cache.renew(testing.arena_allocator, .{ .url = req.url, .timestamp = now + 100, .headers = &.{} });
    try testing.expectEqual("compiled", (try getScriptCache(&cache, now + 100)).?);

    try cache.put(req, "var a = 1;");
    try testing.expectEqual(null, try getScriptCache(&cache, now));
}

fn getScriptCache(cache: *Cache, now: u64) !?[]const u8 {
    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com/a.js",
        .timestamp = now,
        .request_headers = &.{},
    });
    return result.hit.script_cache;
}

test "SqliteCache: renew merges the 304's headers into the stored ones" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "application/json",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 0 },
        .etag = "\"v1\"",
        .headers = &.{
            .{ .name = "content-type", .value = "application/json" },
            .{ .name = "content-length", .value = "11" },
            .{ .name = "etag", .value = "\"v1\"" },
            .{ .name = "link", .value = "</old>" },
        },
        .vary_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},
    }, "hello world");

    try cache.renew(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 100,
        .headers = &.{
            .{ .name = "etag", .value = "\"v2\"" },
            .{ .name = "cache-control", .value = "max-age=600" },
            .{ .name = "content-length", .value = "0" },
            .{ .name = "link", .value = "</a>" },
            .{ .name = "link", .value = "</b>" },
        },
    });

    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now + 100,
        .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},
    });
    try testing.expect(result == .hit);

    const expected = [_]Http.Header{
        .{ .name = "content-type", .value = "application/json" },
        .{ .name = "content-length", .value = "11" },
        .{ .name = "etag", .value = "\"v2\"" },
        .{ .name = "cache-control", .value = "max-age=600" },
        .{ .name = "link", .value = "</a>" },
        .{ .name = "link", .value = "</b>" },
    };
    const headers = result.hit.headers;
    try testing.expectEqual(expected.len, headers.len);
    for (expected, headers) |e, h| {
        try testing.expectEqual(e.name, h.name);
        try testing.expectEqual(e.value, h.value);
    }
}

test "SqliteCache: maintenance purges expired entries without validators" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    // Expires at 6000, no validators: purged.
    try cache.put(.{
        .url = "https://example.com/expired",
        .content_type = "text/html",
        .status = 200,
        .stored_at = 5000,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 1000 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "expired");

    // Expires at 6000 but has an etag: kept for revalidation.
    try cache.put(.{
        .url = "https://example.com/revalidate",
        .content_type = "text/html",
        .status = 200,
        .stored_at = 5000,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 1000 },
        .etag = "ABC",
        .headers = &.{},
        .vary_headers = &.{},
    }, "revalidate");

    // Expires at 7900: kept.
    try cache.put(.{
        .url = "https://example.com/fresh",
        .content_type = "text/html",
        .status = 200,
        .stored_at = 6900,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 1000 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "fresh");

    cache.maintenance(7000);

    try testing.expect((try cache.get(testing.arena_allocator, .{
        .url = "https://example.com/expired",
        .timestamp = 7000,
        .request_headers = &.{},
    })) == .miss);
    try testing.expect((try cache.get(testing.arena_allocator, .{
        .url = "https://example.com/revalidate",
        .timestamp = 7000,
        .request_headers = &.{},
    })) == .revalidate);
    try testing.expect((try cache.get(testing.arena_allocator, .{
        .url = "https://example.com/fresh",
        .timestamp = 7000,
        .request_headers = &.{},
    })) == .hit);
}

test "SqliteCache: expired purge uses the cache_expiry index" {
    var cache = try SqliteCache.init(testing.allocator, .memory, 0);
    defer cache.deinit();

    const conn = try cache.pool.acquire();
    defer cache.pool.release(conn);

    var plan = (try conn.row(
        "explain query plan " ++ purge_expired_sql,
        .{@as(i64, 0)},
    )) orelse return error.NoQueryPlan;
    defer plan.deinit();

    const detail = plan.get([]const u8, 3);
    if (std.mem.find(u8, detail, "cache_expiry") == null) {
        std.debug.print("query plan: {s}\n", .{detail});
        return error.FullTableScan;
    }
}

test "SqliteCache: evicts oldest entries over the limit" {
    var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .memory, 3) } };
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());

    const urls = [_][:0]const u8{
        "https://example.com/a",
        "https://example.com/b",
        "https://example.com/c",
        "https://example.com/d",
    };

    for (urls, 0..) |url, i| {
        try cache.put(.{
            .url = url,
            .content_type = "text/html",
            .status = 200,
            .stored_at = now + i,
            .age_at_store = 0,
            .cache_control = .{ .max_age = 600 },
            .headers = &.{},
            .vary_headers = &.{},
        }, url);
    }

    cache.maintenance(now + urls.len);

    const evicted = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com/a",
        .timestamp = now,
        .request_headers = &.{},
    });
    try testing.expect(evicted == .miss);

    for (urls[1..]) |url| {
        const hit = try cache.get(testing.arena_allocator, .{
            .url = url,
            .timestamp = now + urls.len,
            .request_headers = &.{},
        });
        try testing.expect(hit == .hit);
        try testing.expectEqual(url, hit.hit.data.buffer);
    }
}

test "SqliteCache: starts over from an unreadable database" {
    testing.expectLog(&.{.cache});
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    try tmp.dir.writeFile(testing.io, .{ .sub_path = "cache.db", .data = "This is not an SQLite database, only a page of text where its header would be." });
    try tmp.dir.writeFile(testing.io, .{ .sub_path = "cache.db-wal", .data = "stale" });

    const cache_dir = try std.Io.Dir.path.join(testing.allocator, &.{ ".zig-cache", "tmp", &tmp.sub_path });
    defer testing.allocator.free(cache_dir);

    var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .{ .path = cache_dir }, 0) } };
    defer cache.deinit();

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{},
        .vary_headers = &.{},
    }, "hello world");

    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    });
    try testing.expect(result == .hit);
    try testing.expectEqual("hello world", result.hit.data.buffer);
}

test "SqliteCache: starts over from another schema version" {
    testing.expectLog(&.{.cache});
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();

    const cache_dir = try std.Io.Dir.path.join(testing.allocator, &.{ ".zig-cache", "tmp", &tmp.sub_path });
    defer testing.allocator.free(cache_dir);

    {
        // The schema from before user_version was used.
        const db_path = try testing.allocator.printSentinel("{s}/" ++ db_name, .{cache_dir}, 0);
        defer testing.allocator.free(db_path);
        const conn = try Sqlite.Conn.open(db_path);
        defer conn.close();
        try conn.exec("create table migrations (id integer primary key, applied_at integer not null) strict", .{});
        try conn.exec("create table cache (url text not null primary key, body blob not null) strict", .{});
    }

    var cache = Cache{ .kind = .{ .sqlite = try .init(testing.allocator, .{ .path = cache_dir }, 0) } };
    defer cache.deinit();

    {
        const conn = try cache.kind.sqlite.pool.acquire();
        defer cache.kind.sqlite.pool.release(conn);
        try testing.expectEqual(schema_version, (try conn.scalar(i64, "pragma user_version", .{})).?);
        try testing.expectEqual(0, (try conn.scalar(i64, "select count(*) from sqlite_schema where name = 'migrations'", .{})).?);
    }

    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .headers = &.{.{ .name = "x-test", .value = "1" }},
        .vary_headers = &.{},
    }, "hello world");

    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = now,
        .request_headers = &.{},
    });
    try testing.expect(result == .hit);
    try testing.expectEqual("hello world", result.hit.data.buffer);
    try testing.expectEqual(1, result.hit.headers.len);
    try testing.expectEqual("x-test", result.hit.headers[0].name);
}

fn putCorruptible(cache: *Cache, now: u64) !void {
    try cache.put(.{
        .url = "https://example.com",
        .content_type = "text/html",
        .status = 200,
        .stored_at = now,
        .age_at_store = 0,
        .cache_control = .{ .max_age = 600 },
        .etag = "ABC",
        .headers = &.{.{ .name = "x-test", .value = "1" }},
        .vary_headers = &.{},
    }, "hello world");
}

fn corrupt(cache: *Cache, sql: [:0]const u8) !void {
    const conn = try cache.kind.sqlite.pool.acquire();
    defer cache.kind.sqlite.pool.release(conn);
    try conn.exec(sql, .{});
}

fn entryCount(cache: *Cache) !i64 {
    const conn = try cache.kind.sqlite.pool.acquire();
    defer cache.kind.sqlite.pool.release(conn);
    return (try conn.scalar(i64, "select count(*) from cache", .{})).?;
}

test "SqliteCache: get discards a corrupt entry" {
    const corruptions = [_][:0]const u8{
        // count says 2, one pair (aaa: b) present
        "update cache set headers = x'03006161610100620200'",
        // count says 0, a pair present
        "update cache set headers = x'0100610100620000'",
        // value length past the end
        "update cache set headers = x'010061ff00620100'",
        // count far larger than the pairs could hold
        "update cache set headers = x'ffff'",
        // too short for the count
        "update cache set vary_headers = x'00'",
        "update cache set status = -1",
        "update cache set status = 70000",
        "update cache set max_age = -5",
    };

    const now: u64 = 5000;
    for (corruptions) |sql| {
        testing.expectLog(&.{.cache});
        var cache = try setupCache(testing.allocator);
        defer cache.deinit();

        try putCorruptible(&cache, now);
        try corrupt(&cache, sql);

        const result = try cache.get(testing.arena_allocator, .{
            .url = "https://example.com",
            .timestamp = now,
            .request_headers = &.{},
        });
        try testing.expect(result == .miss);
        try testing.expectEqual(0, try entryCount(&cache));
    }
}

test "SqliteCache: renew discards a corrupt entry" {
    testing.expectLog(&.{.cache});

    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    try putCorruptible(&cache, 5000);
    try corrupt(&cache, "update cache set headers = x'01'");

    try cache.renew(testing.arena_allocator, .{ .url = "https://example.com", .timestamp = 5100, .headers = &.{} });
    try testing.expectEqual(0, try entryCount(&cache));
}

test "SqliteCache: clock gone backwards" {
    var cache = try setupCache(testing.allocator);
    defer cache.deinit();

    try putCorruptible(&cache, 5000);
    const result = try cache.get(testing.arena_allocator, .{
        .url = "https://example.com",
        .timestamp = 4000,
        .request_headers = &.{},
    });
    try testing.expect(result == .hit);
    try testing.expectEqual(1, result.hit.headers.len);
}