Skip to content

Build your personal knowledge base with Trilium Notes

6.2 KB
Raw Blame
/**
 * Streams the database out of the browser without ever holding it whole, or copying it first.
 *
 * The SAH pool has no streaming read, and staging a copy of the database beside itself needs
 * storage a browser quota may simply not have. What the pool does have is SQLite, and SQLite can
 * serve its own file a page at a time through the `sqlite_dbpage` table, the same mechanism the
 * official `sqlite3_rsync` utility is built on. The pages come straight off the live database,
 * and consistency is watched rather than arranged: the database's write counters are checked as
 * the stream goes, and a write landing mid-stream errors it so the caller can start over.
 *
 * Deliberately a dumb pipe: {@link streamLiveDatabasePages} turns the live database into a
 * `ReadableStream` of page batches, and where they flow is the caller's story — today that is the
 * backup download, which relays them through the service worker to the browser's disk.
 *
 * @module
 */

/**
 * How many pages are fetched, and travel, per stream chunk.
 *
 * One statement covers the whole batch, so this is the number of rows a single query steps through
 * rather than the number of queries. Larger batches therefore cost almost nothing extra in
 * round trips and only buy fewer of them, while the memory held is one batch: 4 MiB at an 8 KiB
 * page, 2 MiB at 4 KiB, which the container immediately re-cuts into its own 1 MiB frames.
 */
const BATCH_PAGES = 512;

/** Reads the live database through the core SQL layer, which fronts the one connection there is. */
export interface LiveDatabaseReader {
    getValue(sql: string, params?: unknown[]): unknown;
    /** One column of every row, which is how a whole batch of pages arrives in one statement. */
    getColumn(sql: string, params?: unknown[]): unknown[];
}

/** Thrown by {@link streamLiveDatabasePages} when a write lands mid-stream. The caller retries. */
export class DatabaseChangedError extends Error {
    constructor() {
        super("The database changed while it was being backed up.");
        this.name = "DatabaseChangedError";
    }
}

export interface DatabaseStream {
    /** Exact size of the streamed database, known before the first byte flows. */
    byteSize: number;
    stream: ReadableStream<Uint8Array>;
}

/**
 * Streams the live database as its exact logical bytes, without any copy of it anywhere.
 *
 * The database is in use while it is being read, and there is only the one connection, so nothing
 * can hold the content in place across the stream's yields. Consistency comes from detection
 * instead: the write counters are fingerprinted at the start and checked before every batch and
 * once more before the stream closes, and any write anywhere in between errors the stream with
 * {@link DatabaseChangedError} rather than sealing a backup that is part before and part after.
 * Every write in this process goes through the one connection the counters watch, so nothing can
 * slip past them.
 *
 * @throws Error before any stream exists when the engine lacks `sqlite_dbpage`, so a caller fails
 *   before it has created a destination to clean up.
 */
export function streamLiveDatabasePages(db: LiveDatabaseReader): DatabaseStream {
    const pageSize = countOf(db.getValue("PRAGMA page_size"), "page_size");
    const pageCount = countOf(db.getValue("PRAGMA page_count"), "page_count");
    const fingerprint = fingerprintOf(db);

    let nextPage = 1;

    return {
        byteSize: pageSize * pageCount,
        stream: new ReadableStream<Uint8Array>({
            pull(controller) {
                try {
                    if (fingerprintOf(db) !== fingerprint) {
                        throw new DatabaseChangedError();
                    }
                    if (nextPage > pageCount) {
                        controller.close();
                        return;
                    }

                    const pages = Math.min(BATCH_PAGES, pageCount - nextPage + 1);
                    const batch = readLivePages(db, nextPage, pages, pageSize);
                    nextPage += pages;

                    controller.enqueue(batch);
                } catch (e) {
                    controller.error(e);
                }
            }
        })
    };
}

/**
 * A run of pages, in one statement rather than one per page.
 *
 * This is the whole of what a backup costs to read, and the difference between the two shapes is
 * not small: a page at a time means preparing, binding, stepping and finalising a query against a
 * virtual table for every page in the database, which for a multi-gigabyte one is over a million
 * round trips into WebAssembly. Stepping one statement across the range does the same work once.
 */
function readLivePages(
    db: LiveDatabaseReader,
    firstPage: number,
    pages: number,
    pageSize: number
): Uint8Array {
    const rows = db.getColumn(
        "SELECT data FROM sqlite_dbpage WHERE pgno BETWEEN ? AND ? ORDER BY pgno",
        [ firstPage, firstPage + pages - 1 ]
    );

    if (rows.length !== pages) {
        throw new Error(`Pages ${firstPage} to ${firstPage + pages - 1}: got ${rows.length} of ${pages}.`);
    }

    const batch = new Uint8Array(pages * pageSize);
    for (let index = 0; index < pages; index++) {
        const data = rows[index];
        if (!(data instanceof Uint8Array) || data.byteLength !== pageSize) {
            throw new Error(`Page ${firstPage + index} came back malformed.`);
        }
        batch.set(data, index * pageSize);
    }

    return batch;
}

/**
 * What must not have moved between two looks for the pages to belong to one database: rows changed
 * on this connection, the schema's own version, and the file's page count.
 */
function fingerprintOf(db: LiveDatabaseReader): string {
    return [
        db.getValue("SELECT total_changes()"),
        db.getValue("PRAGMA schema_version"),
        db.getValue("PRAGMA page_count")
    ].join("/");
}

/** A pragma answer as the positive integer it must be, or the reason it is not a database. */
function countOf(value: unknown, name: string): number {
    if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
        throw new Error(`The database reports no usable ${name} (got ${String(value)}).`);
    }
    return value;
}