Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
import type { SAHPoolUtil } from "@sqlite.org/sqlite-wasm";
import {
FIXED_HEADER_BYTES,
getInfo,
isBackupContainerError,
type ProgressCallback,
readBackupContainer,
type SupportedBackupContainer,
containerSize
} from "@triliumnext/backup-container/web";
import {
type CandidateDatabase,
type DatabaseValidation,
looksLikeSqlite,
validateDatabase
} from "@triliumnext/core";
/**
* Puts a backup in place of this instance's database, in the browser.
*
* The server exchanges two files on a disk. Here the databases live as entries in the SAH pool,
* which has no rename, so a restore is done by writing the new database in beside the old one and
* then changing which name the next start opens. That single small write is the whole of the swap:
* before it the old database is live, after it the new one is, and there is no moment in between
* where neither is.
*
* Nothing is ever held whole. The backup arrives as a stream and is fed to `importDb` chunk by
* chunk, and the checks read the imported database page by page through the VFS, so a restore costs
* the size of the backup on disk and almost nothing in memory. That matters more here than on a
* server: this database engine is compiled to WebAssembly, whose address space is a fraction of what
* a native process has.
*
* @module
*/
/** Where the pointer to the live database is kept, in the origin's private filesystem. */
const POINTER_FILE = "trilium-current-database";
/**
* The two names a database is ever kept under.
*
* A restore writes the new database in beside the old one and then points at it, so the name it
* writes to has to be the one that is *not* live. Alternating between two settles that: whichever is
* live, the other is free, and a restore can never be preparing over the database it is replacing.
*/
const DATABASE_NAMES = [ "/trilium.db", "/trilium.alt.db" ] as const;
/** The database opened when nothing has ever been restored. */
export const DEFAULT_DATABASE_NAME = DATABASE_NAMES[0];
/** Where a backup is written while it is being checked, which is wherever the live one is not. */
function candidateNameFor(live: string): string {
return DATABASE_NAMES.find((name) => name !== live) ?? DATABASE_NAMES[1];
}
/** What the restore is doing, in the order it does it. Mirrors the server's own steps. */
export type RestoreStage = "staging" | "validating" | "swapping" | "done" | "failed";
export interface RestoreProgress {
stage: RestoreStage;
/** How far through the current step, from 0 to 1, for the step that can say. */
fraction?: number;
error?: string;
reason?: string;
}
/** The database the restore acts on, reduced to what it needs and no more. */
export interface RestoreTarget {
pool: SAHPoolUtil;
/** Closes whatever is open. */
close(): void;
/** Opens the named pool entry as the live database. */
open(dbName: string): void;
}
/**
* Reads which pool entry holds the live database.
*
* Falls back to the original name, which is what every instance that has never restored anything is
* still using, and what a browser that has lost the pointer should reach for.
*/
export async function readCurrentDatabaseName(): Promise<string> {
try {
const root = await navigator.storage.getDirectory();
const handle = await root.getFileHandle(POINTER_FILE);
const name = (await (await handle.getFile()).text()).trim();
return name || DEFAULT_DATABASE_NAME;
} catch {
return DEFAULT_DATABASE_NAME;
}
}
/**
* Restores `backup` over the live database, reporting each step through `report`.
*
* @param backup the file the user chose, read where it lies rather than copied anywhere first.
* @throws Error carrying a `reason` the setup screen maps to something the user can act on.
*/
export async function restoreDatabase(
target: RestoreTarget,
backup: Blob,
options: { passphrase?: string; report?: (progress: RestoreProgress) => void } = {}
): Promise<void> {
const report = options.report ?? (() => {});
const live = await readCurrentDatabaseName();
const candidate = candidateNameFor(live);
try {
report({ stage: "staging" });
await keepingHandles(target, live, () =>
stageCandidate(target.pool, candidate, backup, options.passphrase, (fraction) =>
report({ stage: "staging", fraction })));
report({ stage: "validating" });
const validation = await keepingHandles(target, live, async () => validate(target.pool, candidate));
if (!validation.valid) {
throw new RestoreFailure(validation.rejection, validation.message);
}
report({ stage: "swapping" });
await swapIn(target, live, candidate);
report({ stage: "done" });
} catch (e) {
const failure = asFailure(e);
// The candidate is no use to anyone now, and the pool's slots are finite. Unless a start
// would open it, which is the one case where it is not a leftover but the database itself:
// the pointer is only ever put on it once it has passed its checks, and removing what the
// pointer names would leave nothing to open at all.
if (await readCurrentDatabaseName() !== candidate) {
await removeQuietly(target.pool, candidate);
}
report({ stage: "failed", error: failure.message, reason: failure.reason });
throw failure;
}
}
/** A failure with a reason attached, so the setup screen can tell the cases apart. */
export class RestoreFailure extends Error {
constructor(readonly reason: string, message: string, options?: ErrorOptions) {
super(message, options);
this.name = "RestoreFailure";
}
}
/**
* Writes the backup into the pool as the candidate, unwrapping it on the way where it is a
* container.
*
* The unwrap writes into one end of a pipe while the import pulls from the other, so the database
* passes through in chunks and the pipe's own backpressure keeps the two in step. Neither end ever
* holds more than a chunk.
*/
async function stageCandidate(
pool: SAHPoolUtil,
candidate: string,
backup: Blob,
passphrase: string | undefined,
onProgress: ProgressCallback
): Promise<void> {
await makeRoomFor(pool);
await removeQuietly(pool, candidate);
const format = await readBackupFormat(backup);
if (!format) {
throw new RestoreFailure("not-a-database", "The backup could not be read.");
}
if (!format.container) {
await pool.importDb(candidate, pullFrom(readInSlices(backup)));
onProgress(1);
return;
}
// Asked before the file is read rather than after: an encrypted container without a passphrase
// fails the same way either way, but only one of them takes minutes to get there.
if (format.encrypted && !passphrase) {
throw new RestoreFailure("passphrase-required", "The backup is encrypted.");
}
const relay = new TransformStream<Uint8Array, Uint8Array>();
const unwrapped = readBackupContainer(readInSlices(backup), relay.writable, { passphrase, onProgress });
const imported = pool.importDb(candidate, pullFrom(relay.readable));
// Awaited together: either failing has to stop the other, which is what a rejected pipe does.
await Promise.all([ unwrapped, imported ]);
}
/**
* Opens the candidate and puts it through the checks every platform shares, minus the scan.
*
* `PRAGMA quick_check` reads every page, and here every page comes back through the pool's
* synchronous access handles into a WebAssembly database engine, which is the slowest reader of the
* three platforms and the one with a user watching a setup screen that cannot say how far along it
* is. What it would find is largely accounted for already: a container's payload is checked against
* a SHA-256 recorded when the backup was written, and any file is opened, its schema parsed and its
* options read before it is accepted. The remaining case, a plain `.db` damaged somewhere no schema
* read reaches, surfaces the way it would in any other Trilium, which never scans its database
* either except when asked to from the options screen.
*/
function validate(pool: SAHPoolUtil, candidate: string): DatabaseValidation {
const db = new pool.OpfsSAHPoolDb(candidate);
try {
return validateDatabase(candidateOf(db), { skipIntegrityCheck: true });
} catch (e) {
// Not a verdict on the backup. The checks call a database damaged by looking at it and
// finding it so; this is the looking itself having failed, which says nothing about what
// was being looked at. Calling it damaged because the device ran out of memory, or because
// the browser took the pool's handles away, is how a perfectly good backup gets thrown out.
throw new RestoreFailure("check-failed", messageOf(e), { cause: e });
} finally {
db.close();
}
}
/** The three questions the checks ask, answered by a database opened from the pool. */
function candidateOf(db: InstanceType<SAHPoolUtil["OpfsSAHPoolDb"]>): CandidateDatabase {
return {
integrityCheck: () => String(db.selectValue("PRAGMA quick_check")),
tableNames: () => db.selectValues("SELECT name FROM sqlite_master WHERE type = 'table'") as string[],
option: (name) => db.selectValue("SELECT value FROM options WHERE name = ?", [ name ]) as string | undefined
};
}
/**
* Makes the candidate the live database.
*
* The pointer is written last, and is the only thing that decides which database is opened: until it
* changes, an interrupted restore leaves the old one live with an unused entry beside it, which the
* next attempt overwrites.
*/
async function swapIn(target: RestoreTarget, previous: string, candidate: string): Promise<void> {
target.close();
try {
// Inside the rollback's reach, because a pointer half-written is the one failure that could
// leave a start reading neither name.
await writeCurrentDatabaseName(candidate);
await openWithHandles(target, candidate);
} catch (e) {
// Put back what was live, so a candidate that will not open costs nothing.
throw await rollBackTo(target, previous, e);
}
// Only once the restored database is open: until then the old one is what a restart needs.
await removeQuietly(target.pool, previous);
}
/**
* Opens `dbName`, taking the pool's access handles back first where the browser has closed them.
*
* Retried once, and only for that: a database that will not open for any other reason will not open
* the second time either, and the first failure is the one worth reporting.
*/
async function openWithHandles(target: RestoreTarget, dbName: string): Promise<void> {
try {
target.open(dbName);
} catch (e) {
if (!isClosedAccessHandle(e)) {
throw e;
}
await reacquireAccessHandles(target.pool);
target.open(dbName);
}
}
/**
* Puts the previous database back after a swap that did not take.
*
* The pointer is the whole of what decides which database a start opens, so restoring it is what
* makes a failed swap harmless: whatever happens next, the notes are the ones that were here before.
* Reopening it is the part that can fail again, since the handles the reopen needs may be the very
* thing that was taken, and that failure must not replace the one being reported or the screen would
* explain the wrong problem.
*/
async function rollBackTo(
target: RestoreTarget,
previous: string,
cause: unknown
): Promise<RestoreFailure> {
try {
await writeCurrentDatabaseName(previous);
} catch {
// The pointer is whatever it was left as, which may be the candidate. That one has passed
// its checks, so a start finding it is not in danger; it is simply not the database this
// instance was told to keep, and only a start can settle which it opens.
return new RestoreFailure("swap-failed-reload", messageOf(cause));
}
try {
await openWithHandles(target, previous);
} catch {
// The pointer is back, so a start comes up on the database that was here all along. This
// instance simply cannot get there without one, which is the one thing the user has to be
// told rather than left to discover.
return new RestoreFailure("swap-failed-reload", messageOf(cause));
}
return new RestoreFailure("swap-failed", messageOf(cause));
}
/**
* Runs `work`, and where the browser closed the pool's handles underneath it, takes them back and
* runs it once more.
*
* This is for the part of a restore that happens before the swap, where the database being replaced
* is still open and serving the screen behind all this. Taking the handles back means letting go of
* it for a moment, since the pool refuses to be paused while anything is open at all, so it is
* closed and opened again around the recovery.
*
* The retry starts the work over rather than resuming it, which for the import means reading the
* backup from the beginning. That is worth it against the alternative: without it a restore that
* was suspended halfway is simply lost, and the suspension is usually the user coming back to the
* application rather than leaving it.
*
* @param live the database to put back, which is the one this restore has not replaced yet.
*/
async function keepingHandles<T>(
target: RestoreTarget,
live: string,
work: () => Promise<T>
): Promise<T> {
try {
return await work();
} catch (e) {
if (!isClosedAccessHandle(e)) {
throw e;
}
try {
target.close();
await reacquireAccessHandles(target.pool);
target.open(live);
} catch {
// The handles could not be had back, so the failure to report is the one that was
// actually hit rather than whatever went wrong trying to recover from it.
throw e;
}
return await work();
}
}
/**
* Whether a failure is the browser having closed the pool's access handles underneath it.
*
* WebKit closes them whenever it suspends the page: a file picker, a screen lock, a switch to
* another application. The pool takes its handles once and holds them for its lifetime, so it never
* learns they are gone, and the next operation against it fails with this.
*/
function isClosedAccessHandle(e: unknown): boolean {
if (e instanceof DOMException && e.name === "InvalidStateError") {
return true;
}
// Through whatever wrapped it. A step reports a failure in its own terms, and it has no business
// deciding on the way whether the browser's error underneath is one that can be recovered from.
if (e instanceof Error && e.cause !== undefined && isClosedAccessHandle(e.cause)) {
return true;
}
return /access\s*handle.*closed|closed.*access\s*handle/i.test(messageOf(e));
}
/**
* Takes the pool's access handles back.
*
* The pause is the part that matters, and `isPaused` is no help in deciding whether to do it: it
* reports what the pool did to itself, and a pool whose handles were taken from underneath it still
* believes it holds them, so `unpauseVfs` on its own returns having done nothing. Pausing is what
* clears that belief. It is refused while any database is open, which is why this belongs to the
* moment between closing one and opening the next.
*/
async function reacquireAccessHandles(pool: SAHPoolUtil): Promise<void> {
pool.pauseVfs();
await pool.unpauseVfs();
}
/**
* Erases the live database and leaves an empty one in its place.
*
* What the setup wizard does when the user asks for it. The pool has no notion of "no database", and
* everything after this point in the wizard writes to one — creating a document, converging a sync —
* so the entry is unlinked and a fresh one opened under the name a first run uses, which is exactly
* the state a browser that had never run Trilium is in.
*
* Beside the restore because it is the same dance: close, exchange, open. The order below is what
* keeps a failure part-way through from leaving the two halves of "which database is live"
* disagreeing, and each step says what it costs when it is the one that fails.
*
* @throws Error where the pointer could not be moved, having erased nothing.
*/
export async function eraseDatabase(target: RestoreTarget): Promise<void> {
const live = await readCurrentDatabaseName();
target.close();
// Which entry the caller will be holding when this returns. It moves only once the pointer that
// outlives this worker agrees, so the two can never come out of here naming different databases:
// the next start reads that pointer, and running on one entry while the pointer names another
// would silently strand everything written in between.
let opening = live;
try {
// The pointer first, and the unlink second. The other way round, a pointer write that failed
// would leave it naming an entry that no longer exists while this worker ran on the default
// one. A pointer that is moved and then not followed through only leaks the old entry.
await writeCurrentDatabaseName(DEFAULT_DATABASE_NAME);
opening = DEFAULT_DATABASE_NAME;
try {
target.pool.unlink(live);
} catch {
// Already gone, which is the state the caller asked for.
}
} finally {
// Whatever happened above, the caller has to come back holding a database: everything after
// this point writes to one, and a detached service answers every request with "DB not open"
// for as long as this worker lives. The pool creates the entry where there is none, so this
// opens either the one just cleared out, or the one nothing got around to clearing.
target.open(opening);
}
}
/** Points the next start at `dbName`, which is the whole of what makes a database the live one. */
export async function writeCurrentDatabaseName(dbName: string): Promise<void> {
const root = await navigator.storage.getDirectory();
const handle = await root.getFileHandle(POINTER_FILE, { create: true });
const writable = await handle.createWritable();
try {
await writable.write(dbName);
} finally {
await writable.close();
}
}
/**
* Identifies a backup from its first bytes: a container states what it is in the clear, and anything
* else has to be a database already.
*/
async function readBackupFormat(backup: Blob): Promise<{ container: boolean; encrypted: boolean } | null> {
const head = new Uint8Array(await backup.slice(0, FIXED_HEADER_BYTES).arrayBuffer());
const container = getInfo(head);
if (container.isValid && container.isSupported) {
requireWholeContainer(container, backup.size);
return { container: true, encrypted: container.isEncrypted };
}
return looksLikeSqlite(head) ? { container: false, encrypted: false } : null;
}
/**
* Refuses a container the file is too small to hold, before a minute of work goes into it.
*
* Only an uncompressed one can be measured this way: its length follows exactly from the size its
* header records. Compression breaks the derivation, since the payload is then shorter than the
* plaintext by a ratio nothing states in advance.
*/
function requireWholeContainer(container: SupportedBackupContainer, size: number): void {
if (container.isCompressed || container.size <= 0) {
return;
}
const expected = containerSize(container.size, container.isEncrypted);
if (size < expected) {
throw incompleteBackup(size, expected);
}
}
/**
* Reads a file a slice at a time, rather than as one read that lives for the whole of it.
*
* `Blob.stream()` is that one long read, and a browser is free to end it early. A low-memory Android
* WebView was seen doing exactly that with a complete 4.8 GB backup: the same byte every time, no
* error, just an end of input that a reader can only take at its word — which surfaced as a container
* that had run out of frames. Asking for a bounded range at a time is what the chunked upload has
* always done with files this size on the same hardware, and it is how the header below is read.
*
* Reaching the end before the file says it should is a failure here rather than silence, so a file
* that really is short says so in its own terms instead of through whatever was parsing it.
*/
export function readInSlices(blob: Blob, sliceSize = SLICE_BYTES): ReadableStream<Uint8Array> {
let offset = 0;
return new ReadableStream<Uint8Array>({
async pull(controller) {
if (offset >= blob.size) {
controller.close();
return;
}
const end = Math.min(offset + sliceSize, blob.size);
const slice = new Uint8Array(await blob.slice(offset, end).arrayBuffer());
if (slice.length === 0) {
throw incompleteBackup(offset, blob.size);
}
offset += slice.length;
controller.enqueue(slice);
}
});
}
/**
* How much is asked for at once: large enough that the per-read cost is lost against the work done
* with it, small enough that a device with very little memory never holds much. The container reads
* in frames a quarter of this size, and the stream keeps one slice queued.
*/
const SLICE_BYTES = 4 * 1024 * 1024;
/** A file that stops before it should have, said in bytes because that is what makes it checkable. */
function incompleteBackup(read: number, expected: number): RestoreFailure {
return new RestoreFailure(
"backup-incomplete",
`The backup file ends after ${read} bytes, short of the ${expected} it should hold. `
+ "It was probably not copied or downloaded in full."
);
}
/** Turns a stream into the pull `importDb` asks for: a chunk each call, nothing at the end. */
function pullFrom(stream: ReadableStream<Uint8Array>): () => Promise<Uint8Array | undefined> {
const reader = stream.getReader();
return async () => {
const { done, value } = await reader.read();
return done ? undefined : value;
};
}
/** Adds a slot if every one the pool has is spoken for, since a restore needs one for the candidate. */
async function makeRoomFor(pool: SAHPoolUtil): Promise<void> {
if (pool.getFileCount() >= pool.getCapacity()) {
await pool.addCapacity(1);
}
}
/** Removes a pool entry, if it is there at all. Never the reason a restore fails. */
async function removeQuietly(pool: SAHPoolUtil, dbName: string): Promise<void> {
try {
pool.unlink(dbName);
} catch {
// A leftover entry costs a slot, which is not worth losing the answer over.
}
}
function asFailure(e: unknown): RestoreFailure {
if (e instanceof RestoreFailure) {
return e;
}
if (isBackupContainerError(e)) {
return new RestoreFailure(e.reason, e.message);
}
// Not "swap-failed": everything that reaches here failed before the swap was reached, and
// saying otherwise would tell the user their database had been replaced and put back when it
// was never touched.
return new RestoreFailure("restore-failed", messageOf(e));
}
function messageOf(e: unknown): string {
return e instanceof Error ? e.message : String(e);
}