Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
import { writeBackupContainer } from "@triliumnext/backup-container";
// `sql_init` from core rather than the server's own re-export, which copies the function references
// onto a fresh object: a spy on that one would not be seen by `database_restore`.
import { app_info as appInfo, getBackup, getLog, getSql, sql_init as sqlInit } from "@triliumnext/core";
import Database from "better-sqlite3";
import fs from "fs";
import os from "os";
import path from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type ServerBackupService from "../backup_provider.js";
import dataDir from "./data_dir.js";
import {
adoptBackupPassphrase,
exchangeDatabaseFiles,
getRestoreProgress,
logRestore,
logRestoreError,
reportStepProgress,
readBackupFormat,
removeQuietly,
reportRestoreFailure,
restoreDatabase,
RestoreFailure,
type RestoreRequest,
stageBackup
} from "./database_restore.js";
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-restore-spec-"));
let counter = 0;
beforeEach(() => {
fs.rmSync(tempRoot, { recursive: true, force: true });
fs.mkdirSync(tempRoot, { recursive: true });
// The state a restore actually runs in: the setup screen, with no database of this instance's
// own open. `restoreDatabase` refuses anywhere else, and the shared fixture is initialized.
vi.spyOn(sqlInit, "isDbInitialized").mockReturnValue(false);
});
afterEach(() => vi.restoreAllMocks());
/** A database that would pass validation, so a test can tell a rejected file from a rejected step. */
function validDatabase(name = `db-${counter++}.db`) {
const filePath = path.join(tempRoot, name);
const db = new Database(filePath);
try {
for (const table of [ "options", "notes", "branches", "blobs" ]) {
db.exec(`CREATE TABLE ${table} (name TEXT, value TEXT)`);
}
const insert = db.prepare("INSERT INTO options (name, value) VALUES (?, ?)");
insert.run("initialized", "true");
insert.run("dbVersion", String(appInfo.dbVersion));
} finally {
db.close();
}
return filePath;
}
/** Wraps `source` the way a backup does, with the same writer the backup service uses. */
async function containerOf(source: string, passphrase?: string, compress = false) {
const filePath = path.join(tempRoot, `backup-${counter++}.tnbackup`);
await writeBackupContainer(fs.createReadStream(source), fs.createWriteStream(filePath), {
compress,
passphrase,
plaintextSize: fs.statSync(source).size
});
return filePath;
}
function fileWith(content: string, name = `file-${counter++}.bin`) {
const filePath = path.join(tempRoot, name);
fs.writeFileSync(filePath, content);
return filePath;
}
describe("reading what a backup file is", () => {
it("tells a container from a plain database, going by the header rather than the name", async () => {
const database = validDatabase("looks-like-a-backup.tnbackup");
expect(readBackupFormat(database)).toEqual({ container: false, encrypted: false });
const plain = await containerOf(validDatabase());
expect(readBackupFormat(plain)).toEqual({ container: true, encrypted: false });
const encrypted = await containerOf(validDatabase(), "hunter2");
expect(readBackupFormat(encrypted)).toEqual({ container: true, encrypted: true });
});
it("answers for a file it cannot read at all", () => {
expect(readBackupFormat(path.join(tempRoot, "not-here.db"))).toBe(null);
expect(readBackupFormat(fileWith("short"))).toEqual({ container: false, encrypted: false });
});
});
describe("staging a backup", () => {
it("takes an uploaded database where it lies rather than copying gigabytes for no reason", async () => {
const uploaded = validDatabase();
const candidate = await stageBackup({ path: uploaded, fileName: "backup.db", consumable: true });
expect(candidate).toBe(uploaded);
});
it("copies a backup that has to stay where it is, leaving the original untouched", async () => {
const existing = validDatabase();
const candidate = await stageBackup({ path: existing, fileName: "backup.db", consumable: false });
expect(candidate).not.toBe(existing);
expect(fs.existsSync(existing)).toBe(true);
expect(fs.readFileSync(candidate)).toEqual(fs.readFileSync(existing));
});
it("unwraps a container back into the database it was made from", async () => {
const source = validDatabase();
const backup = await containerOf(source, undefined, true);
const candidate = await stageBackup({ path: backup, fileName: "backup.tnbackup", consumable: true });
expect(fs.readFileSync(candidate)).toEqual(fs.readFileSync(source));
// Consumable: the container has served its purpose and the unwrapped copy is what matters now.
expect(fs.existsSync(backup)).toBe(false);
});
it("asks for a passphrase before reading an encrypted container, not after", async () => {
const backup = await containerOf(validDatabase(), "hunter2");
const readStream = vi.spyOn(fs, "createReadStream");
await expect(stageBackup({ path: backup, fileName: "backup.tnbackup", consumable: false }))
.rejects.toMatchObject({ reason: "passphrase-required" });
expect(readStream).not.toHaveBeenCalled();
// Nothing was consumed, so the same file can be tried again once the passphrase is known.
expect(fs.existsSync(backup)).toBe(true);
});
it("tells a wrong passphrase apart from a file that is damaged beyond it", async () => {
const backup = await containerOf(validDatabase(), "hunter2");
await expect(stageBackup({ path: backup, fileName: "b.tnbackup", consumable: true, passphrase: "wrong" }))
.rejects.toMatchObject({ reason: "wrong-passphrase-or-damaged-header" });
// Kept, because another passphrase is exactly what could still make it work.
expect(fs.existsSync(backup)).toBe(true);
});
it("closes both files when the unwrap fails, so what is left can be deleted and replaced", async () => {
// Windows refuses to delete or overwrite an open file, so a leaked handle turns one wrong
// password into a staging directory that cannot be cleared (ENOTEMPTY) and an uploaded
// backup the next attempt cannot replace (EPERM).
const backup = await containerOf(validDatabase(), "hunter2");
const opened = vi.spyOn(fs, "createReadStream");
const written = vi.spyOn(fs, "createWriteStream");
await expect(stageBackup({ path: backup, fileName: "b.tnbackup", consumable: false, passphrase: "wrong" }))
.rejects.toMatchObject({ reason: "wrong-passphrase-or-damaged-header" });
expect(opened.mock.results[0].value.closed).toBe(true);
expect(written.mock.results[0].value.closed).toBe(true);
// Which is what the caller then relies on, on both counts.
expect(() => fs.rmSync(path.dirname(String(written.mock.calls[0][0])), { recursive: true })).not.toThrow();
expect(() => fs.rmSync(backup)).not.toThrow();
});
it("refuses a file it cannot even open", async () => {
await expect(stageBackup({ path: path.join(tempRoot, "gone.db"), fileName: "gone.db", consumable: false }))
.rejects.toMatchObject({ reason: "not-a-database" });
});
});
describe("what the log is told", () => {
/** Everything the restore said, as one body of text to look through. */
function transcript(lines: string[]) {
return lines.join("\n");
}
it("prefixes every line, so one restore can be picked out of a log by searching for it", () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
vi.spyOn(getLog(), "error").mockImplementation((message) => said.push(String(message)));
logRestore("something happened");
logRestoreError("something went wrong");
expect(said).toEqual([ "Backup restore: something happened", "Backup restore: something went wrong" ]);
});
it("keeps the filesystem out of what it quotes back, so the log can be handed over unread", () => {
const said: string[] = [];
vi.spyOn(getLog(), "error").mockImplementation((message) => said.push(String(message)));
// The shape a filesystem error arrives in: our own words, and a path we did not write.
logRestoreError(`the exchange failed: EPERM, Permission denied: ${dataDir.DOCUMENT_PATH}`);
logRestoreError(`a temporary file would not go away: ENOTEMPTY: ${dataDir.TMP_DIR}`);
expect(transcript(said)).not.toContain(dataDir.TRILIUM_DATA_DIR);
expect(said[0]).toContain("<database>");
expect(said[1]).toContain("<temporary files>");
});
it("leaves ordinary words alone, however the data directory happens to be configured", () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
// A relatively configured data directory ("TRILIUM_DATA_DIR=data", which the development
// scripts use) was replaced as a bare substring, turning every "database" into
// "<data directory>base" and making the log read as nonsense.
logRestore("detaching the database being replaced");
expect(said[0]).toContain("detaching the database being replaced");
});
it("names every step it enters and where it stopped, without naming the backup", async () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
vi.spyOn(getLog(), "error").mockImplementation((message) => said.push(String(message)));
await expect(restoreDatabase({
path: fileWith("a photograph, renamed"),
fileName: "holidays-in-crete.db",
consumable: false
})).rejects.toBeInstanceOf(RestoreFailure);
const log = transcript(said);
expect(log).toContain("Backup restore: starting");
// Staged without complaint — any readable file can be copied — and refused by the check that
// opens it, which is the distinction the step in this line is there to draw.
expect(log).toContain("failed at the 'validating' step");
expect(log).toContain("not-a-database");
// The name is the user's, and says nothing a diagnosis needs.
expect(log).not.toContain("holidays-in-crete");
});
it("reports how far a long step has got, in brackets, once per tenth of the way", () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
const report = reportStepProgress("unwrapping the backup");
// What the container module sends: a position at whatever rate its own throttle allows.
for (const fraction of [ 0, 0.02, 0.05, 0.09, 0.1, 0.15, 0.2, 0.99, 1 ]) {
report(fraction);
}
expect(said).toEqual([
"Backup restore: unwrapping the backup [0%]",
"Backup restore: unwrapping the backup [10%]",
"Backup restore: unwrapping the backup [20%]",
"Backup restore: unwrapping the backup [99%]",
"Backup restore: unwrapping the backup [100%]"
]);
});
it("says nothing more while a step is not getting anywhere, however often it is asked", () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
const report = reportStepProgress("unwrapping the backup");
report(0.42);
for (let i = 0; i < 50; i++) {
report(0.42);
}
// A stall is told apart from slow going by the log not growing, which it cannot do if every
// report writes a line.
expect(said).toEqual([ "Backup restore: unwrapping the backup [42%]" ]);
});
it("reports the progress of a real unwrap", async () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
const backup = await containerOf(validDatabase());
await stageBackup({ path: backup, fileName: "backup.tnbackup", consumable: false });
expect(transcript(said)).toContain("unwrapping the backup [100%]");
});
it("puts the position where the screen can see it, every time rather than every tenth", () => {
vi.spyOn(getLog(), "info").mockImplementation(() => {});
reportRestoreFailure("backup.db", new Error("so that there is a progress to carry it"));
const report = reportStepProgress("unwrapping the backup");
report(0.42);
expect(getRestoreProgress()?.fraction).toBe(0.42);
// Between two logged tenths, and still the latest thing the screen is told.
report(0.47);
expect(getRestoreProgress()?.fraction).toBe(0.47);
});
it("says which steps a restore got through, so one that stops says where", async () => {
const said: string[] = [];
vi.spyOn(getLog(), "info").mockImplementation((message) => said.push(String(message)));
vi.spyOn(getLog(), "error").mockImplementation(() => {});
const detach = vi.spyOn(getSql(), "detachConnection").mockImplementation(() => {
throw new Error("the database is in the middle of a transaction");
});
await expect(restoreDatabase({ path: validDatabase(), fileName: "backup.db", consumable: false }))
.rejects.toBeInstanceOf(RestoreFailure);
// A step's position belongs to that step, so it is not left showing under the next one.
expect(getRestoreProgress()?.fraction).toBeUndefined();
const log = transcript(said);
expect(detach).toHaveBeenCalled();
expect(log).toContain("the backup is a plain database");
expect(log).toContain("step 'validating'");
expect(log).toContain("database version");
expect(log).toContain("step 'swapping'");
// It never got as far as opening anything.
expect(log).not.toContain("step 'migrating'");
});
});
describe("removing what is no longer wanted", () => {
it("removes a file, and a directory when it is told to", () => {
const file = fileWith("temporary");
const directory = path.join(tempRoot, "staging");
fs.mkdirSync(directory, { recursive: true });
fs.writeFileSync(path.join(directory, "candidate.db"), "half a database");
removeQuietly(file);
removeQuietly(directory, { recursive: true });
expect(fs.existsSync(file)).toBe(false);
expect(fs.existsSync(directory)).toBe(false);
});
it("says nothing about something that was never there", () => {
const errors = vi.spyOn(getLog(), "error");
removeQuietly(path.join(tempRoot, "never-existed"));
expect(errors).not.toHaveBeenCalled();
});
it("logs and carries on when the file will not go, rather than raising", () => {
const errors = vi.spyOn(getLog(), "error").mockImplementation(() => {});
vi.spyOn(fs, "rmSync").mockImplementation(() => {
throw new Error("EPERM, Permission denied");
});
expect(() => removeQuietly(fileWith("locked"))).not.toThrow();
expect(errors).toHaveBeenCalledWith(expect.stringContaining("EPERM"));
});
});
describe("exchanging the database files", () => {
it("puts the candidate in place, keeps the old database aside, and drops its sidecars", () => {
const document = path.join(tempRoot, "document.db");
fs.writeFileSync(document, "the old database");
fs.writeFileSync(`${document}-wal`, "old write-ahead log");
fs.writeFileSync(`${document}-shm`, "old shared memory");
const candidate = fileWith("the restored database");
exchangeDatabaseFiles(candidate, document);
expect(fs.readFileSync(document, "utf-8")).toBe("the restored database");
expect(fs.readFileSync(`${document}.pre-restore`, "utf-8")).toBe("the old database");
expect(fs.existsSync(`${document}-wal`)).toBe(false);
expect(fs.existsSync(`${document}-shm`)).toBe(false);
expect(fs.existsSync(candidate)).toBe(false);
});
it("works where there is no database yet, which is the usual case during setup", () => {
const document = path.join(tempRoot, "document.db");
exchangeDatabaseFiles(fileWith("the restored database"), document);
expect(fs.readFileSync(document, "utf-8")).toBe("the restored database");
expect(fs.existsSync(`${document}.pre-restore`)).toBe(false);
});
it("replaces what an earlier restore had set aside rather than refusing to start", () => {
const document = path.join(tempRoot, "document.db");
fs.writeFileSync(document, "the current database");
fs.writeFileSync(`${document}.pre-restore`, "from an earlier attempt");
exchangeDatabaseFiles(fileWith("the restored database"), document);
expect(fs.readFileSync(`${document}.pre-restore`, "utf-8")).toBe("the current database");
});
});
describe("recovering from an interrupted restore", () => {
/** Imported lazily so the default path is bound per call rather than at module load. */
async function recover(document: string) {
const { recoverInterruptedRestore } = await import("./database_restore.js");
vi.spyOn(console, "info").mockImplementation(() => {});
recoverInterruptedRestore(document);
}
it("does nothing when no restore was under way", async () => {
const document = path.join(tempRoot, "document.db");
fs.writeFileSync(document, "the database");
await recover(document);
expect(fs.readFileSync(document, "utf-8")).toBe("the database");
});
it("puts the previous database back when one was set aside", async () => {
const document = path.join(tempRoot, "document.db");
fs.writeFileSync(document, "half a restored database");
fs.writeFileSync(`${document}-wal`, "a log belonging to neither");
fs.writeFileSync(`${document}.pre-restore`, "the database it had");
fs.writeFileSync(`${document}.restore-in-progress`, "");
await recover(document);
expect(fs.readFileSync(document, "utf-8")).toBe("the database it had");
expect(fs.existsSync(`${document}-wal`)).toBe(false);
expect(fs.existsSync(`${document}.pre-restore`)).toBe(false);
expect(fs.existsSync(`${document}.restore-in-progress`)).toBe(false);
});
it("keeps the database in place when nothing was set aside, and clears the marker", async () => {
const document = path.join(tempRoot, "document.db");
fs.writeFileSync(document, "a whole database");
fs.writeFileSync(`${document}.restore-in-progress`, "");
await recover(document);
expect(fs.readFileSync(document, "utf-8")).toBe("a whole database");
expect(fs.existsSync(`${document}.restore-in-progress`)).toBe(false);
});
});
describe("restoring a database", () => {
it("refuses outright on an instance that already has a database", async () => {
// The route this is reached through is guarded by `checkAppNotInitialized`, and until this
// check existed that middleware was the only thing between a running instance and a restore
// over the top of it. Every other way of replacing a database refuses on its own account.
vi.spyOn(sqlInit, "isDbInitialized").mockReturnValue(true);
const detach = vi.spyOn(getSql(), "detachConnection").mockImplementation(() => {});
await expect(restoreDatabase({
path: validDatabase(),
fileName: "holiday.db",
consumable: false
})).rejects.toMatchObject({ reason: "restore-refused" });
expect(detach).not.toHaveBeenCalled();
});
it("stops before touching the live database when the backup is not one", async () => {
const detach = vi.spyOn(getSql(), "detachConnection").mockImplementation(() => {});
await expect(restoreDatabase({
path: fileWith("a photograph, renamed"),
fileName: "holiday.db",
consumable: false
})).rejects.toBeInstanceOf(RestoreFailure);
expect(detach).not.toHaveBeenCalled();
expect(getRestoreProgress()).toMatchObject({
stage: "failed", fileName: "holiday.db", reason: "not-a-database"
});
});
it("reports why it failed even when the tidying up afterwards fails too", async () => {
// A throw from the `finally` would replace the failure on its way out, and the client would
// be told the temporary directory would not delete rather than that the password was wrong.
vi.spyOn(fs, "rmSync").mockImplementation(() => {
throw new Error("ENOTEMPTY, Directory not empty");
});
await expect(restoreDatabase({
path: fileWith("a photograph, renamed"),
fileName: "holiday.db",
consumable: false
})).rejects.toMatchObject({ reason: "not-a-database" });
expect(getRestoreProgress()).toMatchObject({ stage: "failed", reason: "not-a-database" });
});
it("records a failure that happened instead of a restore, so nothing waits on a run that never began", () => {
reportRestoreFailure("backup.db", new Error("setup is already busy with 'new-document'"));
expect(getRestoreProgress()).toEqual({
stage: "failed",
fileName: "backup.db",
error: "setup is already busy with 'new-document'",
reason: "restore-refused"
});
});
it("stops before touching the live database when the backup is from a newer version", async () => {
const detach = vi.spyOn(getSql(), "detachConnection").mockImplementation(() => {});
const database = validDatabase();
const db = new Database(database);
db.prepare("UPDATE options SET value = ? WHERE name = 'dbVersion'").run(String(appInfo.dbVersion + 1));
db.close();
await expect(restoreDatabase({ path: database, fileName: "newer.db", consumable: false }))
.rejects.toMatchObject({ reason: "database-too-new" });
expect(detach).not.toHaveBeenCalled();
expect(getRestoreProgress()).toMatchObject({ stage: "failed", reason: "database-too-new" });
});
});
describe("the backup password after a restore", () => {
/** What the restore reaches for: the desktop's keyring, by way of the backup service. */
const backupService = () => getBackup() as ServerBackupService;
function request(passphrase?: string): RestoreRequest {
const fileName = "backup.tnbackup";
return { path: fileName, fileName, consumable: false, passphrase };
}
it("takes on the password that opened the backup, which is this database's now", async () => {
const adopted = vi.spyOn(backupService(), "adoptPassphrase").mockResolvedValue();
await adoptBackupPassphrase(request("the newer database's password"), true);
expect(adopted).toHaveBeenCalledWith("the newer database's password");
});
it.each([
[ "an unlocked backup", undefined ],
[ "a password typed for a backup that turned out not to need one", "typed anyway" ]
])("lets go of the stored password after %s", async (_label, passphrase) => {
// The stored one belongs to the database that was here before. Kept, this instance goes on
// encrypting with it and writes backups the password the user expects will not open.
const adopted = vi.spyOn(backupService(), "adoptPassphrase").mockResolvedValue();
await adoptBackupPassphrase(request(passphrase), false);
expect(adopted).toHaveBeenCalledWith(null);
});
it("does not fail a restore that already happened over a keyring that will not answer", async () => {
const said: string[] = [];
vi.spyOn(getLog(), "error").mockImplementation((message) => said.push(String(message)));
vi.spyOn(backupService(), "adoptPassphrase")
.mockRejectedValue(new Error("the keyring is locked"));
// The database is in place and open by this point; there is nothing left to undo and no
// honest way to call the restore failed.
await expect(adoptBackupPassphrase(request("a password"), true)).resolves.toBeUndefined();
expect(said.join("\n")).toContain("the keyring is locked");
});
});