Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

web: make shared sqlite more robust

Abdullah Attaebe3f2b771b0
diff --git a/apps/web/src/common/sqlite/shared-service.ts b/apps/web/src/common/sqlite/shared-service.ts
index 400398876..0fb5f8518 100644
--- a/apps/web/src/common/sqlite/shared-service.ts
+++ b/apps/web/src/common/sqlite/shared-service.ts
@@ -38,7 +38,13 @@ export class SharedService<T extends object> extends EventTarget {
#providerPort: Promise<MessagePort | null>;
providerCallbacks: Map<
string,
- { resolve: (result: unknown) => void; reject: (reason: unknown) => void }
+ {
+ resolve: (result: unknown) => void;
+ reject: (reason: unknown) => void;
+ nonce: string;
+ method: string | symbol;
+ args: any[];
+ }
> = new Map();
#providerCounter = 0;
#providerChangeCleanup: (() => void)[] = [];
@@ -61,8 +67,9 @@ export class SharedService<T extends object> extends EventTarget {
) {
// A context (possibly this one) announced itself as the new provider.
// Discard any old provider and connect to the new one.
- this.#closeProviderPort(this.#providerPort);
+ this.#providerPort.then((port) => port?.close());
this.#providerPort = this.#providerChange();
+ await this.#resendPendingCallbacks();
}
},
{ signal: this.#onClose.signal }
@@ -71,7 +78,10 @@ export class SharedService<T extends object> extends EventTarget {
this.proxy = this.#createProxy();
}
- activate(portProviderFunc: () => MessagePort | Promise<MessagePort>) {
+ activate(
+ portProviderFunc: () => MessagePort | Promise<MessagePort>,
+ onClientConnected: () => Promise<void>
+ ) {
if (this.#onDeactivate) return;
// When acquire a lock on the service name then we become the service
@@ -80,10 +90,8 @@ export class SharedService<T extends object> extends EventTarget {
this.#onDeactivate = new AbortController();
const LOCK_NAME = `SharedService-${this.serviceName}`;
- navigator.locks.request(
- LOCK_NAME,
- { signal: this.#onDeactivate.signal },
- async () => {
+ navigator.locks
+ .request(LOCK_NAME, { signal: this.#onDeactivate.signal }, async () => {
// Get the port to request client ports.
const port = await portProviderFunc();
port.start();
@@ -93,12 +101,16 @@ export class SharedService<T extends object> extends EventTarget {
// request.
const providerId = await this.#clientId;
const broadcastChannel = new BroadcastChannel("SharedService");
+
+ // eslint-disable-next-line @typescript-eslint/no-this-alias
+ const thisArg = this;
broadcastChannel.addEventListener(
"message",
- async ({ data }) => {
+ async function onMessage(event) {
+ const { data } = event;
if (
data?.type === "request" &&
- data?.sharedService === this.serviceName
+ data?.sharedService === thisArg.serviceName
) {
// Get a port to send to the client.
const requestedPort = await new Promise<MessagePort>(
@@ -112,7 +124,22 @@ export class SharedService<T extends object> extends EventTarget {
}
);
- this.#sendPortToClient(data, requestedPort);
+ if (data.clientId !== providerId) {
+ await onClientConnected();
+ }
+
+ try {
+ thisArg.#sendPortToClient(data, requestedPort);
+ } catch (e) {
+ console.error(e, providerId, data);
+ // retry if port has been neutered, this can happen when
+ // closing the provider tab/window
+ if (e instanceof Error && e.message.includes("neutered")) {
+ console.warn("Retrying in 100ms...");
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ onMessage.bind(broadcastChannel, event)();
+ } else throw e;
+ }
}
},
{ signal: this.#onDeactivate?.signal }
@@ -132,8 +159,8 @@ export class SharedService<T extends object> extends EventTarget {
reject(this.#onDeactivate?.signal.reason);
});
});
- }
- );
+ })
+ .catch(console.error);
}
deactivate() {
@@ -142,14 +169,14 @@ export class SharedService<T extends object> extends EventTarget {
}
close() {
- this.deactivate();
- this.#onClose.abort();
for (const { reject } of this.providerCallbacks.values()) {
- reject(new Error("SharedService closed"));
+ reject(new Error("SharedService closed."));
}
+ this.deactivate();
+ this.#onClose.abort();
}
- async #sendPortToClient(message: any, port: MessagePort) {
+ #sendPortToClient(message: any, port: MessagePort) {
sharedWorker?.port.postMessage(message, [port]);
}
@@ -271,10 +298,10 @@ export class SharedService<T extends object> extends EventTarget {
}
}
- #closeProviderPort(providerPort: Promise<MessagePort | null>) {
- providerPort.then((port) => port?.close());
- for (const { reject } of this.providerCallbacks.values()) {
- reject(new Error("SharedService provider change"));
+ async #resendPendingCallbacks() {
+ const port = await this.getProviderPort();
+ for (const { method, args, nonce } of this.providerCallbacks.values()) {
+ port.postMessage({ nonce, method, args });
}
}
@@ -288,7 +315,13 @@ export class SharedService<T extends object> extends EventTarget {
const providerPort = await this.getProviderPort();
return new Promise((resolve, reject) => {
- this.providerCallbacks.set(nonce, { resolve, reject });
+ this.providerCallbacks.set(nonce, {
+ resolve,
+ reject,
+ nonce,
+ method,
+ args
+ });
providerPort.postMessage({ nonce, method, args });
}).finally(() => {
this.providerCallbacks.delete(nonce);
diff --git a/apps/web/src/common/sqlite/sqlite.worker.ts b/apps/web/src/common/sqlite/sqlite.worker.ts
index 942d0f462..243eb51b8 100644
--- a/apps/web/src/common/sqlite/sqlite.worker.ts
+++ b/apps/web/src/common/sqlite/sqlite.worker.ts
@@ -19,15 +19,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import type { SQLiteAPI, SQLiteCompatibleType } from "./sqlite-types";
import { Factory, SQLITE_ROW } from "./sqlite-api";
-import SQLiteAsyncESMFactory from "./wa-sqlite-async";
-import SQLiteSyncESMFactory from "./wa-sqlite";
-import { IDBBatchAtomicVFS } from "./IDBBatchAtomicVFS";
-import { AccessHandlePoolVFS } from "./AccessHandlePoolVFS";
import { transfer } from "comlink";
import type { RunMode } from "./type";
import { QueryResult } from "kysely";
import { DatabaseSource } from "./sqlite-export";
import { createSharedServicePort } from "./shared-service";
+import type { IDBBatchAtomicVFS } from "./IDBBatchAtomicVFS";
+import type { AccessHandlePoolVFS } from "./AccessHandlePoolVFS";
type PreparedStatement = {
stmt: number;
@@ -35,22 +33,29 @@ type PreparedStatement = {
};
let sqlite: SQLiteAPI;
-let db: number;
+let db: number | undefined = undefined;
let vfs: IDBBatchAtomicVFS | AccessHandlePoolVFS | null = null;
+let initialized = false;
const preparedStatements: Map<string, PreparedStatement> = new Map();
+const retryCounter: Record<string, number> = {};
+console.log("new sqlite worker");
-async function init(dbName: string, async: boolean, url?: string) {
- if (db) return;
+async function open(dbName: string, async: boolean, url?: string) {
+ if (db) {
+ console.error("Database is already initialized", db);
+ return;
+ }
const option = url ? { locateFile: () => url } : {};
- const SQLiteAsyncModule = async
- ? await SQLiteAsyncESMFactory(option)
- : await SQLiteSyncESMFactory(option);
- sqlite = Factory(SQLiteAsyncModule);
- vfs = async
- ? new IDBBatchAtomicVFS(dbName, { durability: "strict" })
- : new AccessHandlePoolVFS(dbName);
- if ("isReady" in vfs) await vfs.isReady;
+ const sqliteModule = async
+ ? await import("./wa-sqlite-async").then(
+ ({ default: SQLiteAsyncESMFactory }) => SQLiteAsyncESMFactory(option)
+ )
+ : await import("./wa-sqlite").then(({ default: SQLiteSyncESMFactory }) =>
+ SQLiteSyncESMFactory(option)
+ );
+ sqlite = Factory(sqliteModule);
+ vfs = await getVFS(dbName, async);
sqlite.vfs_register(vfs, false);
db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);
@@ -61,21 +66,39 @@ async function init(dbName: string, async: boolean, url?: string) {
* to avoid unnecessary computations.
*/
async function prepare(sql: string) {
- const cached = preparedStatements.get(sql);
- if (cached !== undefined) return cached;
-
- const str = sqlite.str_new(db, sql);
- const prepared = await sqlite.prepare_v2(db, sqlite.str_value(str));
- if (!prepared) return;
-
- const statement: PreparedStatement = {
- stmt: prepared.stmt,
- columns: sqlite.column_names(prepared.stmt)
- };
- preparedStatements.set(sql, statement);
-
- sqlite.str_finish(str);
- return statement;
+ if (!db) throw new Error("Database is not initialized.");
+ try {
+ const cached = preparedStatements.get(sql);
+ if (cached !== undefined) return cached;
+
+ const str = sqlite.str_new(db, sql);
+ const prepared = await sqlite.prepare_v2(db, sqlite.str_value(str));
+ if (!prepared) return;
+
+ const statement: PreparedStatement = {
+ stmt: prepared.stmt,
+ columns: sqlite.column_names(prepared.stmt)
+ };
+ preparedStatements.set(sql, statement);
+
+ sqlite.str_finish(str);
+
+ // reset retry count on success
+ retryCounter[sql] = 0;
+ return statement;
+ } catch (ex) {
+ console.error(ex);
+
+ // statement prepare process can be flaky so retry at least 5 times
+ // before giving up.
+ if (retryCounter[sql] < 5) {
+ retryCounter[sql] = (retryCounter[sql] || 0) + 1;
+ console.warn("Failed to prepare statement. Retrying:", sql);
+ return prepare(sql);
+ } else retryCounter[sql] = 0;
+
+ throw ex;
+ }
}
async function run(
@@ -122,9 +145,19 @@ async function exec<R>(
sql: string,
parameters?: SQLiteCompatibleType[]
): Promise<QueryResult<R>> {
+ if (!sql.startsWith("PRAGMA key")) {
+ await waitForDatabase();
+ }
+ if (!db) throw new Error("No database is not opened.");
+
const rows = (await run(sql, mode, parameters)) as R[];
if (mode === "query") return { rows };
+ // initialize the database after it has been successfully decrypted.
+ // all queries prior to that must wait otherwise we get the
+ // "file is not a database" error
+ if (sql.startsWith("PRAGMA key")) await initialize();
+
return {
insertId: BigInt(sqlite.last_insert_rowid(db)),
numAffectedRows: BigInt(sqlite.changes(db)),
@@ -133,32 +166,55 @@ async function exec<R>(
}
async function close() {
+ if (!db) return;
+
for (const [_, prepared] of preparedStatements) {
await sqlite.finalize(prepared.stmt);
}
preparedStatements.clear();
await sqlite.close(db);
await vfs?.close();
+
+ db = undefined;
+ initialized = false;
}
async function exportDatabase(dbName: string, async: boolean) {
- const vfs = async
- ? new IDBBatchAtomicVFS(dbName, { durability: "strict" })
- : new AccessHandlePoolVFS(dbName);
- if ("isReady" in vfs) await vfs.isReady;
-
+ const vfs = await getVFS(dbName, async);
const stream = new ReadableStream(new DatabaseSource(vfs, dbName));
return transfer(stream, [stream]);
}
-async function deleteDatabase() {
+async function deleteDatabase(dbName: string, async: boolean) {
await close();
- await vfs?.delete();
+ if (vfs) await vfs.delete();
+ else await (await getVFS(dbName, async)).delete();
+}
+
+async function getVFS(dbName: string, async: boolean) {
+ const vfs = async
+ ? await import("./IDBBatchAtomicVFS").then(
+ ({ IDBBatchAtomicVFS }) =>
+ new IDBBatchAtomicVFS(dbName, { durability: "strict" })
+ )
+ : await import("./AccessHandlePoolVFS").then(
+ ({ AccessHandlePoolVFS }) => new AccessHandlePoolVFS(dbName)
+ );
+ if ("isReady" in vfs) await vfs.isReady;
+ return vfs;
+}
+
+async function initialize() {
+ self.dispatchEvent(
+ new MessageEvent("message", { data: { type: "databaseInitialized" } })
+ );
+ console.log("Database initialized", db);
+ initialized = true;
}
const worker = {
close,
- init,
+ open,
run: exec,
export: exportDatabase,
delete: deleteDatabase
@@ -167,7 +223,22 @@ const worker = {
export type SQLiteWorker = typeof worker;
addEventListener("message", async (event) => {
- await worker.init(event.data.dbName, event.data.async, event.data.uri);
- const providerPort = createSharedServicePort(worker);
- postMessage(null, [providerPort]);
+ if (!event.data.type) {
+ await worker.open(event.data.dbName, event.data.async, event.data.uri);
+ const providerPort = createSharedServicePort(worker);
+ postMessage(null, [providerPort]);
+ }
});
+
+async function waitForDatabase() {
+ // if the database hasn't yet been initialized.
+ if (!initialized) {
+ console.log("Waiting for database to be initialized...", db);
+ return await new Promise<boolean>((resolve) =>
+ self.addEventListener("message", (ev) => {
+ if (ev.data.type === "databaseInitialized") resolve(true);
+ })
+ );
+ }
+ return true;
+}
diff --git a/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts b/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts
index a118c016e..4bfe0f4f6 100644
--- a/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts
+++ b/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts
@@ -27,63 +27,135 @@ import { Mutex } from "async-mutex";
import { SharedService } from "./shared-service";
type Config = { dbName: string; async: boolean; init?: () => Promise<void> };
-const SHARED_SERVICE_NAME = "notesnook-sqlite";
+
+const servicePool = new Map<
+ string,
+ { service: SharedService<SQLiteWorker>; activated: boolean; closed: boolean }
+>();
+
export class WaSqliteWorkerDriver implements Driver {
private connection?: DatabaseConnection;
- private connectionMutex = new ConnectionMutex();
- private worker?: SQLiteWorker;
- constructor(private readonly config: Config) {}
+ private connectionMutex = new Mutex();
+ private initializationMutex = new Mutex();
+ private readonly serviceName = `${this.config.dbName}-service`;
+
+ constructor(private readonly config: Config) {
+ console.log("new driver");
+ }
async init(): Promise<void> {
- const sharedService = new SharedService<SQLiteWorker>(SHARED_SERVICE_NAME);
- sharedService.activate(
+ const { service, activated, closed } = servicePool.get(
+ this.serviceName
+ ) || {
+ service: new SharedService<SQLiteWorker>(this.serviceName),
+ activated: false,
+ closed: true
+ };
+ if (activated) {
+ if (closed) {
+ console.log("Already activated. Reinitializing...");
+ await service.proxy.open(
+ this.config.dbName,
+ this.config.async,
+ this.config.async ? SQLiteAsyncURI : SQLiteSyncURI
+ );
+ this.needsInitialization = true;
+ servicePool.set(this.serviceName, {
+ service,
+ activated: true,
+ closed: false
+ });
+ this.connection = new WaSqliteWorkerConnection(service.proxy);
+ }
+ return;
+ }
+
+ service.activate(
() =>
new Promise<MessagePort>((resolve) => {
+ console.log("initializing worker");
this.needsInitialization = true;
- const baseWorker = new Worker();
- baseWorker.addEventListener(
+ const worker = new Worker();
+ worker.addEventListener(
"message",
(event) => resolve(event.ports[0]),
{ once: true }
);
- baseWorker.postMessage({
+ worker.postMessage({
dbName: this.config.dbName,
async: this.config.async,
uri: this.config.async ? SQLiteAsyncURI : SQLiteSyncURI
});
- })
+ }),
+ async () => {
+ console.log(
+ "new client connected.",
+ this.needsInitialization,
+ this.initializing
+ );
+ await this.#initialize();
+ }
);
console.log("waiting to initialize");
// we have to wait until a provider becomes available, otherwise
// a race condition is created where the client starts executing
// queries before it is initialized.
- await sharedService.getProviderPort();
+ await service.getProviderPort();
+
+ this.connection = new WaSqliteWorkerConnection(service.proxy);
- this.worker = sharedService.proxy;
- this.connection = new WaSqliteWorkerConnection(this.worker);
+ servicePool.set(this.serviceName, {
+ service,
+ activated: true,
+ closed: false
+ });
}
private needsInitialization = false;
+ private initializing = false;
async #initialize() {
- if (this.needsInitialization) {
- this.needsInitialization = false;
+ if (this.needsInitialization && !this.initializing) {
try {
+ console.log("Starting initialization...");
+ this.initializing = true;
await this.config.init?.();
+ this.needsInitialization = false;
+ console.log("Initialization done...");
} catch (e) {
+ console.error(e);
this.needsInitialization = true;
throw e;
+ } finally {
+ this.initializing = false;
+ // there can be multiple locks on this mutex
+ while (this.initializationMutex.isLocked())
+ this.initializationMutex.release();
}
}
}
async acquireConnection(): Promise<DatabaseConnection> {
+ if (!this.connection) throw new Error("Driver not initialized.");
await this.#initialize();
+
+ // We don't want to create deadlock in cases where database
+ // hasn't yet been initialized but another query has already taken
+ // the connection.
+ // Secondly, we don't want to give any other connection until
+ // database has finished initializing.
+ await this.initializationMutex.waitForUnlock();
+ if (this.initializing) {
+ await this.initializationMutex.acquire();
+ return this.connection;
+ }
+
// SQLite only has one single connection. We use a mutex here to wait
// until the single connection has been released.
- await this.connectionMutex.lock();
- return this.connection!;
+ await this.connectionMutex.waitForUnlock();
+ await this.connectionMutex.acquire();
+ return this.connection;
}
async beginTransaction(connection: DatabaseConnection): Promise<void> {
@@ -99,48 +171,29 @@ export class WaSqliteWorkerDriver implements Driver {
}
async releaseConnection(): Promise<void> {
- this.connectionMutex.unlock();
+ this.connectionMutex.release();
}
async destroy(): Promise<void> {
- return await this.worker?.close();
+ const service = servicePool.get(this.serviceName);
+ if (!service) return;
+ await service.service?.proxy.close();
+ service.closed = true;
}
async delete() {
- return this.worker?.delete();
+ const { service } = servicePool.get(this.serviceName) || {};
+ await service?.proxy?.delete(this.config.dbName, this.config.async);
}
async export() {
- return this.worker?.export(this.config.dbName, this.config.async);
- }
-}
-
-class ConnectionMutex {
- private promise?: Promise<void>;
- private resolve?: () => void;
-
- async lock(): Promise<void> {
- while (this.promise) {
- await this.promise;
- }
-
- this.promise = new Promise((resolve) => {
- this.resolve = resolve;
- });
- }
-
- unlock(): void {
- const resolve = this.resolve;
-
- this.promise = undefined;
- this.resolve = undefined;
-
- resolve?.();
+ return servicePool
+ .get(this.serviceName)
+ ?.service?.proxy?.export(this.config.dbName, this.config.async);
}
}
class WaSqliteWorkerConnection implements DatabaseConnection {
- private readonly queryMutex = new Mutex();
constructor(private readonly worker: SQLiteWorker) {}
streamQuery<R>(): AsyncIterableIterator<QueryResult<R>> {
@@ -157,8 +210,6 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
: query.kind === "RawNode"
? "raw"
: "exec";
- return this.queryMutex.runExclusive(() =>
- this.worker.run(mode, sql, parameters as any)
- );
+ return this.worker.run(mode, sql, parameters as any);
}
}