A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
web: make shared sqlite more robust
Abdullah Attaebe3f2b771b0
diff --git a/apps/web/src/common/sqlite/shared-service.ts b/apps/web/src/common/sqlite/shared-service.tsindex 400398876..0fb5f8518 100644--- a/apps/web/src/common/sqlite/shared-service.ts+++ b/apps/web/src/common/sqlite/shared-service.ts@@ -38,7 +38,13 @@ export class SharedService<T extends object> extends EventTarget {#providerPort: Promise<MessagePort | null>;providerCallbacks: Map<string,- { resolve: (result: unknown) => void; reject: (reason: unknown) => void }+ {+ resolve: (result: unknown) => void;+ reject: (reason: unknown) => void;+ nonce: string;+ method: string | symbol;+ args: any[];+ }> = new Map();#providerCounter = 0;#providerChangeCleanup: (() => void)[] = [];@@ -61,8 +67,9 @@ export class SharedService<T extends object> extends EventTarget {) {// A context (possibly this one) announced itself as the new provider.// Discard any old provider and connect to the new one.- this.#closeProviderPort(this.#providerPort);+ this.#providerPort.then((port) => port?.close());this.#providerPort = this.#providerChange();+ await this.#resendPendingCallbacks();}},{ signal: this.#onClose.signal }@@ -71,7 +78,10 @@ export class SharedService<T extends object> extends EventTarget {this.proxy = this.#createProxy();}- activate(portProviderFunc: () => MessagePort | Promise<MessagePort>) {+ activate(+ portProviderFunc: () => MessagePort | Promise<MessagePort>,+ onClientConnected: () => Promise<void>+ ) {if (this.#onDeactivate) return;// When acquire a lock on the service name then we become the service@@ -80,10 +90,8 @@ export class SharedService<T extends object> extends EventTarget {this.#onDeactivate = new AbortController();const LOCK_NAME = `SharedService-${this.serviceName}`;- navigator.locks.request(- LOCK_NAME,- { signal: this.#onDeactivate.signal },- async () => {+ navigator.locks+ .request(LOCK_NAME, { signal: this.#onDeactivate.signal }, async () => {// Get the port to request client ports.const port = await portProviderFunc();port.start();@@ -93,12 +101,16 @@ export class SharedService<T extends object> extends EventTarget {// request.const providerId = await this.#clientId;const broadcastChannel = new BroadcastChannel("SharedService");++ // eslint-disable-next-line @typescript-eslint/no-this-alias+ const thisArg = this;broadcastChannel.addEventListener("message",- async ({ data }) => {+ async function onMessage(event) {+ const { data } = event;if (data?.type === "request" &&- data?.sharedService === this.serviceName+ data?.sharedService === thisArg.serviceName) {// Get a port to send to the client.const requestedPort = await new Promise<MessagePort>(@@ -112,7 +124,22 @@ export class SharedService<T extends object> extends EventTarget {});- this.#sendPortToClient(data, requestedPort);+ if (data.clientId !== providerId) {+ await onClientConnected();+ }++ try {+ thisArg.#sendPortToClient(data, requestedPort);+ } catch (e) {+ console.error(e, providerId, data);+ // retry if port has been neutered, this can happen when+ // closing the provider tab/window+ if (e instanceof Error && e.message.includes("neutered")) {+ console.warn("Retrying in 100ms...");+ await new Promise((resolve) => setTimeout(resolve, 100));+ onMessage.bind(broadcastChannel, event)();+ } else throw e;+ }}},{ signal: this.#onDeactivate?.signal }@@ -132,8 +159,8 @@ export class SharedService<T extends object> extends EventTarget {reject(this.#onDeactivate?.signal.reason);});});- }- );+ })+ .catch(console.error);}deactivate() {@@ -142,14 +169,14 @@ export class SharedService<T extends object> extends EventTarget {}close() {- this.deactivate();- this.#onClose.abort();for (const { reject } of this.providerCallbacks.values()) {- reject(new Error("SharedService closed"));+ reject(new Error("SharedService closed."));}+ this.deactivate();+ this.#onClose.abort();}- async #sendPortToClient(message: any, port: MessagePort) {+ #sendPortToClient(message: any, port: MessagePort) {sharedWorker?.port.postMessage(message, [port]);}@@ -271,10 +298,10 @@ export class SharedService<T extends object> extends EventTarget {}}- #closeProviderPort(providerPort: Promise<MessagePort | null>) {- providerPort.then((port) => port?.close());- for (const { reject } of this.providerCallbacks.values()) {- reject(new Error("SharedService provider change"));+ async #resendPendingCallbacks() {+ const port = await this.getProviderPort();+ for (const { method, args, nonce } of this.providerCallbacks.values()) {+ port.postMessage({ nonce, method, args });}}@@ -288,7 +315,13 @@ export class SharedService<T extends object> extends EventTarget {const providerPort = await this.getProviderPort();return new Promise((resolve, reject) => {- this.providerCallbacks.set(nonce, { resolve, reject });+ this.providerCallbacks.set(nonce, {+ resolve,+ reject,+ nonce,+ method,+ args+ });providerPort.postMessage({ nonce, method, args });}).finally(() => {this.providerCallbacks.delete(nonce);diff --git a/apps/web/src/common/sqlite/sqlite.worker.ts b/apps/web/src/common/sqlite/sqlite.worker.tsindex 942d0f462..243eb51b8 100644--- a/apps/web/src/common/sqlite/sqlite.worker.ts+++ b/apps/web/src/common/sqlite/sqlite.worker.ts@@ -19,15 +19,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import type { SQLiteAPI, SQLiteCompatibleType } from "./sqlite-types";import { Factory, SQLITE_ROW } from "./sqlite-api";-import SQLiteAsyncESMFactory from "./wa-sqlite-async";-import SQLiteSyncESMFactory from "./wa-sqlite";-import { IDBBatchAtomicVFS } from "./IDBBatchAtomicVFS";-import { AccessHandlePoolVFS } from "./AccessHandlePoolVFS";import { transfer } from "comlink";import type { RunMode } from "./type";import { QueryResult } from "kysely";import { DatabaseSource } from "./sqlite-export";import { createSharedServicePort } from "./shared-service";+import type { IDBBatchAtomicVFS } from "./IDBBatchAtomicVFS";+import type { AccessHandlePoolVFS } from "./AccessHandlePoolVFS";type PreparedStatement = {stmt: number;@@ -35,22 +33,29 @@ type PreparedStatement = {};let sqlite: SQLiteAPI;-let db: number;+let db: number | undefined = undefined;let vfs: IDBBatchAtomicVFS | AccessHandlePoolVFS | null = null;+let initialized = false;const preparedStatements: Map<string, PreparedStatement> = new Map();+const retryCounter: Record<string, number> = {};+console.log("new sqlite worker");-async function init(dbName: string, async: boolean, url?: string) {- if (db) return;+async function open(dbName: string, async: boolean, url?: string) {+ if (db) {+ console.error("Database is already initialized", db);+ return;+ }const option = url ? { locateFile: () => url } : {};- const SQLiteAsyncModule = async- ? await SQLiteAsyncESMFactory(option)- : await SQLiteSyncESMFactory(option);- sqlite = Factory(SQLiteAsyncModule);- vfs = async- ? new IDBBatchAtomicVFS(dbName, { durability: "strict" })- : new AccessHandlePoolVFS(dbName);- if ("isReady" in vfs) await vfs.isReady;+ const sqliteModule = async+ ? await import("./wa-sqlite-async").then(+ ({ default: SQLiteAsyncESMFactory }) => SQLiteAsyncESMFactory(option)+ )+ : await import("./wa-sqlite").then(({ default: SQLiteSyncESMFactory }) =>+ SQLiteSyncESMFactory(option)+ );+ sqlite = Factory(sqliteModule);+ vfs = await getVFS(dbName, async);sqlite.vfs_register(vfs, false);db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);@@ -61,21 +66,39 @@ async function init(dbName: string, async: boolean, url?: string) {* to avoid unnecessary computations.*/async function prepare(sql: string) {- const cached = preparedStatements.get(sql);- if (cached !== undefined) return cached;-- const str = sqlite.str_new(db, sql);- const prepared = await sqlite.prepare_v2(db, sqlite.str_value(str));- if (!prepared) return;-- const statement: PreparedStatement = {- stmt: prepared.stmt,- columns: sqlite.column_names(prepared.stmt)- };- preparedStatements.set(sql, statement);-- sqlite.str_finish(str);- return statement;+ if (!db) throw new Error("Database is not initialized.");+ try {+ const cached = preparedStatements.get(sql);+ if (cached !== undefined) return cached;++ const str = sqlite.str_new(db, sql);+ const prepared = await sqlite.prepare_v2(db, sqlite.str_value(str));+ if (!prepared) return;++ const statement: PreparedStatement = {+ stmt: prepared.stmt,+ columns: sqlite.column_names(prepared.stmt)+ };+ preparedStatements.set(sql, statement);++ sqlite.str_finish(str);++ // reset retry count on success+ retryCounter[sql] = 0;+ return statement;+ } catch (ex) {+ console.error(ex);++ // statement prepare process can be flaky so retry at least 5 times+ // before giving up.+ if (retryCounter[sql] < 5) {+ retryCounter[sql] = (retryCounter[sql] || 0) + 1;+ console.warn("Failed to prepare statement. Retrying:", sql);+ return prepare(sql);+ } else retryCounter[sql] = 0;++ throw ex;+ }}async function run(@@ -122,9 +145,19 @@ async function exec<R>(sql: string,parameters?: SQLiteCompatibleType[]): Promise<QueryResult<R>> {+ if (!sql.startsWith("PRAGMA key")) {+ await waitForDatabase();+ }+ if (!db) throw new Error("No database is not opened.");+const rows = (await run(sql, mode, parameters)) as R[];if (mode === "query") return { rows };+ // initialize the database after it has been successfully decrypted.+ // all queries prior to that must wait otherwise we get the+ // "file is not a database" error+ if (sql.startsWith("PRAGMA key")) await initialize();+return {insertId: BigInt(sqlite.last_insert_rowid(db)),numAffectedRows: BigInt(sqlite.changes(db)),@@ -133,32 +166,55 @@ async function exec<R>(}async function close() {+ if (!db) return;+for (const [_, prepared] of preparedStatements) {await sqlite.finalize(prepared.stmt);}preparedStatements.clear();await sqlite.close(db);await vfs?.close();++ db = undefined;+ initialized = false;}async function exportDatabase(dbName: string, async: boolean) {- const vfs = async- ? new IDBBatchAtomicVFS(dbName, { durability: "strict" })- : new AccessHandlePoolVFS(dbName);- if ("isReady" in vfs) await vfs.isReady;-+ const vfs = await getVFS(dbName, async);const stream = new ReadableStream(new DatabaseSource(vfs, dbName));return transfer(stream, [stream]);}-async function deleteDatabase() {+async function deleteDatabase(dbName: string, async: boolean) {await close();- await vfs?.delete();+ if (vfs) await vfs.delete();+ else await (await getVFS(dbName, async)).delete();+}++async function getVFS(dbName: string, async: boolean) {+ const vfs = async+ ? await import("./IDBBatchAtomicVFS").then(+ ({ IDBBatchAtomicVFS }) =>+ new IDBBatchAtomicVFS(dbName, { durability: "strict" })+ )+ : await import("./AccessHandlePoolVFS").then(+ ({ AccessHandlePoolVFS }) => new AccessHandlePoolVFS(dbName)+ );+ if ("isReady" in vfs) await vfs.isReady;+ return vfs;+}++async function initialize() {+ self.dispatchEvent(+ new MessageEvent("message", { data: { type: "databaseInitialized" } })+ );+ console.log("Database initialized", db);+ initialized = true;}const worker = {close,- init,+ open,run: exec,export: exportDatabase,delete: deleteDatabase@@ -167,7 +223,22 @@ const worker = {export type SQLiteWorker = typeof worker;addEventListener("message", async (event) => {- await worker.init(event.data.dbName, event.data.async, event.data.uri);- const providerPort = createSharedServicePort(worker);- postMessage(null, [providerPort]);+ if (!event.data.type) {+ await worker.open(event.data.dbName, event.data.async, event.data.uri);+ const providerPort = createSharedServicePort(worker);+ postMessage(null, [providerPort]);+ }});++async function waitForDatabase() {+ // if the database hasn't yet been initialized.+ if (!initialized) {+ console.log("Waiting for database to be initialized...", db);+ return await new Promise<boolean>((resolve) =>+ self.addEventListener("message", (ev) => {+ if (ev.data.type === "databaseInitialized") resolve(true);+ })+ );+ }+ return true;+}diff --git a/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts b/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.tsindex a118c016e..4bfe0f4f6 100644--- a/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts+++ b/apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts@@ -27,63 +27,135 @@ import { Mutex } from "async-mutex";import { SharedService } from "./shared-service";type Config = { dbName: string; async: boolean; init?: () => Promise<void> };-const SHARED_SERVICE_NAME = "notesnook-sqlite";++const servicePool = new Map<+ string,+ { service: SharedService<SQLiteWorker>; activated: boolean; closed: boolean }+>();+export class WaSqliteWorkerDriver implements Driver {private connection?: DatabaseConnection;- private connectionMutex = new ConnectionMutex();- private worker?: SQLiteWorker;- constructor(private readonly config: Config) {}+ private connectionMutex = new Mutex();+ private initializationMutex = new Mutex();+ private readonly serviceName = `${this.config.dbName}-service`;++ constructor(private readonly config: Config) {+ console.log("new driver");+ }async init(): Promise<void> {- const sharedService = new SharedService<SQLiteWorker>(SHARED_SERVICE_NAME);- sharedService.activate(+ const { service, activated, closed } = servicePool.get(+ this.serviceName+ ) || {+ service: new SharedService<SQLiteWorker>(this.serviceName),+ activated: false,+ closed: true+ };+ if (activated) {+ if (closed) {+ console.log("Already activated. Reinitializing...");+ await service.proxy.open(+ this.config.dbName,+ this.config.async,+ this.config.async ? SQLiteAsyncURI : SQLiteSyncURI+ );+ this.needsInitialization = true;+ servicePool.set(this.serviceName, {+ service,+ activated: true,+ closed: false+ });+ this.connection = new WaSqliteWorkerConnection(service.proxy);+ }+ return;+ }++ service.activate(() =>new Promise<MessagePort>((resolve) => {+ console.log("initializing worker");this.needsInitialization = true;- const baseWorker = new Worker();- baseWorker.addEventListener(+ const worker = new Worker();+ worker.addEventListener("message",(event) => resolve(event.ports[0]),{ once: true });- baseWorker.postMessage({+ worker.postMessage({dbName: this.config.dbName,async: this.config.async,uri: this.config.async ? SQLiteAsyncURI : SQLiteSyncURI});- })+ }),+ async () => {+ console.log(+ "new client connected.",+ this.needsInitialization,+ this.initializing+ );+ await this.#initialize();+ });console.log("waiting to initialize");// we have to wait until a provider becomes available, otherwise// a race condition is created where the client starts executing// queries before it is initialized.- await sharedService.getProviderPort();+ await service.getProviderPort();++ this.connection = new WaSqliteWorkerConnection(service.proxy);- this.worker = sharedService.proxy;- this.connection = new WaSqliteWorkerConnection(this.worker);+ servicePool.set(this.serviceName, {+ service,+ activated: true,+ closed: false+ });}private needsInitialization = false;+ private initializing = false;async #initialize() {- if (this.needsInitialization) {- this.needsInitialization = false;+ if (this.needsInitialization && !this.initializing) {try {+ console.log("Starting initialization...");+ this.initializing = true;await this.config.init?.();+ this.needsInitialization = false;+ console.log("Initialization done...");} catch (e) {+ console.error(e);this.needsInitialization = true;throw e;+ } finally {+ this.initializing = false;+ // there can be multiple locks on this mutex+ while (this.initializationMutex.isLocked())+ this.initializationMutex.release();}}}async acquireConnection(): Promise<DatabaseConnection> {+ if (!this.connection) throw new Error("Driver not initialized.");await this.#initialize();++ // We don't want to create deadlock in cases where database+ // hasn't yet been initialized but another query has already taken+ // the connection.+ // Secondly, we don't want to give any other connection until+ // database has finished initializing.+ await this.initializationMutex.waitForUnlock();+ if (this.initializing) {+ await this.initializationMutex.acquire();+ return this.connection;+ }+// SQLite only has one single connection. We use a mutex here to wait// until the single connection has been released.- await this.connectionMutex.lock();- return this.connection!;+ await this.connectionMutex.waitForUnlock();+ await this.connectionMutex.acquire();+ return this.connection;}async beginTransaction(connection: DatabaseConnection): Promise<void> {@@ -99,48 +171,29 @@ export class WaSqliteWorkerDriver implements Driver {}async releaseConnection(): Promise<void> {- this.connectionMutex.unlock();+ this.connectionMutex.release();}async destroy(): Promise<void> {- return await this.worker?.close();+ const service = servicePool.get(this.serviceName);+ if (!service) return;+ await service.service?.proxy.close();+ service.closed = true;}async delete() {- return this.worker?.delete();+ const { service } = servicePool.get(this.serviceName) || {};+ await service?.proxy?.delete(this.config.dbName, this.config.async);}async export() {- return this.worker?.export(this.config.dbName, this.config.async);- }-}--class ConnectionMutex {- private promise?: Promise<void>;- private resolve?: () => void;-- async lock(): Promise<void> {- while (this.promise) {- await this.promise;- }-- this.promise = new Promise((resolve) => {- this.resolve = resolve;- });- }-- unlock(): void {- const resolve = this.resolve;-- this.promise = undefined;- this.resolve = undefined;-- resolve?.();+ return servicePool+ .get(this.serviceName)+ ?.service?.proxy?.export(this.config.dbName, this.config.async);}}class WaSqliteWorkerConnection implements DatabaseConnection {- private readonly queryMutex = new Mutex();constructor(private readonly worker: SQLiteWorker) {}streamQuery<R>(): AsyncIterableIterator<QueryResult<R>> {@@ -157,8 +210,6 @@ class WaSqliteWorkerConnection implements DatabaseConnection {: query.kind === "RawNode"? "raw": "exec";- return this.queryMutex.runExclusive(() =>- this.worker.run(mode, sql, parameters as any)- );+ return this.worker.run(mode, sql, parameters as any);}}