A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
web: make sqlite slightly faster
Abdullah Atta779f000f3e9b
diff --git a/apps/web/src/common/db.ts b/apps/web/src/common/db.tsindex e84968f36..e00bd05ed 100644--- a/apps/web/src/common/db.ts+++ b/apps/web/src/common/db.ts@@ -49,8 +49,8 @@ async function initializeDatabase(persistence: DatabasePersistence) {database.setup({sqliteOptions: {dialect: createDialect,- ...(isFeatureSupported("opfs")- ? { journalMode: "WAL" }+ ...(IS_DESKTOP_APP || isFeatureSupported("opfs")+ ? { journalMode: "WAL", lockingMode: "exclusive" }: {journalMode: "MEMORY",lockingMode: "exclusive"@@ -58,7 +58,7 @@ async function initializeDatabase(persistence: DatabasePersistence) {tempStore: "memory",synchronous: "normal",pageSize: 8192,- cacheSize: -16000,+ cacheSize: -32000,password: databaseKey},storage: storage,diff --git a/apps/web/src/common/sqlite/index.ts b/apps/web/src/common/sqlite/index.tsindex ac9a0a485..2e9da1b57 100644--- a/apps/web/src/common/sqlite/index.ts+++ b/apps/web/src/common/sqlite/index.ts@@ -36,7 +36,7 @@ export const createDialect = (name: string): Dialect => {return {createDriver: () =>new WaSqliteWorkerDriver({- async: isFeatureSupported("opfs") ? false : true,+ async: !isFeatureSupported("opfs"),dbName: name}),createAdapter: () => new SqliteAdapter(),diff --git a/apps/web/src/common/sqlite/sqlite-api.js b/apps/web/src/common/sqlite/sqlite-api.jsindex 7f2b5fa6d..14fae03f3 100644--- a/apps/web/src/common/sqlite/sqlite-api.js+++ b/apps/web/src/common/sqlite/sqlite-api.js@@ -121,7 +121,7 @@ export function Factory(Module) {};sqlite3.bind = function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);switch (typeof value) {case "number":if (value === (value | 0)) {@@ -152,14 +152,14 @@ export function Factory(Module) {const fname = "sqlite3_bind_blob";const f = Module.cwrap(fname, ...decl("nnnnn:n"));return function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);// @ts-ignoreconst byteLength = value.byteLength ?? value.length;const ptr = Module._sqlite3_malloc(byteLength);Module.HEAPU8.subarray(ptr).set(value);const result = f(stmt, i, ptr, byteLength, sqliteFreeAddress);// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -167,7 +167,7 @@ export function Factory(Module) {const fname = "sqlite3_bind_parameter_count";const f = Module.cwrap(fname, ...decl("n:n"));return function (stmt) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = f(stmt);// trace(fname, result);return result;@@ -178,7 +178,7 @@ export function Factory(Module) {const fname = "sqlite3_clear_bindings";const f = Module.cwrap(fname, ...decl("n:n"));return function (stmt) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = f(stmt);// trace(fname, result);return result;@@ -189,10 +189,10 @@ export function Factory(Module) {const fname = "sqlite3_bind_double";const f = Module.cwrap(fname, ...decl("nnn:n"));return function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = f(stmt, i, value);// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -200,12 +200,12 @@ export function Factory(Module) {const fname = "sqlite3_bind_int";const f = Module.cwrap(fname, ...decl("nnn:n"));return function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);if (value > 0x7fffffff || value < -0x80000000) return SQLite.SQLITE_RANGE;const result = f(stmt, i, value);// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -213,14 +213,14 @@ export function Factory(Module) {const fname = "sqlite3_bind_int64";const f = Module.cwrap(fname, ...decl("nnnn:n"));return function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);if (value > MAX_INT64 || value < MIN_INT64) return SQLite.SQLITE_RANGE;const lo32 = value & 0xffffffffn;const hi32 = value >> 32n;const result = f(stmt, i, Number(lo32), Number(hi32));// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -228,10 +228,10 @@ export function Factory(Module) {const fname = "sqlite3_bind_null";const f = Module.cwrap(fname, ...decl("nn:n"));return function (stmt, i) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = f(stmt, i);// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -239,7 +239,7 @@ export function Factory(Module) {const fname = "sqlite3_bind_parameter_name";const f = Module.cwrap(fname, ...decl("n:s"));return function (stmt, i) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = f(stmt, i);// trace(fname, result);return result;@@ -250,11 +250,11 @@ export function Factory(Module) {const fname = "sqlite3_bind_text";const f = Module.cwrap(fname, ...decl("nnnnn:n"));return function (stmt, i, value) {- verifyStatement(stmt);+ // verifyStatement(stmt);const ptr = createUTF8(value);const result = f(stmt, i, ptr, -1, sqliteFreeAddress);// trace(fname, result);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -504,7 +504,7 @@ export function Factory(Module) {}const result = await f(stmt);- const db = mapStmtToDB.get(stmt);+ // const db = mapStmtToDB.get(stmt);mapStmtToDB.delete(stmt);// Don't throw on error here. Typically the error has already been@@ -594,7 +594,7 @@ export function Factory(Module) {return async function (stmt) {verifyStatement(stmt);const result = await f(stmt);- return check(fname, result, mapStmtToDB.get(stmt));+ return check(fname, result, null, stmt);};})();@@ -737,15 +737,25 @@ export function Factory(Module) {const fname = "sqlite3_step";const f = Module.cwrap(fname, ...decl("n:n"), { async });return async function (stmt) {- verifyStatement(stmt);+ // verifyStatement(stmt);const result = await f(stmt);- return check(fname, result, mapStmtToDB.get(stmt), [+ return check(fname, result, null, stmt, [SQLite.SQLITE_ROW,SQLite.SQLITE_DONE]);};})();+ sqlite3.last_insert_rowid = (function () {+ const fname = "sqlite3_last_insert_rowid";+ const f = Module.cwrap(fname, ...decl("n:n"));+ return function (db) {+ verifyDatabase(db);+ const result = f(db);+ return result;+ };+ })();+// Duplicate some of the SQLite dynamic string API but without// calling SQLite (except for memory allocation). We need some way// to transfer Javascript strings and might as well use an API@@ -907,9 +917,16 @@ export function Factory(Module) {return check("sqlite3_vfs_register", result);};- function check(fname, result, db = null, allowed = [SQLite.SQLITE_OK]) {+ function check(+ fname,+ result,+ db = null,+ stmt = null,+ allowed = [SQLite.SQLITE_OK]+ ) {// trace(fname, result);if (allowed.includes(result)) return result;+ db = db || (stmt !== null ? mapStmtToDB.get(stmt) : null);const message = db? Module.ccall("sqlite3_errmsg", "string", ["number"], [db]): fname;diff --git a/apps/web/src/common/sqlite/sqlite-types.ts b/apps/web/src/common/sqlite/sqlite-types.tsindex d5884ca73..d4ecd1e61 100644--- a/apps/web/src/common/sqlite/sqlite-types.ts+++ b/apps/web/src/common/sqlite/sqlite-types.ts@@ -938,6 +938,13 @@ export interface SQLiteAPI {*/step(stmt: number): Promise<number>;+ /**+ * @see https://www.sqlite.org/c3ref/last_insert_rowid.html+ * @param db database pointer+ * @returns last rowid+ */+ last_insert_rowid(db: number): number;+/*** Create a new `sqlite3_str` dynamic string instance*diff --git a/apps/web/src/common/sqlite/sqlite.worker.ts b/apps/web/src/common/sqlite/sqlite.worker.tsindex efb9fec9e..8c7fe1586 100644--- a/apps/web/src/common/sqlite/sqlite.worker.ts+++ b/apps/web/src/common/sqlite/sqlite.worker.ts@@ -48,7 +48,7 @@ async function init(dbName: string, async: boolean, url?: string) {? new IDBBatchAtomicVFS(dbName, { durability: "strict" }): new AccessHandlePoolVFS(dbName);if ("isReady" in vfs) await vfs.isReady;- console.log(vfs, SQLiteAsyncModule);+sqlite.vfs_register(vfs, false);db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);}@@ -70,15 +70,27 @@ async function prepare(sql: string) {columns: sqlite.column_names(prepared.stmt)};preparedStatements.set(sql, statement);++ sqlite.str_finish(str);return statement;}-async function run(sql: string, parameters?: SQLiteCompatibleType[]) {+async function run(+ sql: string,+ mode: RunMode,+ parameters?: SQLiteCompatibleType[]+) {const prepared = await prepare(sql);if (!prepared) return [];try {if (parameters) sqlite.bind_collection(prepared.stmt, parameters);+ // fast path for exec statements+ if (mode === "exec") {+ while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW);+ return [];+ }+const rows: Record<string, SQLiteCompatibleType>[] = [];while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW) {const row = sqlite.row(prepared.stmt);@@ -96,7 +108,7 @@ async function run(sql: string, parameters?: SQLiteCompatibleType[]) {sqlite.finalize(prepared.stmt)// ignore error (we will just prepare a new statement)- .catch(() => false)+ .catch(console.error).finally(() => preparedStatements.delete(sql)));}@@ -107,14 +119,11 @@ async function exec<R>(sql: string,parameters?: SQLiteCompatibleType[]): Promise<QueryResult<R>> {- console.time(sql);- const rows = (await run(sql, parameters)) as R[];- console.timeEnd(sql);+ const rows = (await run(sql, mode, parameters)) as R[];if (mode === "query") return { rows };- const v = await run("SELECT last_insert_rowid() as id");return {- insertId: BigInt(v[0].id as number),+ insertId: BigInt(sqlite.last_insert_rowid(db)),numAffectedRows: BigInt(sqlite.changes(db)),rows: mode === "raw" ? rows : []};