Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

web: make sqlite slightly faster

Abdullah Atta779f000f3e9b
diff --git a/apps/web/src/common/db.ts b/apps/web/src/common/db.ts
index e84968f36..e00bd05ed 100644
--- a/apps/web/src/common/db.ts
+++ b/apps/web/src/common/db.ts
@@ -49,8 +49,8 @@ async function initializeDatabase(persistence: DatabasePersistence) {
database.setup({
sqliteOptions: {
dialect: createDialect,
- ...(isFeatureSupported("opfs")
- ? { journalMode: "WAL" }
+ ...(IS_DESKTOP_APP || isFeatureSupported("opfs")
+ ? { journalMode: "WAL", lockingMode: "exclusive" }
: {
journalMode: "MEMORY",
lockingMode: "exclusive"
@@ -58,7 +58,7 @@ async function initializeDatabase(persistence: DatabasePersistence) {
tempStore: "memory",
synchronous: "normal",
pageSize: 8192,
- cacheSize: -16000,
+ cacheSize: -32000,
password: databaseKey
},
storage: storage,
diff --git a/apps/web/src/common/sqlite/index.ts b/apps/web/src/common/sqlite/index.ts
index ac9a0a485..2e9da1b57 100644
--- a/apps/web/src/common/sqlite/index.ts
+++ b/apps/web/src/common/sqlite/index.ts
@@ -36,7 +36,7 @@ export const createDialect = (name: string): Dialect => {
return {
createDriver: () =>
new WaSqliteWorkerDriver({
- async: isFeatureSupported("opfs") ? false : true,
+ async: !isFeatureSupported("opfs"),
dbName: name
}),
createAdapter: () => new SqliteAdapter(),
diff --git a/apps/web/src/common/sqlite/sqlite-api.js b/apps/web/src/common/sqlite/sqlite-api.js
index 7f2b5fa6d..14fae03f3 100644
--- a/apps/web/src/common/sqlite/sqlite-api.js
+++ b/apps/web/src/common/sqlite/sqlite-api.js
@@ -121,7 +121,7 @@ export function Factory(Module) {
};
sqlite3.bind = function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
switch (typeof value) {
case "number":
if (value === (value | 0)) {
@@ -152,14 +152,14 @@ export function Factory(Module) {
const fname = "sqlite3_bind_blob";
const f = Module.cwrap(fname, ...decl("nnnnn:n"));
return function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
// @ts-ignore
const byteLength = value.byteLength ?? value.length;
const ptr = Module._sqlite3_malloc(byteLength);
Module.HEAPU8.subarray(ptr).set(value);
const result = f(stmt, i, ptr, byteLength, sqliteFreeAddress);
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -167,7 +167,7 @@ export function Factory(Module) {
const fname = "sqlite3_bind_parameter_count";
const f = Module.cwrap(fname, ...decl("n:n"));
return function (stmt) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = f(stmt);
// trace(fname, result);
return result;
@@ -178,7 +178,7 @@ export function Factory(Module) {
const fname = "sqlite3_clear_bindings";
const f = Module.cwrap(fname, ...decl("n:n"));
return function (stmt) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = f(stmt);
// trace(fname, result);
return result;
@@ -189,10 +189,10 @@ export function Factory(Module) {
const fname = "sqlite3_bind_double";
const f = Module.cwrap(fname, ...decl("nnn:n"));
return function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = f(stmt, i, value);
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -200,12 +200,12 @@ export function Factory(Module) {
const fname = "sqlite3_bind_int";
const f = Module.cwrap(fname, ...decl("nnn:n"));
return function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
if (value > 0x7fffffff || value < -0x80000000) return SQLite.SQLITE_RANGE;
const result = f(stmt, i, value);
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -213,14 +213,14 @@ export function Factory(Module) {
const fname = "sqlite3_bind_int64";
const f = Module.cwrap(fname, ...decl("nnnn:n"));
return function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
if (value > MAX_INT64 || value < MIN_INT64) return SQLite.SQLITE_RANGE;
const lo32 = value & 0xffffffffn;
const hi32 = value >> 32n;
const result = f(stmt, i, Number(lo32), Number(hi32));
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -228,10 +228,10 @@ export function Factory(Module) {
const fname = "sqlite3_bind_null";
const f = Module.cwrap(fname, ...decl("nn:n"));
return function (stmt, i) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = f(stmt, i);
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -239,7 +239,7 @@ export function Factory(Module) {
const fname = "sqlite3_bind_parameter_name";
const f = Module.cwrap(fname, ...decl("n:s"));
return function (stmt, i) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = f(stmt, i);
// trace(fname, result);
return result;
@@ -250,11 +250,11 @@ export function Factory(Module) {
const fname = "sqlite3_bind_text";
const f = Module.cwrap(fname, ...decl("nnnnn:n"));
return function (stmt, i, value) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const ptr = createUTF8(value);
const result = f(stmt, i, ptr, -1, sqliteFreeAddress);
// trace(fname, result);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -504,7 +504,7 @@ export function Factory(Module) {
}
const result = await f(stmt);
- const db = mapStmtToDB.get(stmt);
+ // const db = mapStmtToDB.get(stmt);
mapStmtToDB.delete(stmt);
// Don't throw on error here. Typically the error has already been
@@ -594,7 +594,7 @@ export function Factory(Module) {
return async function (stmt) {
verifyStatement(stmt);
const result = await f(stmt);
- return check(fname, result, mapStmtToDB.get(stmt));
+ return check(fname, result, null, stmt);
};
})();
@@ -737,15 +737,25 @@ export function Factory(Module) {
const fname = "sqlite3_step";
const f = Module.cwrap(fname, ...decl("n:n"), { async });
return async function (stmt) {
- verifyStatement(stmt);
+ // verifyStatement(stmt);
const result = await f(stmt);
- return check(fname, result, mapStmtToDB.get(stmt), [
+ return check(fname, result, null, stmt, [
SQLite.SQLITE_ROW,
SQLite.SQLITE_DONE
]);
};
})();
+ sqlite3.last_insert_rowid = (function () {
+ const fname = "sqlite3_last_insert_rowid";
+ const f = Module.cwrap(fname, ...decl("n:n"));
+ return function (db) {
+ verifyDatabase(db);
+ const result = f(db);
+ return result;
+ };
+ })();
+
// Duplicate some of the SQLite dynamic string API but without
// calling SQLite (except for memory allocation). We need some way
// to transfer Javascript strings and might as well use an API
@@ -907,9 +917,16 @@ export function Factory(Module) {
return check("sqlite3_vfs_register", result);
};
- function check(fname, result, db = null, allowed = [SQLite.SQLITE_OK]) {
+ function check(
+ fname,
+ result,
+ db = null,
+ stmt = null,
+ allowed = [SQLite.SQLITE_OK]
+ ) {
// trace(fname, result);
if (allowed.includes(result)) return result;
+ db = db || (stmt !== null ? mapStmtToDB.get(stmt) : null);
const message = db
? Module.ccall("sqlite3_errmsg", "string", ["number"], [db])
: fname;
diff --git a/apps/web/src/common/sqlite/sqlite-types.ts b/apps/web/src/common/sqlite/sqlite-types.ts
index d5884ca73..d4ecd1e61 100644
--- a/apps/web/src/common/sqlite/sqlite-types.ts
+++ b/apps/web/src/common/sqlite/sqlite-types.ts
@@ -938,6 +938,13 @@ export interface SQLiteAPI {
*/
step(stmt: number): Promise<number>;
+ /**
+ * @see https://www.sqlite.org/c3ref/last_insert_rowid.html
+ * @param db database pointer
+ * @returns last rowid
+ */
+ last_insert_rowid(db: number): number;
+
/**
* Create a new `sqlite3_str` dynamic string instance
*
diff --git a/apps/web/src/common/sqlite/sqlite.worker.ts b/apps/web/src/common/sqlite/sqlite.worker.ts
index efb9fec9e..8c7fe1586 100644
--- a/apps/web/src/common/sqlite/sqlite.worker.ts
+++ b/apps/web/src/common/sqlite/sqlite.worker.ts
@@ -48,7 +48,7 @@ async function init(dbName: string, async: boolean, url?: string) {
? new IDBBatchAtomicVFS(dbName, { durability: "strict" })
: new AccessHandlePoolVFS(dbName);
if ("isReady" in vfs) await vfs.isReady;
- console.log(vfs, SQLiteAsyncModule);
+
sqlite.vfs_register(vfs, false);
db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);
}
@@ -70,15 +70,27 @@ async function prepare(sql: string) {
columns: sqlite.column_names(prepared.stmt)
};
preparedStatements.set(sql, statement);
+
+ sqlite.str_finish(str);
return statement;
}
-async function run(sql: string, parameters?: SQLiteCompatibleType[]) {
+async function run(
+ sql: string,
+ mode: RunMode,
+ parameters?: SQLiteCompatibleType[]
+) {
const prepared = await prepare(sql);
if (!prepared) return [];
try {
if (parameters) sqlite.bind_collection(prepared.stmt, parameters);
+ // fast path for exec statements
+ if (mode === "exec") {
+ while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW);
+ return [];
+ }
+
const rows: Record<string, SQLiteCompatibleType>[] = [];
while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW) {
const row = sqlite.row(prepared.stmt);
@@ -96,7 +108,7 @@ async function run(sql: string, parameters?: SQLiteCompatibleType[]) {
sqlite
.finalize(prepared.stmt)
// ignore error (we will just prepare a new statement)
- .catch(() => false)
+ .catch(console.error)
.finally(() => preparedStatements.delete(sql))
);
}
@@ -107,14 +119,11 @@ async function exec<R>(
sql: string,
parameters?: SQLiteCompatibleType[]
): Promise<QueryResult<R>> {
- console.time(sql);
- const rows = (await run(sql, parameters)) as R[];
- console.timeEnd(sql);
+ const rows = (await run(sql, mode, parameters)) as R[];
if (mode === "query") return { rows };
- const v = await run("SELECT last_insert_rowid() as id");
return {
- insertId: BigInt(v[0].id as number),
+ insertId: BigInt(sqlite.last_insert_rowid(db)),
numAffectedRows: BigInt(sqlite.changes(db)),
rows: mode === "raw" ? rows : []
};