Skip to content

Build your personal knowledge base with Trilium Notes

perf(becca): read the load queries as one JSON document

Elian Doran475a22cf66e7
diff --git a/packages/trilium-core/src/becca/becca_loader.spec.ts b/packages/trilium-core/src/becca/becca_loader.spec.ts
index f446d57859..3cf14b52e2 100644
--- a/packages/trilium-core/src/becca/becca_loader.spec.ts
+++ b/packages/trilium-core/src/becca/becca_loader.spec.ts
@@ -199,6 +199,31 @@ describe("becca_loader", () => {
expect(token?.name).toBe("loader-test-token");
expect(becca.loaded).toBe(true);
});
+
+ it("orders a note's children by notePosition rather than by insertion order", () => {
+ const parent = createNote("root");
+ const children = [
+ createNote(parent.noteId), createNote(parent.noteId), createNote(parent.noteId)
+ ];
+
+ getContext().init(() => {
+ // Reverse the positions, so the rowid order the table is scanned in disagrees
+ // with the order the children belong in.
+ const update = /*sql*/`UPDATE branches SET notePosition = ?
+ WHERE noteId = ? AND parentNoteId = ?`;
+ for (const [index, child] of children.entries()) {
+ const position = (children.length - index) * 10;
+ getSql().execute(update, [position, child.noteId, parent.noteId]);
+ }
+
+ load();
+ });
+
+ const loadedParent = becca.getNoteOrThrow(parent.noteId);
+ expect(loadedParent.children.map((child) => child.noteId)).toEqual(
+ [...children].reverse().map((child) => child.noteId)
+ );
+ });
});
describe("reload()", () => {
diff --git a/packages/trilium-core/src/becca/becca_loader.ts b/packages/trilium-core/src/becca/becca_loader.ts
index 1c9d113dbb..87f6f91278 100644
--- a/packages/trilium-core/src/becca/becca_loader.ts
+++ b/packages/trilium-core/src/becca/becca_loader.ts
@@ -1,4 +1,4 @@
-import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";
+import type { BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";
import eventService from "../services/events";
import entityConstructor from "../becca/entity_constructor.js";
@@ -35,22 +35,27 @@ function load() {
// we know this is slow and the total becca load time is logged
const sql = getSql();
sql.disableSlowQueryLogging(() => {
- // using a raw query and passing arrays to avoid allocating new objects,
- // this is worth it for the becca load since it happens every run and blocks the app until finished
+ // Rows arrive as positional arrays, which the entities consume without allocating
+ // an object per row, and in one bulk read rather than a column at a time. Both
+ // matter here: the load happens every run and blocks the app until it finishes.
- for (const row of sql.getRawRows(/*sql*/`SELECT noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) {
+ const notesFrom = /*sql*/`FROM notes WHERE isDeleted = 0`;
+ for (const row of sql.getRawRowsBulk(NOTE_COLUMNS, notesFrom)) {
new BNote().update(row).init();
}
- const branchRows = sql.getRawRows<BranchRow>(/*sql*/`SELECT branchId, noteId, parentNoteId, prefix, notePosition, isExpanded, utcDateModified FROM branches WHERE isDeleted = 0`);
+ const branchesFrom = /*sql*/`FROM branches WHERE isDeleted = 0`;
+ const branchRows = sql.getRawRowsBulk(BRANCH_COLUMNS, branchesFrom);
// in-memory sort is faster than in the DB
- branchRows.sort((a, b) => (a.notePosition || 0) - (b.notePosition || 0));
+ const positionIdx = BRANCH_COLUMNS.indexOf("notePosition");
+ branchRows.sort((a, b) => (Number(a[positionIdx]) || 0) - (Number(b[positionIdx]) || 0));
for (const row of branchRows) {
new BBranch().update(row).init();
}
- for (const row of sql.getRawRows<AttributeRow>(/*sql*/`SELECT attributeId, noteId, type, name, value, isInheritable, position, utcDateModified FROM attributes WHERE isDeleted = 0`)) {
+ const attributesFrom = /*sql*/`FROM attributes WHERE isDeleted = 0`;
+ for (const row of sql.getRawRowsBulk(ATTRIBUTE_COLUMNS, attributesFrom)) {
new BAttribute().update(row).init();
}
@@ -73,6 +78,19 @@ function load() {
getLog().info(`Becca (note cache) load took ${Date.now() - start}ms`);
}
+// The order each entity's update() destructures its row in. None of these columns holds a
+// BLOB or an integer large enough for getRawRowsBulk()'s JSON round-trip to lose precision.
+const NOTE_COLUMNS = [
+ "noteId", "title", "type", "mime", "isProtected", "blobId",
+ "dateCreated", "dateModified", "utcDateCreated", "utcDateModified"
+];
+const BRANCH_COLUMNS = [
+ "branchId", "noteId", "parentNoteId", "prefix", "notePosition", "isExpanded", "utcDateModified"
+];
+const ATTRIBUTE_COLUMNS = [
+ "attributeId", "noteId", "type", "name", "value", "isInheritable", "position", "utcDateModified"
+];
+
function reload(reason: string) {
load();
diff --git a/packages/trilium-core/src/services/sql/sql.spec.ts b/packages/trilium-core/src/services/sql/sql.spec.ts
index f6a05052ce..b60de25e18 100644
--- a/packages/trilium-core/src/services/sql/sql.spec.ts
+++ b/packages/trilium-core/src/services/sql/sql.spec.ts
@@ -154,6 +154,31 @@ describe("SqlService (real DB)", () => {
expect(raw).toEqual([["x", 7]]);
});
+ it("getRawRowsBulk returns the same positional arrays getRawRows does", () => {
+ const table = createTempTable();
+ getSql().insert(table, { name: "x", val: 7 });
+ getSql().insert(table, { name: null, val: 0 });
+ getSql().insert(table, { name: `quote" and \\ backslash and ünïcode ☃`, val: -3 });
+
+ const columns = ["name", "val"];
+ const from = /*sql*/`FROM "${table}" ORDER BY rowid`;
+
+ expect(getSql().getRawRowsBulk<[string, number]>(columns, from)).toEqual(
+ getSql().getRawRows<[string, number]>(`SELECT ${columns.join(", ")} ${from}`)
+ );
+ });
+
+ it("getRawRowsBulk returns an empty array when nothing matches, and binds params", () => {
+ const table = createTempTable();
+ getSql().insert(table, { name: "keep", val: 1 });
+ getSql().insert(table, { name: "drop", val: 2 });
+
+ const name = ["name"];
+ expect(getSql().getRawRowsBulk(name, /*sql*/`FROM "${table}" WHERE val > 99`)).toEqual([]);
+ expect(getSql().getRawRowsBulk(name, /*sql*/`FROM "${table}" WHERE val = ?`, [1]))
+ .toEqual([["keep"]]);
+ });
+
it("iterateRows yields rows lazily", () => {
const table = createTempTable();
getSql().insert(table, { name: "i1", val: 1 });
diff --git a/packages/trilium-core/src/services/sql/sql.ts b/packages/trilium-core/src/services/sql/sql.ts
index 22b5515244..c2b680e3fe 100644
--- a/packages/trilium-core/src/services/sql/sql.ts
+++ b/packages/trilium-core/src/services/sql/sql.ts
@@ -240,6 +240,35 @@ export class SqlService {
return (this.wrap(query, (s) => s.all(params), "raw") as T[]) || [];
}
+ /**
+ * Gets every row of a whole-table read as a positional array, the way {@link getRawRows} does,
+ * but crossing between JavaScript and the database once instead of once per column.
+ *
+ * `@sqlite.org/sqlite-wasm` charges roughly 7µs per column a row exposes, so the Becca
+ * load spends most of its time marshalling rather than querying. `json_group_array()`
+ * hands the result set over as one value, which `JSON.parse()` turns into rows about ten
+ * times faster; better-sqlite3 is marginally faster this way too.
+ *
+ * Going through JSON puts two limits on `columns` that {@link getRawRows} does not have:
+ * `json_array()` rejects a BLOB column, and an integer above `Number.MAX_SAFE_INTEGER`
+ * arrives rounded here where {@link getRawRows} returns a BigInt.
+ *
+ * @param columns - the columns to read, in the order the returned arrays carry them
+ * @param fromClause - the rest of the query from `FROM` on, with ? as parameter
+ * placeholder. It must select a single group, since one JSON document holds every row.
+ * @param params - array of params if needed
+ */
+ getRawRowsBulk<T extends unknown[]>(
+ columns: string[],
+ fromClause: string,
+ params: Params = []
+ ): T[] {
+ const query = `SELECT json_group_array(json_array(${columns.join(", ")})) ${fromClause}`;
+ const json = this.getValue<string | null>(query, params);
+
+ return json ? JSON.parse(json) : [];
+ }
+
iterateRows<T>(query: string, params: Params = []): IterableIterator<T> {
if (LOG_ALL_QUERIES) {
console.log(query);