Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
perf(becca): read the load queries as one JSON document
Elian Doran475a22cf66e7
diff --git a/packages/trilium-core/src/becca/becca_loader.spec.ts b/packages/trilium-core/src/becca/becca_loader.spec.tsindex f446d57859..3cf14b52e2 100644--- a/packages/trilium-core/src/becca/becca_loader.spec.ts+++ b/packages/trilium-core/src/becca/becca_loader.spec.ts@@ -199,6 +199,31 @@ describe("becca_loader", () => {expect(token?.name).toBe("loader-test-token");expect(becca.loaded).toBe(true);});++ it("orders a note's children by notePosition rather than by insertion order", () => {+ const parent = createNote("root");+ const children = [+ createNote(parent.noteId), createNote(parent.noteId), createNote(parent.noteId)+ ];++ getContext().init(() => {+ // Reverse the positions, so the rowid order the table is scanned in disagrees+ // with the order the children belong in.+ const update = /*sql*/`UPDATE branches SET notePosition = ?+ WHERE noteId = ? AND parentNoteId = ?`;+ for (const [index, child] of children.entries()) {+ const position = (children.length - index) * 10;+ getSql().execute(update, [position, child.noteId, parent.noteId]);+ }++ load();+ });++ const loadedParent = becca.getNoteOrThrow(parent.noteId);+ expect(loadedParent.children.map((child) => child.noteId)).toEqual(+ [...children].reverse().map((child) => child.noteId)+ );+ });});describe("reload()", () => {diff --git a/packages/trilium-core/src/becca/becca_loader.ts b/packages/trilium-core/src/becca/becca_loader.tsindex 1c9d113dbb..87f6f91278 100644--- a/packages/trilium-core/src/becca/becca_loader.ts+++ b/packages/trilium-core/src/becca/becca_loader.ts@@ -1,4 +1,4 @@-import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";+import type { BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";import eventService from "../services/events";import entityConstructor from "../becca/entity_constructor.js";@@ -35,22 +35,27 @@ function load() {// we know this is slow and the total becca load time is loggedconst sql = getSql();sql.disableSlowQueryLogging(() => {- // using a raw query and passing arrays to avoid allocating new objects,- // this is worth it for the becca load since it happens every run and blocks the app until finished+ // Rows arrive as positional arrays, which the entities consume without allocating+ // an object per row, and in one bulk read rather than a column at a time. Both+ // matter here: the load happens every run and blocks the app until it finishes.- for (const row of sql.getRawRows(/*sql*/`SELECT noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) {+ const notesFrom = /*sql*/`FROM notes WHERE isDeleted = 0`;+ for (const row of sql.getRawRowsBulk(NOTE_COLUMNS, notesFrom)) {new BNote().update(row).init();}- const branchRows = sql.getRawRows<BranchRow>(/*sql*/`SELECT branchId, noteId, parentNoteId, prefix, notePosition, isExpanded, utcDateModified FROM branches WHERE isDeleted = 0`);+ const branchesFrom = /*sql*/`FROM branches WHERE isDeleted = 0`;+ const branchRows = sql.getRawRowsBulk(BRANCH_COLUMNS, branchesFrom);// in-memory sort is faster than in the DB- branchRows.sort((a, b) => (a.notePosition || 0) - (b.notePosition || 0));+ const positionIdx = BRANCH_COLUMNS.indexOf("notePosition");+ branchRows.sort((a, b) => (Number(a[positionIdx]) || 0) - (Number(b[positionIdx]) || 0));for (const row of branchRows) {new BBranch().update(row).init();}- for (const row of sql.getRawRows<AttributeRow>(/*sql*/`SELECT attributeId, noteId, type, name, value, isInheritable, position, utcDateModified FROM attributes WHERE isDeleted = 0`)) {+ const attributesFrom = /*sql*/`FROM attributes WHERE isDeleted = 0`;+ for (const row of sql.getRawRowsBulk(ATTRIBUTE_COLUMNS, attributesFrom)) {new BAttribute().update(row).init();}@@ -73,6 +78,19 @@ function load() {getLog().info(`Becca (note cache) load took ${Date.now() - start}ms`);}+// The order each entity's update() destructures its row in. None of these columns holds a+// BLOB or an integer large enough for getRawRowsBulk()'s JSON round-trip to lose precision.+const NOTE_COLUMNS = [+ "noteId", "title", "type", "mime", "isProtected", "blobId",+ "dateCreated", "dateModified", "utcDateCreated", "utcDateModified"+];+const BRANCH_COLUMNS = [+ "branchId", "noteId", "parentNoteId", "prefix", "notePosition", "isExpanded", "utcDateModified"+];+const ATTRIBUTE_COLUMNS = [+ "attributeId", "noteId", "type", "name", "value", "isInheritable", "position", "utcDateModified"+];+function reload(reason: string) {load();diff --git a/packages/trilium-core/src/services/sql/sql.spec.ts b/packages/trilium-core/src/services/sql/sql.spec.tsindex f6a05052ce..b60de25e18 100644--- a/packages/trilium-core/src/services/sql/sql.spec.ts+++ b/packages/trilium-core/src/services/sql/sql.spec.ts@@ -154,6 +154,31 @@ describe("SqlService (real DB)", () => {expect(raw).toEqual([["x", 7]]);});+ it("getRawRowsBulk returns the same positional arrays getRawRows does", () => {+ const table = createTempTable();+ getSql().insert(table, { name: "x", val: 7 });+ getSql().insert(table, { name: null, val: 0 });+ getSql().insert(table, { name: `quote" and \\ backslash and ünïcode ☃`, val: -3 });++ const columns = ["name", "val"];+ const from = /*sql*/`FROM "${table}" ORDER BY rowid`;++ expect(getSql().getRawRowsBulk<[string, number]>(columns, from)).toEqual(+ getSql().getRawRows<[string, number]>(`SELECT ${columns.join(", ")} ${from}`)+ );+ });++ it("getRawRowsBulk returns an empty array when nothing matches, and binds params", () => {+ const table = createTempTable();+ getSql().insert(table, { name: "keep", val: 1 });+ getSql().insert(table, { name: "drop", val: 2 });++ const name = ["name"];+ expect(getSql().getRawRowsBulk(name, /*sql*/`FROM "${table}" WHERE val > 99`)).toEqual([]);+ expect(getSql().getRawRowsBulk(name, /*sql*/`FROM "${table}" WHERE val = ?`, [1]))+ .toEqual([["keep"]]);+ });+it("iterateRows yields rows lazily", () => {const table = createTempTable();getSql().insert(table, { name: "i1", val: 1 });diff --git a/packages/trilium-core/src/services/sql/sql.ts b/packages/trilium-core/src/services/sql/sql.tsindex 22b5515244..c2b680e3fe 100644--- a/packages/trilium-core/src/services/sql/sql.ts+++ b/packages/trilium-core/src/services/sql/sql.ts@@ -240,6 +240,35 @@ export class SqlService {return (this.wrap(query, (s) => s.all(params), "raw") as T[]) || [];}+ /**+ * Gets every row of a whole-table read as a positional array, the way {@link getRawRows} does,+ * but crossing between JavaScript and the database once instead of once per column.+ *+ * `@sqlite.org/sqlite-wasm` charges roughly 7µs per column a row exposes, so the Becca+ * load spends most of its time marshalling rather than querying. `json_group_array()`+ * hands the result set over as one value, which `JSON.parse()` turns into rows about ten+ * times faster; better-sqlite3 is marginally faster this way too.+ *+ * Going through JSON puts two limits on `columns` that {@link getRawRows} does not have:+ * `json_array()` rejects a BLOB column, and an integer above `Number.MAX_SAFE_INTEGER`+ * arrives rounded here where {@link getRawRows} returns a BigInt.+ *+ * @param columns - the columns to read, in the order the returned arrays carry them+ * @param fromClause - the rest of the query from `FROM` on, with ? as parameter+ * placeholder. It must select a single group, since one JSON document holds every row.+ * @param params - array of params if needed+ */+ getRawRowsBulk<T extends unknown[]>(+ columns: string[],+ fromClause: string,+ params: Params = []+ ): T[] {+ const query = `SELECT json_group_array(json_array(${columns.join(", ")})) ${fromClause}`;+ const json = this.getValue<string | null>(query, params);++ return json ? JSON.parse(json) : [];+ }+iterateRows<T>(query: string, params: Params = []): IterableIterator<T> {if (LOG_ALL_QUERIES) {console.log(query);