A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: improve notes lookup performance
Abdullah Atta456927548a0d
diff --git a/packages/core/__benches__/lookup.bench.ts b/packages/core/__benches__/lookup.bench.tsnew file mode 100644index 000000000..76765a797--- /dev/null+++ b/packages/core/__benches__/lookup.bench.ts@@ -0,0 +1,65 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { bench, describe } from "vitest";+import { databaseTest } from "../__tests__/utils";+import Database from "../src/api";++async function addNotes(db: Database) {+ const words = [+ "title",+ "who",+ "what",+ "never",+ "works",+ "here",+ "say",+ "there"+ ];+ for (let i = 0; i < 10000; ++i) {+ await db.notes.add({+ title: new Array(10)+ .fill(0)+ .map(() => words[getRandom(0, words.length)])+ .join(" "),+ content: {+ type: "tiptap",+ data: new Array(50)+ .fill(0)+ .map(() => words[getRandom(0, words.length)])+ .join(" ")+ }+ });+ if (i % 100 === 0) console.log(i);+ }+ console.log("DONE");+}++describe("lookup", async () => {+ const db = await databaseTest();+ await addNotes(db);++ bench("using multiple queries", async () => {+ await db.lookup.notes("never works here say").ids();+ });+});++function getRandom(min: number, max: number) {+ return Math.round(Math.random() * (max - min) + min);+}diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex cc2ee075f..75e97e169 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -20,7 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { match } from "fuzzyjs";import Database from ".";import { Item, Note, TrashItem } from "../types";-import { DatabaseSchema, RawDatabaseSchema, isFalse } from "../database";+import { DatabaseSchema, RawDatabaseSchema } from "../database";import { AnyColumnWithTable, Kysely, sql } from "kysely";import { FilteredSelector } from "../database/sql-collection";import { VirtualizedGrouping } from "../utils/virtualized-grouping";@@ -45,34 +45,41 @@ export default class Lookup {const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;query = query.replace(/"/, '""');- const result = await db- .with("matching", (eb) =>++ const excludedIds = this.db.trash.cache.notes;+ const results = await db+ .selectFrom((eb) =>eb- .selectFrom("content_fts")- .where("data", "match", `"${query}"`)- .select(["noteId as id", "rank"])- .unionAll(+ .selectFrom("notes_fts")+ .$if(!!notes, (eb) =>+ eb.where("id", "in", notes!.filter.select("id"))+ )+ .$if(excludedIds.length > 0, (eb) =>+ eb.where("id", "not in", excludedIds)+ )+ .where("title", "match", `"${query}"`)+ .select(["id", sql<number>`rank * 10`.as("rank")])+ .unionAll((eb) =>eb- .selectFrom("notes_fts")- .where("title", "match", `"${query}"`)- // add 10 weight to title- .select(["id", sql.raw<number>(`rank * 10`).as("rank")])+ .selectFrom("content_fts")+ .$if(!!notes, (eb) =>+ eb.where("id", "in", notes!.filter.select("id"))+ )+ .$if(excludedIds.length > 0, (eb) =>+ eb.where("id", "not in", excludedIds)+ )+ .where("data", "match", `"${query}"`)+ .select(["noteId as id", "rank"])+ .$castTo<{ id: string; rank: number }>())+ .as("results"))- .selectFrom("notes")- .$if(!!notes, (eb) =>- eb.where("notes.id", "in", notes!.filter.select("id"))- )+ .select(["results.id"])+ .groupBy("results.id")+ .orderBy(sql`SUM(results.rank)`, "asc").$if(!!limit, (eb) => eb.limit(limit!))- .where(isFalse("notes.deleted"))- .where(isFalse("notes.dateDeleted"))- .innerJoin("matching", (eb) =>- eb.onRef("notes.id", "==", "matching.id")- )- .orderBy("matching.rank desc")- .select(["notes.id"]).execute();- return result.map((id) => id.id);+ return results.map((r) => r.id);}, notes || this.db.notes.all);}@@ -212,10 +219,10 @@ export default class Lookup {this.db.options.batchSize,() => Promise.resolve(ids),async (start, end) => {- const items = await selector.items(ids);+ const items = await selector.records(ids);return {ids: ids.slice(start, end),- items: items.slice(start, end)+ items: Object.values(items).slice(start, end)};});