A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: add support for incremental grouping
Abdullah Atta96e36a6f7344
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex 66d8fb27c..85755b5ab 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/+import dayjs from "dayjs";import Database from "../src/api";import { groupArray } from "../src/utils/grouping";import {@@ -29,6 +30,7 @@ import {loginFakeUser} from "./utils";import { test, expect } from "vitest";+import { MONTHS_FULL } from "../src/utils/date";async function createAndAddNoteToNotebook(db: Database,@@ -192,16 +194,6 @@ test("get pinned notes", () =>expect(await db.notes.pinned.count()).toBeGreaterThan(0);}));-test.todo("get grouped notes by abc", () => groupedTest("abc"));--test.todo("get grouped notes by month", () => groupedTest("month"));--test.todo("get grouped notes by year", () => groupedTest("year"));--test.todo("get grouped notes by weak", () => groupedTest("week"));--test.todo("get grouped notes default", () => groupedTest("default"));-test("pin note", () =>noteTest().then(async ({ db, id }) => {await db.notes.pin(true, id);@@ -388,3 +380,151 @@ test("adding a note with an invalid tag should clean the tag array", () =>await db.relations.to({ id: "helloworld", type: "note" }, "tag").count()).toBe(0);}));++test("get grouped notes by abc", () =>+ databaseTest().then(async (db) => {+ const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";+ for (const letter of alphabet) {+ for (const letter2 of alphabet) {+ await db.notes.add({ title: `${letter}${letter2}` });+ }+ }+ await db.notes.add({ title: `Pinned note`, pinned: true });+ await db.notes.add({+ title: `Conflicted`,+ conflicted: true+ });+ const { grouping, ids } = await db.notes.all.grouped({+ groupBy: "abc",+ sortDirection: "asc",+ sortBy: "title"+ });++ expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");+ expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");+ for (let i = 0; i < alphabet.length; ++i) {+ expect(+ (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title+ ).toBe(alphabet[i]);+ }+ }));++test("get grouped notes by month", () =>+ databaseTest().then(async (db) => {+ for (let month = 0; month <= 11; ++month) {+ for (let i = 0; i < 5; ++i) {+ const date = dayjs().month(month).toDate().getTime();+ await db.notes.add({+ title: `Note in ${month} - ${i}`,+ dateCreated: date+ });+ }+ }++ const { grouping, ids } = await db.notes.all.grouped({+ groupBy: "month",+ sortDirection: "desc",+ sortBy: "dateCreated"+ });++ for (let month = 11; month >= 0; --month) {+ expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(+ MONTHS_FULL[month]+ );+ }+ }));++test("get grouped notes by year", () =>+ databaseTest().then(async (db) => {+ for (let year = 2020; year <= 2025; ++year) {+ for (let i = 0; i < 5; ++i) {+ const date = dayjs().year(year).toDate().getTime();+ await db.notes.add({+ title: `Note in ${year} - ${i}`,+ dateCreated: date+ });+ }+ }++ const { grouping, ids } = await db.notes.all.grouped({+ groupBy: "year",+ sortDirection: "desc",+ sortBy: "dateCreated"+ });++ for (let year = 2020; year <= 2025; ++year) {+ expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(+ year.toString()+ );+ }+ }));++test("get grouped notes by week", () =>+ databaseTest().then(async (db) => {+ for (let i = 1; i <= 6; ++i) {+ for (let week = 1; week <= 4; ++week) {+ const date = dayjs()+ .month(i)+ .date(week * 7)+ .year(2023)+ .startOf("week")+ .toDate()+ .getTime();+ await db.notes.add({+ title: `Note in ${week} - ${i}`,+ dateCreated: date+ });+ }+ }++ const { grouping, ids } = await db.notes.all.grouped({+ groupBy: "week",+ sortDirection: "desc",+ sortBy: "dateCreated"+ });++ const weeks = [+ "19 - 25 Jun, 2023",+ "22 - 28 May, 2023",+ "17 - 23 Apr, 2023",+ "20 - 26 Mar, 2023",+ "20 - 26 Feb, 2023"+ ];+ for (let i = 1; i <= 5; ++i) {+ expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(+ weeks[i - 1]+ );+ }+ }));++test("get grouped notes default", () =>+ databaseTest().then(async (db) => {+ const ranges = {+ Recent: [0, 7],+ "Last week": [7, 14],+ Older: [14, 28]+ };+ for (const key in ranges) {+ const range = ranges[key];+ for (let i = range[0]; i < range[1]; i++) {+ const date = dayjs().subtract(i, "days").toDate().getTime();++ await db.notes.add({+ title: `Note in ${key} - ${i}`,+ dateCreated: date+ });+ }+ }++ const { grouping, ids } = await db.notes.all.grouped({+ groupBy: "default",+ sortDirection: "desc",+ sortBy: "dateCreated"+ });++ let i = 0;+ for (const key in ranges) {+ expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);+ ++i;+ }+ }));diff --git a/packages/core/__tests__/utils.test.js b/packages/core/__tests__/utils.test.jsdeleted file mode 100644index 7b100e7d9..000000000--- a/packages/core/__tests__/utils.test.js+++ /dev/null@@ -1,41 +0,0 @@-/*-This file is part of the Notesnook project (https://notesnook.com/)--Copyright (C) 2023 Streetwriters (Private) Limited--This program is free software: you can redistribute it and/or modify-it under the terms of the GNU General Public License as published by-the Free Software Foundation, either version 3 of the License, or-(at your option) any later version.--This program is distributed in the hope that it will be useful,-but WITHOUT ANY WARRANTY; without even the implied warranty of-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-GNU General Public License for more details.--You should have received a copy of the GNU General Public License-along with this program. If not, see <http://www.gnu.org/licenses/>.-*/--import { groupArray } from "../src/utils/grouping";-import { test, expect } from "vitest";--test("group alphabetically", () => {- const sortedAlphabet = "abcdefghijlmnopqrstuvwxyz"- .split("")- .map((a) => ({ title: a }));- const alphabet = "nopqrstuvwxyzabcdefghijlm"- .split("")- .map((a) => ({ title: a, item: true }));- let ret = groupArray(alphabet, {- groupBy: "abc",- sortDirection: "asc",- sortBy: "title"- }).filter((v) => !v.item);-- expect(- sortedAlphabet.every((alpha, index) => {- return ret[index].title === alpha.title.toUpperCase();- })- ).toBeTruthy();-});diff --git a/packages/core/incremental-grouping.md b/packages/core/incremental-grouping.mdnew file mode 100644index 000000000..d2df4cdc7--- /dev/null+++ b/packages/core/incremental-grouping.md@@ -0,0 +1,81 @@+# Incremental grouping of list items++## Requirements++1. We do not want to bring all the items to the client side as that would take a lot of memory+2. We want to preserve & customize sorting+3. We want to avoid creating duplicate groups+4. Each batch of items should append and add groups as necesary++## Approach++1. Get all the IDs of all the items sorted using the required configuration+2. Request the first batch of `n` items.+3. Group the first batch+4. Bring the second batch and similarly group it.+5. If the last group of the previous batch & the first group of the new batch match up, we can remove the new group, otherwise keep it.++Example:++Items: A, Aa, B, Ba, Bb, C, Ca, Cb, Cc, Cd+Batch size: 2++Batch 1: [A, Aa]+Grouped: [A, [A, Aa]]++Batch 2: [B, Ba]+Grouped: [B, [B, Ba]]++Merged: [A, [A, Aa], B, [B, Ba]]++Batch 3: [Bb, C]+Grouped: [B, [Bb], C, [C]]++Merged: [A, [A, Aa], B, [B, Ba, Bb], C, [C]]++This is a simple incremental grouping. However, what we require is windowed grouping.++## Windowed incremental grouping of list items++Windowed grouping means that we only keep `n` number of items in memory at any point in time.+As the user scrolls up & down, we fetch only the batch that is needed for rendering the items+in the current window.++Requirements:++1. Only keep N items in memory+2. Remove the previous window's items as soon as we don't need them+3. Respect sorting & grouping while doing append/prepend++Approach:++1. Get all the IDs of all the items sorted using the required configuration+2. Request the first batch of `n` items.+3. Group the first batch+4. If user is moving downward:+ 1. When user moves closer to the the end of the first batch, load the next batch & group it accordingly+ 2. Compare last & first groups of both batches+ 3. If last & first groups are the same, remove the first group of the new batch+ 4. Otherwise keep it & do nothing+ 5. If user reaches the end of the second batch, remove the first batch+5. If user is moving upward:+ Suppose we have batches 2 & 3 in memory, user is at the end of batch 2 and starts scrolling upwards:+ 1. When user reaches closer to the start of the batch 2, load batch 1 & remove batch 3+ 2. For grouping compare the last & first groups of batch 1 & 2 respectively+ 3. Remove first group of batch 2 if both match, otherwise do nothing++Questions:++1. What if new items are added at random places during sync or something else?++The naive but stable approach would be to refetch everything and recalculate the current batch.+Another approach would be to diff the new & old set of IDs and determine which batches have+changed.++We can then reload the batches only if they are currently cached, otherwise do nothing.++2. What will be the responsibility of the core module?++The core module will only manage the cache. It will also be responsible for loading the appropriate+batch based on if the item is found in the cache or not. This should be automatic for maximum+optimization opportunities.diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex 48bd481c3..b3be4baab 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { EVENTS } from "../common";-import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types";+import { GroupOptions, Item, MaybeDeletedItem, isDeleted } from "../types";import EventManager from "../utils/event-manager";import {DatabaseAccessor,@@ -28,6 +28,8 @@ import {isFalse} from ".";import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely";+import { VirtualizedGrouping } from "../utils/virtualized-grouping";+import { groupArray } from "../utils/grouping";export class SQLCollection<TCollectionType extends keyof DatabaseSchema,@@ -164,16 +166,12 @@ export class SQLCollection<}async items(- ids: string[],- sortOptions?: GroupOptions+ ids: string[]): Promise<Record<string, MaybeDeletedItem<T> | undefined>> {const results = await this.db().selectFrom<keyof DatabaseSchema>(this.type).selectAll().where("id", "in", ids)- .$if(!!sortOptions, (eb) =>- eb.orderBy(sortOptions!.sortBy, sortOptions!.sortDirection)- ).execute();const items: Record<string, MaybeDeletedItem<T>> = {};for (const item of results) {@@ -227,7 +225,7 @@ export class SQLCollection<}}- createFilter<T>(+ createFilter<T extends Item>(selector: (qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>) => SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>,@@ -240,7 +238,7 @@ export class SQLCollection<}}-export class FilteredSelector<T> {+export class FilteredSelector<T extends Item> {constructor(readonly filter: SelectQueryBuilder<DatabaseSchema,@@ -250,13 +248,23 @@ export class FilteredSelector<T> {readonly batchSize: number) {}- async ids() {- return (await this.filter.select("id").execute()).map((i) => i.id);+ async ids(sortOptions?: GroupOptions) {+ return (+ await this.filter+ .$if(!!sortOptions, (eb) =>+ eb.$call(this.buildSortExpression(sortOptions!))+ )+ .select("id")+ .execute()+ ).map((i) => i.id);}- async items(ids?: string[]) {+ async items(ids?: string[], sortOptions?: GroupOptions) {return (await this.filter.$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!))+ .$if(!!sortOptions, (eb) =>+ eb.$call(this.buildSortExpression(sortOptions!))+ ).selectAll().execute()) as T[];}@@ -298,6 +306,40 @@ export class FilteredSelector<T> {}}+ async grouped(options: GroupOptions) {+ const ids = await this.ids(options);+ return {+ ids,+ grouping: new VirtualizedGrouping<T>(+ ids,+ this.batchSize,+ async (ids) => {+ const results = await this.filter+ .where("id", "in", ids)+ .selectAll()+ .execute();+ const items: Record<string, T> = {};+ for (const item of results) {+ items[item.id] = item as T;+ }+ return items;+ },+ (ids, items) => groupArray(ids, items, options)+ )+ };+ }++ private buildSortExpression(options: GroupOptions) {+ return <T>(+ qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, T>+ ) => {+ return qb+ .orderBy("conflicted desc")+ .orderBy("pinned desc")+ .orderBy(options.sortBy, options.sortDirection);+ };+ }+async *[Symbol.asyncIterator]() {let index = 0;while (true) {diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex 80792b0bc..f0d13021b 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -81,6 +81,7 @@ export type GroupableItem = ValueOf<| "shortcut"| "relation"| "tiny"+ | "topic"| "tiptap"| "content"| "session"diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.tsnew file mode 100644index 000000000..e85e77d6d--- /dev/null+++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts@@ -0,0 +1,162 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { expect, test, vi } from "vitest";+import { VirtualizedGrouping } from "../virtualized-grouping";++function item<T>(value: T) {+ return { item: value };+}+function createMock() {+ return vi.fn(async (ids: string[]) =>+ Object.fromEntries(ids.map((id) => [id, id]))+ );+}+test("fetch items in batch if not found in cache", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7"],+ 3,+ mocked+ );+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(mocked).toHaveBeenCalledOnce();+});++test("do not fetch items in batch if found in cache", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7"],+ 3,+ mocked+ );+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(mocked).toHaveBeenCalledOnce();+});++test("clear old cached batches", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],+ 3,+ mocked+ );+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));+ t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);+ t.expect(await grouping.item("7")).toStrictEqual(item("7"));+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);+});++test("clear old cached batches (random access)", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],+ 3,+ mocked+ );+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);++ t.expect(await grouping.item("7")).toStrictEqual(item("7"));+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);++ t.expect(await grouping.item("11")).toStrictEqual(item("11"));+ t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);++ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);++ t.expect(await grouping.item("7")).toStrictEqual(item("7"));+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);+});++test("reloading ids should clear all cached batches", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "3", "4", "5", "7", "6", "50"],+ 3,+ mocked+ );++ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);++ grouping.refresh([+ "1",+ "2",+ "3",+ "4",+ "5",+ "6",+ "7",+ "8",+ "9",+ "10",+ "11",+ "12"+ ]);++ t.expect(await grouping.item("1")).toStrictEqual(item("1"));+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);+});++test("merge groups if last & first groups are the same (sequential)", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],+ 3,+ mocked,+ (ids) => [{ title: "Hello", id: ids[0] }]+ );+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");+ expect((await grouping.item("4"))?.group).toBeUndefined();+});++test("merge groups if last & first groups are the same (random)", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],+ 3,+ mocked,+ (ids) => [{ title: "Hello", id: ids[0] }]+ );+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");+ expect((await grouping.item("7"))?.group).toBeUndefined();+});++test("merge groups if last & first groups are the same (reverse)", async (t) => {+ const mocked = createMock();+ const grouping = new VirtualizedGrouping<string>(+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],+ 3,+ mocked,+ (ids) => [{ title: "Hello", id: ids[0] }]+ );+ expect((await grouping.item("7"))?.group?.title).toBe("Hello");+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");+ expect((await grouping.item("7"))?.group).toBeUndefined();+});diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.tsindex 905928ecb..853e811da 100644--- a/packages/core/src/utils/grouping.ts+++ b/packages/core/src/utils/grouping.ts@@ -18,14 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { isReminderActive } from "../collections/reminders";-import {- GroupedItems,- GroupOptions,- GroupableItem,- Item,- Reminder-} from "../types";+import { GroupOptions, Item } from "../types";import { getWeekGroupFromTimestamp, MONTHS_FULL } from "./date";+import { VirtualizedGroupHeader } from "./virtualized-grouping";type EvaluateKeyFunction<T> = (item: T) => string;@@ -45,33 +40,19 @@ export const getSortValue = <T extends Item>(return item.dateCreated;};-function getSortSelectors<T extends GroupableItem>(options: GroupOptions) {- if (options.sortBy === "title")- return {- asc: (a: T, b: T) =>- getTitle(a).localeCompare(getTitle(b), undefined, { numeric: true }),- desc: (a: T, b: T) =>- getTitle(b).localeCompare(getTitle(a), undefined, { numeric: true })- };-- return {- asc: (a: T, b: T) => getSortValue(options, a) - getSortValue(options, b),- desc: (a: T, b: T) => getSortValue(options, b) - getSortValue(options, a)- };-}-const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;-function getKeySelector<T extends GroupableItem>(- options: GroupOptions-): EvaluateKeyFunction<T> {- return (item: T) => {+function getKeySelector(options: GroupOptions): EvaluateKeyFunction<Item> {+ return (item: Item) => {if ("pinned" in item && item.pinned) return "Pinned";else if ("conflicted" in item && item.conflicted) return "Conflicted";const date = new Date();- if (options.sortBy === "title") return getFirstCharacter(getTitle(item));+ if (item.type === "reminder")+ return isReminderActive(item) ? "Active" : "Inactive";+ else if (options.sortBy === "title")+ return getFirstCharacter(getTitle(item));else {const value = getSortValue(options, item);switch (options.groupBy) {@@ -98,78 +79,37 @@ function getKeySelector<T extends GroupableItem>(};}-export function groupArray<T extends GroupableItem>(- array: T[],+export function groupArray(+ ids: string[],+ items: Record<string, Item>,options: GroupOptions = {groupBy: "default",sortBy: "dateEdited",sortDirection: "desc"}-): GroupedItems<T> {- if (options.sortBy && options.sortDirection) {- const selector = getSortSelectors(options)[options.sortDirection];- array.sort(selector);- }-- if (options.groupBy === "none") {- const conflicted: T[] = [];- const pinned: T[] = [];- const others: T[] = [];- for (const item of array) {- if ("pinned" in item && item.pinned) {- pinned.push(item);- continue;- } else if ("conflicted" in item && item.conflicted) {- conflicted.push(item);- continue;- } else others.push(item);- }- const groups: GroupedItems<T> = [];- if (conflicted.length > 0)- groups.push(- { title: "Conflicted", type: "header", id: "conflicted" },- ...conflicted- );- if (pinned.length > 0)- groups.push({ title: "Pinned", type: "header", id: "pinned" }, ...pinned);- if (others.length > 0)- groups.push({ title: "All", type: "header", id: "all" }, ...others);- return groups;- }-- const groups = new Map<string, T[]>([- ["Conflicted", []],- ["Pinned", []]+): VirtualizedGroupHeader[] {+ const groups = new Map<string, VirtualizedGroupHeader>([+ ["Conflicted", { title: "Conflicted", id: "" }],+ ["Pinned", { title: "Pinned", id: "" }],+ ["Active", { title: "Active", id: "" }],+ ["Inactive", { title: "Inactive", id: "" }]]);const keySelector = getKeySelector(options);- array.forEach((item) => addToGroup(groups, keySelector(item), item));-- return flattenGroups(groups);-}+ for (const id of ids) {+ const item = items[id];+ if (!item) continue;-export function groupReminders(array: Reminder[]): GroupedItems<Reminder> {- const groups = new Map([- ["Active", []],- ["Inactive", []]- ]);-- array.forEach((item) => {- const groupTitle = isReminderActive(item) ? "Active" : "Inactive";- addToGroup(groups, groupTitle, item);- });-- return flattenGroups(groups);-}+ const groupTitle = keySelector(item);+ const group = groups.get(groupTitle) || {+ title: groupTitle,+ id: ""+ };+ if (group.id === "") group.id = id;+ groups.set(groupTitle, group);+ }-function addToGroup<T extends GroupableItem>(- groups: Map<string, T[]>,- groupTitle: string,- item: T-) {- const group = groups.get(groupTitle) || [];- group.push(item);- groups.set(groupTitle, group);+ return Array.from(groups.values());}function getFirstCharacter(str: string) {@@ -179,21 +119,10 @@ function getFirstCharacter(str: string) {return str[0].toUpperCase();}-function getTitle<T extends GroupableItem>(item: T): string {- return item.type === "attachment" ? item.metadata.filename : item.title;-}--function flattenGroups<T extends GroupableItem>(groups: Map<string, T[]>) {- const items: GroupedItems<T> = [];- groups.forEach((groupItems, groupTitle) => {- if (groupItems.length <= 0) return;- items.push({- title: groupTitle,- id: groupTitle.toLowerCase(),- type: "header"- });- groupItems.forEach((item) => items.push(item));- });-- return items;+function getTitle(item: Item): string {+ return item.type === "attachment"+ ? item.filename+ : "title" in item+ ? item.title+ : "Unknown";}diff --git a/packages/core/src/utils/set.ts b/packages/core/src/utils/set.tsindex 683253dbc..19e414178 100644--- a/packages/core/src/utils/set.ts+++ b/packages/core/src/utils/set.ts@@ -85,14 +85,14 @@ class SetManipulator {// Symmetric difference. Items from either set that// are not in both sets.// Set.difference([1, 1, 2], [2, 3, 3]) => [1, 3]- difference<T>(a: T[], b: T[], key: KeySelector<T>) {- return this.process(a, b, key, (freq) => freq < 3);+ difference<T>(a: T[], b: T[], key?: KeySelector<T>) {+ return <T[]>this.process(a, b, key, (freq) => freq < 3);}// Relative complement. Items from 'a' which are// not also in 'b'.// Set.complement([1, 2, 2], [2, 2, 3]) => [3]- complement<T>(a: T[], b: T[], key: KeySelector<T>) {+ complement<T>(a: T[], b: T[], key?: KeySelector<T>) {return this.process(a, b, key, (freq) => freq === 1);}diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.tsnew file mode 100644index 000000000..c748c253d--- /dev/null+++ b/packages/core/src/utils/virtualized-grouping.ts@@ -0,0 +1,111 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++export type VirtualizedGroupHeader = {+ title: string;+ id: string;+};+export class VirtualizedGrouping<T> {+ private cache: Map<number, Record<string, T>> = new Map();+ private groups: Map<number, VirtualizedGroupHeader[]> = new Map();++ constructor(+ private ids: string[],+ private readonly batchSize: number,+ private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,+ private readonly groupItems: (+ ids: string[],+ items: Record<string, T>+ ) => VirtualizedGroupHeader[] = () => []+ ) {+ this.ids = ids;+ }++ /**+ * Get item from cache or request the appropriate batch for caching+ * and load it from there.+ */+ async item(id: string) {+ const index = this.ids.indexOf(id);+ if (index <= -1) return;++ const batchIndex = Math.floor(index / this.batchSize);+ const batch = this.cache.get(batchIndex) || (await this.load(batchIndex));+ const groups = this.groups.get(batchIndex);++ const group = groups?.find((g) => g.id === id);+ if (group)+ return {+ group: { type: "header", id: group.title, title: group.title },+ item: batch[id]+ };+ return { item: batch[id] };+ }++ /**+ * Reload the cache+ */+ refresh(ids: string[]) {+ this.ids = ids;+ this.cache.clear();+ }++ /**+ *+ * @param index+ */+ private async load(batch: number) {+ const start = batch * this.batchSize;+ const end = start + this.batchSize;+ const batchIds = this.ids.slice(start, end);+ const items = await this.fetchItems(batchIds);+ const groups = this.groupItems(batchIds, items);++ const lastBatchIndex = this.last;+ const prevGroups = this.groups.get(lastBatchIndex);+ if (prevGroups && prevGroups.length > 0 && groups.length > 0) {+ const lastGroup = prevGroups[prevGroups.length - 1];+ if (lastGroup.title === groups[0].title) {+ // if user is moving downwards, we remove the last group from the+ // current batch, otherwise we remove the first group from the previous+ // batch.+ lastBatchIndex < batch ? groups.pop() : prevGroups.shift();+ }+ }++ this.cache.set(batch, items);+ this.groups.set(batch, groups);+ this.clear();+ return items;+ }++ private clear() {+ if (this.cache.size <= 2) return;+ for (const [key] of this.cache) {+ this.cache.delete(key);+ this.groups.delete(key);+ if (this.cache.size === 2) break;+ }+ }++ private get last() {+ const keys = Array.from(this.cache.keys());+ return keys[keys.length - 1];+ }+}