A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: add tests for virtualized grouping
Abdullah Atta0e4cd087d407
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex 3b3dc0040..bd2a8f8e8 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -19,7 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import dayjs from "dayjs";import Database from "../src/api";-import { groupArray } from "../src/utils/grouping";+import { createKeySelector, groupArray } from "../src/utils/grouping";import {databaseTest,noteTest,@@ -346,11 +346,14 @@ test("grouping items where item.title is empty or undefined shouldn't throw", ()expect(// eslint-disable-next-line @typescript-eslint/ban-ts-comment// @ts-ignore- groupArray([{ title: "" }], {- groupBy: "abc",- sortBy: "title",- sortDirection: "asc"- })+ groupArray(+ [{ title: "" }],+ createKeySelector({+ groupBy: "abc",+ sortBy: "title",+ sortDirection: "asc"+ })+ )).toBeTruthy();});diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.tsindex 584cea888..b62e8d937 100644--- a/packages/core/src/collections/trash.ts+++ b/packages/core/src/collections/trash.ts@@ -22,7 +22,11 @@ import Database from "../api";import { deleteItems, toChunks } from "../utils/array";import { GroupOptions, TrashItem } from "../types";import { VirtualizedGrouping } from "../utils/virtualized-grouping";-import { getSortSelectors, groupArray } from "../utils/grouping";+import {+ createKeySelector,+ getSortSelectors,+ groupArray+} from "../utils/grouping";import { sql } from "kysely";import { MAX_SQL_PARAMETERS } from "../database/sql-collection";@@ -282,11 +286,13 @@ export default class Trash {items};},- (items) => groupArray(items, options),+ (items) => groupArray(items, createKeySelector(options)),async () => {const items = await this.all();items.sort(selector);- return Array.from(groupArray(items, options).values());+ return Array.from(+ groupArray(items, createKeySelector(options)).values()+ );});}diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex cd4c63537..88df76639 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -47,7 +47,7 @@ import {sql} from "kysely";import { VirtualizedGrouping } from "../utils/virtualized-grouping";-import { groupArray } from "../utils/grouping";+import { createKeySelector, groupArray } from "../utils/grouping";import { toChunks } from "../utils/array";import { Sanitizer } from "./sanitizer";import {@@ -450,7 +450,7 @@ export class FilteredSelector<T extends Item> {items};},- (items) => groupArray(items as any, options),+ (items) => groupArray(items as any, createKeySelector(options)),() => this.groups(options));}@@ -486,7 +486,7 @@ export class FilteredSelector<T extends Item> {.select(fields).$call(this.buildSortExpression(options, true)).execute(),- options+ createKeySelector(options)).values());}diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.tsindex 1fc943f4a..ca0e7416a 100644--- a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts+++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts@@ -17,110 +17,108 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { test, vi } from "vitest";+import { test } from "vitest";import { VirtualizedGrouping } from "../virtualized-grouping";--function item<T>(value: T) {- return { item: value };+import { groupArray } from "../grouping";++function generateItems(length: number, groupSize: number) {+ const items: { group: string; id: string }[] = [];+ const ids: string[] = [];+ const divider = length / groupSize;+ for (let i = 0; i < length; ++i) {+ items.push({ group: `${i % divider}`, id: `${i}` });+ ids.push(`${i}`);+ }+ items.sort((a, b) => a.group.localeCompare(b.group));+ return { items, ids };}-function createMock() {- return vi.fn(async (ids: string[]) =>- Object.fromEntries(ids.map((id) => [id, id]))++function createVirtualizedGrouping(+ length: number,+ groupSize: number,+ batchSize: number+) {+ const { ids, items } = generateItems(length, groupSize);+ return new VirtualizedGrouping<{ group: string; id: string }>(+ items.length,+ batchSize,+ () => Promise.resolve(ids),+ async (start, end) => ({+ ids: ids.slice(start, end),+ items: items.slice(start, end)+ }),+ (items) => groupArray(items, (item) => item.group));}-test.todo("renable virtualized grouping tests");-// test("fetch items in batch if not found in cache", async (t) => {-// const mocked = createMock();-// const grouping = new VirtualizedGrouping<string>(-// ["1", "2", "3", "4", "5", "6", "7"],-// 3,-// mocked-// );-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(mocked).toHaveBeenCalledOnce();-// });--// test("do not fetch items in batch if found in cache", async (t) => {-// const mocked = createMock();-// const grouping = new VirtualizedGrouping<string>(-// ["1", "2", "3", "4", "5", "6", "7"],-// 3,-// mocked-// );-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(mocked).toHaveBeenCalledOnce();-// });--// test("clear old cached batches", async (t) => {-// const mocked = createMock();-// const grouping = new VirtualizedGrouping<string>(-// ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],-// 3,-// mocked-// );-// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);-// t.expect(await grouping.item("4")).toStrictEqual(item("4"));-// t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);-// t.expect(await grouping.item("7")).toStrictEqual(item("7"));-// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);-// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);-// });--// test("clear old cached batches (random access)", async (t) => {-// const mocked = createMock();-// const grouping = new VirtualizedGrouping<string>(-// ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],-// 3,-// mocked-// );-// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);--// t.expect(await grouping.item("7")).toStrictEqual(item("7"));-// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);--// t.expect(await grouping.item("11")).toStrictEqual(item("11"));-// t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);--// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);--// t.expect(await grouping.item("7")).toStrictEqual(item("7"));-// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);-// });--// test("reloading ids should clear all cached batches", async (t) => {-// const mocked = createMock();-// const grouping = new VirtualizedGrouping<string>(-// ["1", "3", "4", "5", "7", "6", "50"],-// 3,-// mocked-// );--// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);--// grouping.refresh([-// "1",-// "2",-// "3",-// "4",-// "5",-// "6",-// "7",-// "8",-// "9",-// "10",-// "11",-// "12"-// ]);--// t.expect(await grouping.item("1")).toStrictEqual(item("1"));-// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);-// });++test("load first batch with a single group", async (t) => {+ const grouping = createVirtualizedGrouping(100, 10, 10);++ t.expect((await grouping.item(0)).group?.title).toBe("0");+ for (let i = 1; i < 10; ++i)+ t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined();+});++test("load first batch with a multiple groups", async (t) => {+ const grouping = createVirtualizedGrouping(100, 2, 10);++ t.expect((await grouping.item(0)).group?.title).toBe(`0`);+ t.expect(grouping.cacheItem(2)?.group?.title).toBe(`1`);+ t.expect(grouping.cacheItem(4)?.group?.title).toBe(`10`);+ t.expect(grouping.cacheItem(6)?.group?.title).toBe(`11`);+ t.expect(grouping.cacheItem(8)?.group?.title).toBe(`12`);+});++test("load last batch with a single group", async (t) => {+ const grouping = createVirtualizedGrouping(100, 10, 10);++ t.expect((await grouping.item(90)).group?.title).toBe("9");+ for (let i = 91; i < 100; ++i)+ t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined();+});++test("load last batch with a multiple groups", async (t) => {+ const grouping = createVirtualizedGrouping(100, 2, 10);++ t.expect((await grouping.item(90)).group?.title).toBe(`5`);+ t.expect(grouping.cacheItem(92)?.group?.title).toBe(`6`);+ t.expect(grouping.cacheItem(94)?.group?.title).toBe(`7`);+ t.expect(grouping.cacheItem(96)?.group?.title).toBe(`8`);+ t.expect(grouping.cacheItem(98)?.group?.title).toBe(`9`);+});++test("group spanning multiple batches (down)", async (t) => {+ const grouping = createVirtualizedGrouping(140, 14, 10);++ t.expect((await grouping.item(0)).group?.title).toBe(`0`);+ t.expect((await grouping.item(12)).group).toBeUndefined();+ t.expect((await grouping.item(14)).group?.title).toBe("1");+ t.expect((await grouping.item(24)).group).toBeUndefined();+ t.expect((await grouping.item(28)).group?.title).toBe("2");+});++test("single group in all batches", async (t) => {+ const grouping = createVirtualizedGrouping(100, 100, 10);++ t.expect((await grouping.item(0)).group?.title).toBe(`0`);+ for (let i = 1; i < 100; ++i) {+ t.expect((await grouping.item(i)).group).toBeUndefined();+ }+});++test("group at start of each batch", async (t) => {+ const grouping = createVirtualizedGrouping(100, 10, 10);++ for (let i = 0; i < 100; i += 10) {+ t.expect((await grouping.item(i)).group?.title).toBe(`${i / 10}`);+ }+});++test("group spanning multiple batches (up)", async (t) => {+ const grouping = createVirtualizedGrouping(140, 28, 10);++ t.expect((await grouping.item(130)).group).toBeUndefined();+ t.expect((await grouping.item(120)).group).toBeUndefined();+ t.expect((await grouping.item(140 - 28)).group).toBeDefined();+ t.expect((await grouping.item(110)).group).toBeUndefined();+});diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.tsindex 8a9659598..ce75608f5 100644--- a/packages/core/src/utils/grouping.ts+++ b/packages/core/src/utils/grouping.ts@@ -30,7 +30,7 @@ type PartialGroupableItem = {dateEdited?: number | null;dateCreated?: number | null;};-type EvaluateKeyFunction<T> = (item: T) => string;+export type GroupKeySelectorFunction<T> = (item: T) => string;export const getSortValue = (options: GroupOptions | undefined,@@ -72,9 +72,13 @@ export function getSortSelectors<T extends PartialGroupableItem>(const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;-function getKeySelector(- options: GroupOptions-): EvaluateKeyFunction<PartialGroupableItem> {+export function createKeySelector(+ options: GroupOptions = {+ groupBy: "default",+ sortBy: "dateEdited",+ sortDirection: "desc"+ }+): GroupKeySelectorFunction<PartialGroupableItem> {return (item) => {if ("pinned" in item && item.pinned) return "Pinned";else if ("conflicted" in item && item.conflicted) return "Conflicted";@@ -110,20 +114,15 @@ function getKeySelector(};}-export function groupArray(- items: PartialGroupableItem[],- options: GroupOptions = {- groupBy: "default",- sortBy: "dateEdited",- sortDirection: "desc"- }+export function groupArray<T>(+ items: T[],+ keySelector: GroupKeySelectorFunction<T>): Map<number, { index: number; group: GroupHeader }> {const groups = new Map<string,[number, { index: number; group: GroupHeader }]>();- const keySelector = getKeySelector(options);for (let i = 0; i < items.length; ++i) {const item = items[i];const groupTitle = keySelector(item);diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.tsindex 61a97b438..9ff1311ce 100644--- a/packages/core/src/utils/virtualized-grouping.ts+++ b/packages/core/src/utils/virtualized-grouping.ts@@ -123,6 +123,9 @@ export class VirtualizedGrouping<T> {const { ids, items } = await this.fetchItems(start, end);const groups = this.groupItems?.(items);+ if (items.length > this.batchSize)+ throw new Error("Got more items than the batch size.");+if (direction === "down") {const [, firstGroup] = groups ? firstInMap(groups) : [];const group = lastBatch?.groups