A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix many issues with virtualized grouping
Abdullah Atta57311397e014
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex e87407773..3270fadd8 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -375,23 +375,13 @@ export class FilteredSelector<T extends Item> {async grouped(options: GroupOptions) {const count = await this.count();- const sortFields = this.sortFields(options, true);- const cursorRowValue = sql.join(sortFields.map((f) => sql.ref(f)));return new VirtualizedGrouping<T>(count,this.batchSize,- async (start, end, cursor) => {+ async (start, end) => {const items = (await this.filter.$call(this.buildSortExpression(options))- .$if(!cursor, (qb) => qb.offset(start))- .$if(!!cursor, (qb) =>- qb.where(- (eb) => eb.parens(cursorRowValue),- ">",- (eb) =>- eb.parens(sql.join(sortFields.map((f) => (cursor as any)[f])))- )- )+ .offset(start).limit(end - start).selectAll().execute()) as T[];@@ -424,12 +414,14 @@ export class FilteredSelector<T extends Item> {"reminders.snoozeUntil");}- return groupArray(- await this.filter- .$call(this.buildSortExpression(options))- .select(fields)- .execute(),- options+ return Array.from(+ groupArray(+ await this.filter+ .$call(this.buildSortExpression(options))+ .select(fields)+ .execute(),+ options+ ).values());}@@ -489,8 +481,19 @@ export class FilteredSelector<T extends Item> {.$if(this.type === "notes" || this.type === "notebooks", (eb) =>eb.orderBy("pinned desc"))- .orderBy(options.sortBy, options.sortDirection)- .$if(!!persistent, (eb) => eb.orderBy("id"));+ .$if(options.sortBy === "title", (eb) =>+ eb.orderBy(+ sql`ltrim(${sql.raw(+ options.sortBy+ )}, ' \u00a0\r\n\t\v') COLLATE NOCASE ${sql.raw(+ options.sortDirection+ )}`+ )+ )+ .$if(options.sortBy !== "title", (eb) =>+ eb.orderBy(options.sortBy, options.sortDirection)+ )+ .$if(!!persistent, (eb) => eb.orderBy("id asc"));};}diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.tsindex e118b8223..b0709211b 100644--- a/packages/core/src/utils/grouping.ts+++ b/packages/core/src/utils/grouping.ts@@ -61,7 +61,7 @@ function getKeySelector(const date = new Date();if (item.type === "reminder")return isReminderActive(item as Reminder) ? "Active" : "Inactive";- else if (options.sortBy === "title")+ else if (options.sortBy === "title" && options.groupBy === "abc")return getFirstCharacter(getTitle(item));else {const value = getSortValue(options, item) || 0;@@ -96,43 +96,29 @@ export function groupArray(sortBy: "dateEdited",sortDirection: "desc"}-): { index: number; group: GroupHeader }[] {- const groups = new Map<string, number>();- // [- // ["Conflicted", 0],- // ["Pinned", 1]- // ]+): Map<number, { index: number; group: GroupHeader }> {+ const groups = new Map<+ string,+ [number, { index: number; group: GroupHeader }]+ >();const keySelector = getKeySelector(options);for (let i = 0; i < items.length; ++i) {const item = items[i];const groupTitle = keySelector(item);const group = groups.get(groupTitle);- if (typeof group === "undefined") groups.set(groupTitle, i);+ if (typeof group === "undefined")+ groups.set(groupTitle, [+ i,+ {+ index: i,+ group: { id: groupTitle, title: groupTitle, type: "header" }+ }+ ]);}- const groupIndices: { index: number; group: GroupHeader }[] = [];- groups.forEach((index, title) =>- groupIndices.push({ index, group: { id: title, title, type: "header" } })- );- return groupIndices;- // return flattenGroups(groups);+ return new Map(groups.values());}-// function flattenGroups<T extends GroupableItem>(groups: Map<string, T[]>) {-// const items: GroupedItems<T> = [];-// groups.forEach((groupItems, groupTitle) => {-// if (groupItems.length <= 0) return;-// items.push({-// title: groupTitle,-// id: groupTitle.toLowerCase(),-// type: "header"-// });-// items.push(...groupItems);-// });--// return items;-// }-function getFirstCharacter(str: string) {if (!str) return "-";str = str.trim();diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.tsindex d28733df1..d81f1e6f0 100644--- a/packages/core/src/utils/virtualized-grouping.ts+++ b/packages/core/src/utils/virtualized-grouping.ts@@ -22,7 +22,7 @@ import { GroupHeader } from "../types";type BatchOperator<T> = (ids: string[], items: T[]) => Promise<unknown[]>;type Batch<T> = {items: T[];- groups?: { index: number; hidden?: boolean; group: GroupHeader }[];+ groups?: Map<number, { index: number; hidden?: boolean; group: GroupHeader }>;data?: unknown[];};export class VirtualizedGrouping<T> {@@ -30,28 +30,38 @@ export class VirtualizedGrouping<T> {private pending: Map<number, Promise<Batch<T>>> = new Map();public ids: number[];private loadBatchTimeout?: number;- private cacheHits = 0;constructor(count: number,private readonly batchSize: number,private readonly fetchItems: (start: number,- end: number,- cursor?: T+ end: number) => Promise<{ ids: string[]; items: T[] }>,private readonly groupItems?: (items: T[]- ) => { index: number; hidden?: boolean; group: GroupHeader }[],+ ) => Map<number, { index: number; hidden?: boolean; group: GroupHeader }>,readonly groups?: () => Promise<{ index: number; group: GroupHeader }[]>) {this.ids = new Array(count).fill(0);}- getKey(index: number) {+ key(index: number) {return `${index}`;}+ type(index: number) {+ const batchIndex = Math.floor(index / this.batchSize);+ const batch = this.cache.get(batchIndex);+ if (!batch) return "item";+ const { items, groups } = batch;+ const itemIndexInBatch = index - batchIndex * this.batchSize;+ const group = groups?.get(itemIndexInBatch);+ return group && !group.hidden && items[itemIndexInBatch]+ ? "header-item"+ : "item";+ }+item(index: number): Promise<{ item: T; group?: GroupHeader }>;item(index: number,@@ -59,17 +69,15 @@ export class VirtualizedGrouping<T> {): Promise<{ item: T; group?: GroupHeader; data: unknown }>;async item(index: number, operate?: BatchOperator<T>) {const batchIndex = Math.floor(index / this.batchSize);- if (this.cache.has(batchIndex)) this.cacheHits++;const { items, groups, data } =- this.cache.get(batchIndex) || (await this.loadBatch(batchIndex, operate));+ this.cache.get(batchIndex) ||+ (await this.batchLoader(batchIndex, operate));const itemIndexInBatch = index - batchIndex * this.batchSize;- const group = groups?.find(- (f) => f.index === itemIndexInBatch && !f.hidden- );+ const group = groups?.get(itemIndexInBatch);return {item: items[itemIndexInBatch],- group: group?.group,+ group: group && !group.hidden ? group.group : undefined,data: data?.[itemIndexInBatch]};}@@ -90,36 +98,34 @@ export class VirtualizedGrouping<T> {batchIndex: number,operate?: BatchOperator<T>): Promise<Batch<T>> {- const lastBatchIndex = this.last;- const prev = this.cache.get(lastBatchIndex);+ const [lastBatchIndex, lastBatch] = lastInMap(this.cache) || [];const start = batchIndex * this.batchSize;const end = start + this.batchSize;- // we can use a cursor instead of start/end offsets for batches that are- // right next to each other.- const cursor =- lastBatchIndex + 1 === batchIndex- ? prev?.items.at(-1)- : lastBatchIndex - 1 === batchIndex- ? prev?.items[0]- : undefined;- const { ids, items } = await this.fetchItems(start, end, cursor);++ const { ids, items } = await this.fetchItems(start, end);const groups = this.groupItems?.(items);if (- prev &&- prev.groups &&- prev.groups.length > 0 &&+ lastBatch &&+ lastBatch.groups &&+ lastBatch.groups.size > 0 &&groups &&- groups.length > 0+ groups.size > 0 &&+ lastBatchIndex !== undefined) {+ const [, firstGroup] = firstInMap(groups);// if user is moving downwards, we hide the first group from the// current batch, otherwise we hide the last group from the previous// batch.const group =lastBatchIndex < batchIndex- ? groups[0] //groups.length - 1]- : prev.groups[prev.groups.length - 1];- if (group.group.title === groups[0].group.title) {+ ? firstGroup+ : lastInMap(lastBatch.groups)[1];++ // if the last group of the previous batch has the same title as the+ // first group of the current batch, we hide the current group otherwise+ // we will be seeing 2 group headers with the same title.+ if (group && firstGroup && group.group.title === firstGroup.group.title) {group.hidden = true;}}@@ -134,7 +140,7 @@ export class VirtualizedGrouping<T> {return batch;}- private loadBatch(batch: number, operate?: BatchOperator<T>) {+ private batchLoader(batch: number, operate?: BatchOperator<T>) {if (this.pending.has(batch)) return this.pending.get(batch)!;const promise = this.load(batch, operate);this.pending.set(batch, promise);@@ -150,13 +156,20 @@ export class VirtualizedGrouping<T> {if (this.cache.size === 2) break;}}+}- private get last() {- const keys = Array.from(this.cache.keys());- return keys[keys.length - 1];+function lastInMap<K, V>(map: Map<K, V>) {+ let i = 0;+ for (const item of map) {+ if (++i === map.size) return item;}+ return [undefined, undefined];+}- private isLastBatch(batch: number) {- return Math.floor(this.ids.length / this.batchSize) === batch;+function firstInMap<K, V>(map: Map<K, V>) {+ let i = 0;+ for (const item of map) {+ if (++i === 1) return item;}+ return [undefined, undefined];}