Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: fix many issues with virtualized grouping

Abdullah Atta57311397e014
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
index e87407773..3270fadd8 100644
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -375,23 +375,13 @@ export class FilteredSelector<T extends Item> {
async grouped(options: GroupOptions) {
const count = await this.count();
- const sortFields = this.sortFields(options, true);
- const cursorRowValue = sql.join(sortFields.map((f) => sql.ref(f)));
return new VirtualizedGrouping<T>(
count,
this.batchSize,
- async (start, end, cursor) => {
+ async (start, end) => {
const items = (await this.filter
.$call(this.buildSortExpression(options))
- .$if(!cursor, (qb) => qb.offset(start))
- .$if(!!cursor, (qb) =>
- qb.where(
- (eb) => eb.parens(cursorRowValue),
- ">",
- (eb) =>
- eb.parens(sql.join(sortFields.map((f) => (cursor as any)[f])))
- )
- )
+ .offset(start)
.limit(end - start)
.selectAll()
.execute()) as T[];
@@ -424,12 +414,14 @@ export class FilteredSelector<T extends Item> {
"reminders.snoozeUntil"
);
}
- return groupArray(
- await this.filter
- .$call(this.buildSortExpression(options))
- .select(fields)
- .execute(),
- options
+ return Array.from(
+ groupArray(
+ await this.filter
+ .$call(this.buildSortExpression(options))
+ .select(fields)
+ .execute(),
+ options
+ ).values()
);
}
@@ -489,8 +481,19 @@ export class FilteredSelector<T extends Item> {
.$if(this.type === "notes" || this.type === "notebooks", (eb) =>
eb.orderBy("pinned desc")
)
- .orderBy(options.sortBy, options.sortDirection)
- .$if(!!persistent, (eb) => eb.orderBy("id"));
+ .$if(options.sortBy === "title", (eb) =>
+ eb.orderBy(
+ sql`ltrim(${sql.raw(
+ options.sortBy
+ )}, ' \u00a0\r\n\t\v') COLLATE NOCASE ${sql.raw(
+ options.sortDirection
+ )}`
+ )
+ )
+ .$if(options.sortBy !== "title", (eb) =>
+ eb.orderBy(options.sortBy, options.sortDirection)
+ )
+ .$if(!!persistent, (eb) => eb.orderBy("id asc"));
};
}
diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.ts
index e118b8223..b0709211b 100644
--- a/packages/core/src/utils/grouping.ts
+++ b/packages/core/src/utils/grouping.ts
@@ -61,7 +61,7 @@ function getKeySelector(
const date = new Date();
if (item.type === "reminder")
return isReminderActive(item as Reminder) ? "Active" : "Inactive";
- else if (options.sortBy === "title")
+ else if (options.sortBy === "title" && options.groupBy === "abc")
return getFirstCharacter(getTitle(item));
else {
const value = getSortValue(options, item) || 0;
@@ -96,43 +96,29 @@ export function groupArray(
sortBy: "dateEdited",
sortDirection: "desc"
}
-): { index: number; group: GroupHeader }[] {
- const groups = new Map<string, number>();
- // [
- // ["Conflicted", 0],
- // ["Pinned", 1]
- // ]
+): Map<number, { index: number; group: GroupHeader }> {
+ const groups = new Map<
+ string,
+ [number, { index: number; group: GroupHeader }]
+ >();
const keySelector = getKeySelector(options);
for (let i = 0; i < items.length; ++i) {
const item = items[i];
const groupTitle = keySelector(item);
const group = groups.get(groupTitle);
- if (typeof group === "undefined") groups.set(groupTitle, i);
+ if (typeof group === "undefined")
+ groups.set(groupTitle, [
+ i,
+ {
+ index: i,
+ group: { id: groupTitle, title: groupTitle, type: "header" }
+ }
+ ]);
}
- const groupIndices: { index: number; group: GroupHeader }[] = [];
- groups.forEach((index, title) =>
- groupIndices.push({ index, group: { id: title, title, type: "header" } })
- );
- return groupIndices;
- // return flattenGroups(groups);
+ return new Map(groups.values());
}
-// function flattenGroups<T extends GroupableItem>(groups: Map<string, T[]>) {
-// const items: GroupedItems<T> = [];
-// groups.forEach((groupItems, groupTitle) => {
-// if (groupItems.length <= 0) return;
-// items.push({
-// title: groupTitle,
-// id: groupTitle.toLowerCase(),
-// type: "header"
-// });
-// items.push(...groupItems);
-// });
-
-// return items;
-// }
-
function getFirstCharacter(str: string) {
if (!str) return "-";
str = str.trim();
diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts
index d28733df1..d81f1e6f0 100644
--- a/packages/core/src/utils/virtualized-grouping.ts
+++ b/packages/core/src/utils/virtualized-grouping.ts
@@ -22,7 +22,7 @@ import { GroupHeader } from "../types";
type BatchOperator<T> = (ids: string[], items: T[]) => Promise<unknown[]>;
type Batch<T> = {
items: T[];
- groups?: { index: number; hidden?: boolean; group: GroupHeader }[];
+ groups?: Map<number, { index: number; hidden?: boolean; group: GroupHeader }>;
data?: unknown[];
};
export class VirtualizedGrouping<T> {
@@ -30,28 +30,38 @@ export class VirtualizedGrouping<T> {
private pending: Map<number, Promise<Batch<T>>> = new Map();
public ids: number[];
private loadBatchTimeout?: number;
- private cacheHits = 0;
constructor(
count: number,
private readonly batchSize: number,
private readonly fetchItems: (
start: number,
- end: number,
- cursor?: T
+ end: number
) => Promise<{ ids: string[]; items: T[] }>,
private readonly groupItems?: (
items: T[]
- ) => { index: number; hidden?: boolean; group: GroupHeader }[],
+ ) => Map<number, { index: number; hidden?: boolean; group: GroupHeader }>,
readonly groups?: () => Promise<{ index: number; group: GroupHeader }[]>
) {
this.ids = new Array(count).fill(0);
}
- getKey(index: number) {
+ key(index: number) {
return `${index}`;
}
+ type(index: number) {
+ const batchIndex = Math.floor(index / this.batchSize);
+ const batch = this.cache.get(batchIndex);
+ if (!batch) return "item";
+ const { items, groups } = batch;
+ const itemIndexInBatch = index - batchIndex * this.batchSize;
+ const group = groups?.get(itemIndexInBatch);
+ return group && !group.hidden && items[itemIndexInBatch]
+ ? "header-item"
+ : "item";
+ }
+
item(index: number): Promise<{ item: T; group?: GroupHeader }>;
item(
index: number,
@@ -59,17 +69,15 @@ export class VirtualizedGrouping<T> {
): Promise<{ item: T; group?: GroupHeader; data: unknown }>;
async item(index: number, operate?: BatchOperator<T>) {
const batchIndex = Math.floor(index / this.batchSize);
- if (this.cache.has(batchIndex)) this.cacheHits++;
const { items, groups, data } =
- this.cache.get(batchIndex) || (await this.loadBatch(batchIndex, operate));
+ this.cache.get(batchIndex) ||
+ (await this.batchLoader(batchIndex, operate));
const itemIndexInBatch = index - batchIndex * this.batchSize;
- const group = groups?.find(
- (f) => f.index === itemIndexInBatch && !f.hidden
- );
+ const group = groups?.get(itemIndexInBatch);
return {
item: items[itemIndexInBatch],
- group: group?.group,
+ group: group && !group.hidden ? group.group : undefined,
data: data?.[itemIndexInBatch]
};
}
@@ -90,36 +98,34 @@ export class VirtualizedGrouping<T> {
batchIndex: number,
operate?: BatchOperator<T>
): Promise<Batch<T>> {
- const lastBatchIndex = this.last;
- const prev = this.cache.get(lastBatchIndex);
+ const [lastBatchIndex, lastBatch] = lastInMap(this.cache) || [];
const start = batchIndex * this.batchSize;
const end = start + this.batchSize;
- // we can use a cursor instead of start/end offsets for batches that are
- // right next to each other.
- const cursor =
- lastBatchIndex + 1 === batchIndex
- ? prev?.items.at(-1)
- : lastBatchIndex - 1 === batchIndex
- ? prev?.items[0]
- : undefined;
- const { ids, items } = await this.fetchItems(start, end, cursor);
+
+ const { ids, items } = await this.fetchItems(start, end);
const groups = this.groupItems?.(items);
if (
- prev &&
- prev.groups &&
- prev.groups.length > 0 &&
+ lastBatch &&
+ lastBatch.groups &&
+ lastBatch.groups.size > 0 &&
groups &&
- groups.length > 0
+ groups.size > 0 &&
+ lastBatchIndex !== undefined
) {
+ const [, firstGroup] = firstInMap(groups);
// if user is moving downwards, we hide the first group from the
// current batch, otherwise we hide the last group from the previous
// batch.
const group =
lastBatchIndex < batchIndex
- ? groups[0] //groups.length - 1]
- : prev.groups[prev.groups.length - 1];
- if (group.group.title === groups[0].group.title) {
+ ? firstGroup
+ : lastInMap(lastBatch.groups)[1];
+
+ // if the last group of the previous batch has the same title as the
+ // first group of the current batch, we hide the current group otherwise
+ // we will be seeing 2 group headers with the same title.
+ if (group && firstGroup && group.group.title === firstGroup.group.title) {
group.hidden = true;
}
}
@@ -134,7 +140,7 @@ export class VirtualizedGrouping<T> {
return batch;
}
- private loadBatch(batch: number, operate?: BatchOperator<T>) {
+ private batchLoader(batch: number, operate?: BatchOperator<T>) {
if (this.pending.has(batch)) return this.pending.get(batch)!;
const promise = this.load(batch, operate);
this.pending.set(batch, promise);
@@ -150,13 +156,20 @@ export class VirtualizedGrouping<T> {
if (this.cache.size === 2) break;
}
}
+}
- private get last() {
- const keys = Array.from(this.cache.keys());
- return keys[keys.length - 1];
+function lastInMap<K, V>(map: Map<K, V>) {
+ let i = 0;
+ for (const item of map) {
+ if (++i === map.size) return item;
}
+ return [undefined, undefined];
+}
- private isLastBatch(batch: number) {
- return Math.floor(this.ids.length / this.batchSize) === batch;
+function firstInMap<K, V>(map: Map<K, V>) {
+ let i = 0;
+ for (const item of map) {
+ if (++i === 1) return item;
}
+ return [undefined, undefined];
}