Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: changes according to client needs

Abdullah Atta9ac9e21645c2
diff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.ts
index cece9f50e..c4ae0bf05 100644
--- a/packages/core/__tests__/utils/index.ts
+++ b/packages/core/__tests__/utils/index.ts
@@ -28,7 +28,7 @@ import { EventSourcePolyfill as EventSource } from "event-source-polyfill";
import { randomBytes } from "../../src/utils/random";
import { GroupOptions, Note, Notebook } from "../../src/types";
import { NoteContent } from "../../src/collections/session-content";
-import { SqliteDriver } from "kysely";
+import { SqliteDialect } from "kysely";
import BetterSQLite3 from "better-sqlite3";
const TEST_NOTEBOOK: Partial<Notebook> = {
@@ -48,7 +48,7 @@ function databaseTest() {
eventsource: EventSource,
fs: FS,
compressor: Compressor,
- sqlite: new SqliteDriver({ database: BetterSQLite3(":memory:") })
+ dialect: new SqliteDialect({ database: BetterSQLite3(":memory:") })
});
return db.init().then(() => db);
}
diff --git a/packages/core/package.json b/packages/core/package.json
index 8be5b89de..342248570 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,7 +1,7 @@
{
"name": "@notesnook/core",
"version": "7.4.1",
- "main": "dist/api/index.js",
+ "main": "dist/index.js",
"license": "GPL-3.0-or-later",
"repository": {
"type": "git",
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts
index d9029b815..49e17ee5c 100644
--- a/packages/core/src/api/index.ts
+++ b/packages/core/src/api/index.ts
@@ -60,8 +60,13 @@ import {
import TokenManager from "./token-manager";
import { Attachment } from "../types";
import { Settings } from "../collections/settings";
-import { DatabaseAccessor, DatabaseSchema, createDatabase } from "../database";
-import { Kysely, SqliteDriver, Transaction } from "kysely";
+import {
+ DatabaseAccessor,
+ DatabaseSchema,
+ SQLiteOptions,
+ createDatabase
+} from "../database";
+import { Kysely, Transaction } from "kysely";
import { CachedCollection } from "../database/cached-collection";
type EventSourceConstructor = new (
@@ -69,11 +74,12 @@ type EventSourceConstructor = new (
init: EventSourceInit & { headers?: Record<string, string> }
) => EventSource;
type Options = {
- sqlite: SqliteDriver;
+ sqliteOptions: SQLiteOptions;
storage: IStorage;
eventsource?: EventSourceConstructor;
fs: IFileStorage;
compressor: ICompressor;
+ batchSize: number;
};
// const DIFFERENCE_THRESHOLD = 20 * 1000;
@@ -131,6 +137,7 @@ class Database {
transaction = (
executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>
) => {
+ console.time("transaction");
return this.transactionMutex.runExclusive(() =>
this.sql()
.transaction()
@@ -139,11 +146,14 @@ class Database {
await executor(tr);
this._transaction = undefined;
})
- .finally(() => (this._transaction = undefined))
+ .finally(() => {
+ console.timeEnd("transaction");
+ this._transaction = undefined;
+ })
);
};
- private options?: Options;
+ options?: Options;
EventSource?: EventSourceConstructor;
eventSource?: EventSource | null;
@@ -160,7 +170,6 @@ class Database {
vault = new Vault(this);
lookup = new Lookup(this);
backup = new Backup(this);
- legacySettings = new LegacySettings(this);
settings = new Settings(this);
migrations = new Migrations(this);
monographs = new Monographs(this);
@@ -193,7 +202,10 @@ class Database {
* @deprecated only kept here for migration purposes
*/
legacyNotes = new CachedCollection(this.storage, "notes", this.eventManager);
-
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ legacySettings = new LegacySettings(this);
// constructor() {
// this.sseMutex = new Mutex();
// // this.lastHeartbeat = undefined; // { local: 0, server: 0 };
@@ -229,7 +241,8 @@ class Database {
this.disconnectSSE();
});
- if (this.options) this._sql = await createDatabase(this.options.sqlite);
+ if (this.options)
+ this._sql = await createDatabase(this.options.sqliteOptions);
await this._validate();
diff --git a/packages/core/src/api/monographs.ts b/packages/core/src/api/monographs.ts
index 988591639..883a807da 100644
--- a/packages/core/src/api/monographs.ts
+++ b/packages/core/src/api/monographs.ts
@@ -20,9 +20,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import http from "../utils/http";
import Constants from "../utils/constants";
import Database from ".";
-import { isDeleted } from "../types";
+import { Note, isDeleted } from "../types";
import { isUnencryptedContent } from "../collections/content";
import { Cipher } from "@notesnook/crypto";
+import { isFalse } from "../database";
type BaseMonograph = {
id: string;
@@ -163,9 +164,15 @@ export class Monographs {
this.monographs.splice(this.monographs.indexOf(noteId), 1);
}
- async all() {
- if (!this.monographs.length) return [];
- return await this.db.notes.all.items(this.monographs);
+ get all() {
+ return this.db.notes.collection.createFilter<Note>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("id", "in", this.monographs),
+ this.db.options?.batchSize
+ );
}
get(monographId: string) {
diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts
index ece0d4bb9..96cbc8686 100644
--- a/packages/core/src/api/sync/index.ts
+++ b/packages/core/src/api/sync/index.ts
@@ -380,7 +380,7 @@ class Sync {
const collectionType = SYNC_COLLECTIONS_MAP[itemType];
const collection = this.db[collectionType].collection;
- const localItems = await collection.items(chunk.items.map((i) => i.id));
+ const localItems = await collection.records(chunk.items.map((i) => i.id));
let items: (MaybeDeletedItem<Item> | undefined)[] = [];
if (itemType === "content") {
items = await Promise.all(
diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts
index 987ee9347..0ae995289 100644
--- a/packages/core/src/collections/attachments.ts
+++ b/packages/core/src/collections/attachments.ts
@@ -365,8 +365,9 @@ export class Attachments implements ICollection {
}
get pending() {
- return this.collection.createFilter<Attachment>((qb) =>
- qb.where(isFalse("dateUploaded"))
+ return this.collection.createFilter<Attachment>(
+ (qb) => qb.where(isFalse("dateUploaded")),
+ this.db.options?.batchSize
);
}
@@ -383,8 +384,9 @@ export class Attachments implements ICollection {
// }
get deleted() {
- return this.collection.createFilter<Attachment>((qb) =>
- qb.where("dateDeleted", "is not", null)
+ return this.collection.createFilter<Attachment>(
+ (qb) => qb.where("dateDeleted", "is not", null),
+ this.db.options?.batchSize
);
}
@@ -412,8 +414,9 @@ export class Attachments implements ICollection {
// }
get all() {
- return this.collection.createFilter<Attachment>((qb) =>
- qb.where(isFalse("deleted"))
+ return this.collection.createFilter<Attachment>(
+ (qb) => qb.where(isFalse("deleted")),
+ this.db.options?.batchSize
);
}
diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts
index 596ece26b..d32fee675 100644
--- a/packages/core/src/collections/colors.ts
+++ b/packages/core/src/collections/colors.ts
@@ -87,8 +87,9 @@ export class Colors implements ICollection {
// }
get all() {
- return this.collection.createFilter<Color>((qb) =>
- qb.where(isFalse("deleted"))
+ return this.collection.createFilter<Color>(
+ (qb) => qb.where(isFalse("deleted")),
+ this.db.options?.batchSize
);
}
diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts
index 4be3d9b97..977001d5c 100644
--- a/packages/core/src/collections/content.ts
+++ b/packages/core/src/collections/content.ts
@@ -33,6 +33,7 @@ import {
import Database from "../api";
import { getOutputType } from "./attachments";
import { SQLCollection } from "../database/sql-collection";
+import { NoteContent } from "./session-content";
export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
noteId,
@@ -73,49 +74,67 @@ export class Content implements ICollection {
);
const id = content.id || getId();
- const oldContent = content.id ? await this.get(content.id) : undefined;
- const noteId = oldContent?.noteId || content.noteId;
- if (!noteId) throw new Error("No noteId found to link the content to.");
-
- const encryptedData = isCipher(content.data)
- ? content.data
- : oldContent && isCipher(oldContent.data)
- ? oldContent.data
- : null;
-
- const unencryptedData =
- typeof content.data === "string"
- ? content.data
- : oldContent && typeof oldContent.data === "string"
- ? oldContent.data
- : "<p></p>";
-
- const contentItem: ContentItem = {
- type: "tiptap",
- noteId,
- id,
-
- dateEdited: content.dateEdited || oldContent?.dateEdited || Date.now(),
- dateCreated: content.dateCreated || oldContent?.dateCreated || Date.now(),
- dateModified: Date.now(),
- localOnly: content.localOnly || !!oldContent?.localOnly,
-
- conflicted: content.conflicted || oldContent?.conflicted,
- dateResolved: content.dateResolved || oldContent?.dateResolved,
-
- ...(encryptedData
- ? { locked: true, data: encryptedData }
- : { locked: false, data: unencryptedData })
- };
-
- await this.collection.upsert(
- contentItem.locked
- ? contentItem
- : await this.extractAttachments(contentItem)
- );
- if (content.sessionId)
- await this.db.noteHistory.add(content.sessionId, contentItem);
+ if (!content.noteId)
+ throw new Error("No noteId found to link the content to.");
+ if (!content.type) throw new Error("Please specify content's type.");
+
+ const encryptedData = isCipher(content.data) ? content.data : undefined;
+ let unencryptedData =
+ typeof content.data === "string" ? content.data : undefined;
+
+ if (unencryptedData)
+ unencryptedData = await this.extractAttachments({
+ type: content.type,
+ data: unencryptedData,
+ noteId: content.noteId
+ });
+
+ if (content.id && (await this.exists(content.id))) {
+ const contentData = encryptedData
+ ? { locked: true as const, data: encryptedData }
+ : unencryptedData
+ ? { locked: false as const, data: unencryptedData }
+ : undefined;
+
+ await this.collection.update([content.id], {
+ dateEdited: content.dateEdited,
+ localOnly: content.localOnly,
+ conflicted: content.conflicted,
+ dateResolved: content.dateResolved,
+ ...contentData
+ });
+
+ if (content.sessionId && contentData)
+ await this.db.noteHistory.add(content.sessionId, {
+ noteId: content.noteId,
+ type: content.type,
+ ...contentData
+ });
+ } else {
+ const contentItem: ContentItem = {
+ type: "tiptap",
+ noteId: content.noteId,
+ id,
+
+ dateEdited: content.dateEdited || Date.now(),
+ dateCreated: content.dateCreated || Date.now(),
+ dateModified: Date.now(),
+ localOnly: !!content.localOnly,
+
+ conflicted: content.conflicted,
+ dateResolved: content.dateResolved,
+
+ ...(encryptedData
+ ? { locked: true, data: encryptedData }
+ : { locked: false, data: unencryptedData || "<p></p>" })
+ };
+
+ await this.collection.upsert(contentItem);
+
+ if (content.sessionId)
+ await this.db.noteHistory.add(content.sessionId, contentItem);
+ }
return id;
}
@@ -254,18 +273,21 @@ export class Content implements ICollection {
await this.add(contentItem);
}
- async extractAttachments(contentItem: UnencryptedContentItem) {
- if (contentItem.localOnly) return contentItem;
+ async extractAttachments(
+ contentItem: NoteContent<false> & { noteId: string }
+ ) {
+ // if (contentItem.localOnly) return contentItem;
const content = getContentFromData(contentItem.type, contentItem.data);
- if (!content) return contentItem;
+ if (!content) return contentItem.data;
const { data, hashes } = await content.extractAttachments(
this.db.attachments.save
);
const noteAttachments = await this.db.relations
.from({ type: "note", id: contentItem.noteId }, "attachment")
- .resolve();
+ .selector.filter.select(["id", "hash"])
+ .execute();
const toDelete = noteAttachments.filter((attachment) => {
return hashes.every((hash) => hash !== attachment.hash);
@@ -281,11 +303,12 @@ export class Content implements ICollection {
id: contentItem.noteId,
type: "note"
},
- attachment
+ { id: attachment.id, type: "attachment" }
);
}
for (const hash of toAdd) {
+ // TODO: only get id instead of the whole object
const attachment = await this.db.attachments.attachment(hash);
if (!attachment) continue;
await this.db.relations.add(
@@ -297,11 +320,11 @@ export class Content implements ICollection {
);
}
- if (toAdd.length > 0) {
- contentItem.dateModified = Date.now();
- }
- contentItem.data = data;
- return contentItem;
+ // if (toAdd.length > 0) {
+ // contentItem.dateModified = Date.now();
+ // }
+ // contentItem.data = data;
+ return data;
}
// async cleanup() {
diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts
index 404b53f35..3c3b7cf10 100644
--- a/packages/core/src/collections/note-history.ts
+++ b/packages/core/src/collections/note-history.ts
@@ -20,10 +20,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import Database from "../api";
import { isCipher } from "../database/crypto";
import { SQLCollection } from "../database/sql-collection";
-import { ContentItem, HistorySession, isDeleted } from "../types";
+import { HistorySession, isDeleted } from "../types";
import { makeSessionContentId } from "../utils/id";
import { ICollection } from "./collection";
-import { SessionContent } from "./session-content";
+import { NoteContent, SessionContent } from "./session-content";
export class NoteHistory implements ICollection {
name = "notehistory";
@@ -60,39 +60,56 @@ export class NoteHistory implements ICollection {
return history as HistorySession[];
}
- async add(sessionId: string, content: ContentItem) {
+ async add(
+ sessionId: string,
+ content: NoteContent<boolean> & { noteId: string; locked: boolean }
+ ) {
const { noteId, locked } = content;
sessionId = `${noteId}_${sessionId}`;
- const oldSession = await this.collection.get(sessionId);
-
- if (oldSession && isDeleted(oldSession)) return;
-
- const session: HistorySession = {
- type: "session",
- id: sessionId,
- sessionContentId: makeSessionContentId(sessionId),
- noteId,
- dateCreated: oldSession ? oldSession.dateCreated : Date.now(),
- dateModified: Date.now(),
- localOnly: true,
- locked
- };
-
- await this.collection.upsert(session);
+
+ if (await this.collection.exists(sessionId)) {
+ await this.collection.update([sessionId], { locked });
+ } else {
+ await this.collection.upsert({
+ type: "session",
+ id: sessionId,
+ sessionContentId: makeSessionContentId(sessionId),
+ noteId,
+ dateCreated: Date.now(),
+ dateModified: Date.now(),
+ localOnly: true,
+ locked
+ });
+ }
await this.sessionContent.add(sessionId, content, locked);
await this.cleanup(noteId);
- return session;
+ return sessionId;
}
private async cleanup(noteId: string, limit = this.versionsLimit) {
- const history = await this.get(noteId, "asc");
- if (history.length === 0 || history.length < limit) return;
- const deleteCount = history.length - limit;
- for (let i = 0; i < deleteCount; i++) {
- const session = history[i];
+ const history = await this.db
+ .sql()
+ .selectFrom("notehistory")
+ .where("noteId", "==", noteId)
+ .orderBy(`dateModified asc`)
+ .select(["id", "sessionContentId"])
+ .offset(limit)
+ .limit(10)
+ .$narrowType<{ id: string; sessionContentId: string }>()
+ .execute();
+
+ for (const session of history) {
await this._remove(session);
}
+
+ // const history = await this.get(noteId, "asc");
+ // if (history.length === 0 || history.length < limit) return;
+ // const deleteCount = history.length - limit;
+ // for (let i = 0; i < deleteCount; i++) {
+ // const session = history[i];
+ // await this._remove(session);
+ // }
}
async content(sessionId: string) {
@@ -131,7 +148,7 @@ export class NoteHistory implements ICollection {
});
}
- private async _remove(session: HistorySession) {
+ private async _remove(session: { id: string; sessionContentId: string }) {
await this.collection.delete([session.id]);
await this.sessionContent.remove(session.sessionContentId);
}
diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts
index e464ac436..6a5f4d388 100644
--- a/packages/core/src/collections/notebooks.ts
+++ b/packages/core/src/collections/notebooks.ts
@@ -79,17 +79,20 @@ export class Notebooks implements ICollection {
// }
get all() {
- return this.collection.createFilter<Notebook>((qb) =>
- qb.where(isFalse("dateDeleted")).where(isFalse("deleted"))
+ return this.collection.createFilter<Notebook>(
+ (qb) => qb.where(isFalse("dateDeleted")).where(isFalse("deleted")),
+ this.db.options?.batchSize
);
}
get pinned() {
- return this.collection.createFilter<Notebook>((qb) =>
- qb
- .where(isFalse("dateDeleted"))
- .where(isFalse("deleted"))
- .where("pinned", "==", true)
+ return this.collection.createFilter<Notebook>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("pinned", "==", true),
+ this.db.options?.batchSize
);
}
diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts
index 0048f7746..bb7156aef 100644
--- a/packages/core/src/collections/notes.ts
+++ b/packages/core/src/collections/notes.ts
@@ -62,6 +62,7 @@ export class Notes implements ICollection {
throw new Error("Please use db.notes.merge to merge remote notes.");
const id = item.id || getId();
+
const oldNote = await this.note(id);
const note = {
@@ -74,59 +75,61 @@ export class Notes implements ICollection {
if (!oldNote && !item.content && !item.contentId && !item.title)
throw new Error("Note must have a title or content.");
- if (item.content && item.content.data && item.content.type) {
- const { type, data } = item.content;
-
- const content = getContentFromData(type, data);
- if (!content) throw new Error("Invalid content type.");
+ await this.db.transaction(async () => {
+ if (item.content && item.content.data && item.content.type) {
+ const { type, data } = item.content;
+
+ const content = getContentFromData(type, data);
+ if (!content) throw new Error("Invalid content type.");
+
+ note.contentId = await this.db.content.add({
+ noteId: id,
+ sessionId: note.sessionId,
+ id: note.contentId,
+ type,
+ data,
+ localOnly: !!note.localOnly
+ });
- note.contentId = await this.db.content.add({
- noteId: id,
- sessionId: note.sessionId,
- id: note.contentId,
- type,
- data,
- localOnly: !!note.localOnly
- });
+ note.headline = note.locked ? "" : getNoteHeadline(content);
+ if (oldNote) note.dateEdited = Date.now();
+ }
- note.headline = note.locked ? "" : getNoteHeadline(content);
- if (oldNote) note.dateEdited = Date.now();
- }
+ if (item.localOnly !== undefined) {
+ await this.db.content.add({
+ id: note.contentId,
+ localOnly: !!item.localOnly
+ });
+ }
- if (item.localOnly !== undefined) {
- await this.db.content.add({
- id: note.contentId,
- localOnly: !!item.localOnly
- });
- }
+ const noteTitle = await this.getNoteTitle(note, oldNote, note.headline);
+ if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();
- const noteTitle = await this.getNoteTitle(note, oldNote, note.headline);
- if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();
+ await this.collection.upsert({
+ id,
+ contentId: note.contentId,
+ type: "note",
- await this.collection.upsert({
- id,
- contentId: note.contentId,
- type: "note",
+ title: noteTitle,
+ headline: note.headline,
- title: noteTitle,
- headline: note.headline,
+ notebooks: note.notebooks || undefined,
- notebooks: note.notebooks || undefined,
+ pinned: !!note.pinned,
+ locked: !!note.locked,
+ favorite: !!note.favorite,
+ localOnly: !!note.localOnly,
+ conflicted: !!note.conflicted,
+ readonly: !!note.readonly,
- pinned: !!note.pinned,
- locked: !!note.locked,
- favorite: !!note.favorite,
- localOnly: !!note.localOnly,
- conflicted: !!note.conflicted,
- readonly: !!note.readonly,
+ dateCreated: note.dateCreated || Date.now(),
+ dateEdited:
+ item.dateEdited || note.dateEdited || note.dateCreated || Date.now(),
+ dateModified: note.dateModified || Date.now()
+ });
- dateCreated: note.dateCreated || Date.now(),
- dateEdited:
- item.dateEdited || note.dateEdited || note.dateCreated || Date.now(),
- dateModified: note.dateModified || Date.now()
+ if (!oldNote) this.totalNotes++;
});
-
- if (!oldNote) this.totalNotes++;
return id;
}
@@ -149,8 +152,9 @@ export class Notes implements ICollection {
// }
get all() {
- return this.collection.createFilter<Note>((qb) =>
- qb.where(isFalse("dateDeleted")).where(isFalse("deleted"))
+ return this.collection.createFilter<Note>(
+ (qb) => qb.where(isFalse("dateDeleted")).where(isFalse("deleted")),
+ this.db.options?.batchSize
);
}
@@ -165,38 +169,46 @@ export class Notes implements ICollection {
// }
get pinned() {
- return this.collection.createFilter<Note>((qb) =>
- qb
- .where(isFalse("dateDeleted"))
- .where(isFalse("deleted"))
- .where("pinned", "==", true)
+ return this.collection.createFilter<Note>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("pinned", "==", true),
+ this.db.options?.batchSize
);
}
get conflicted() {
- return this.collection.createFilter<Note>((qb) =>
- qb
- .where(isFalse("dateDeleted"))
- .where(isFalse("deleted"))
- .where("conflicted", "==", true)
+ return this.collection.createFilter<Note>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("conflicted", "==", true),
+ this.db.options?.batchSize
);
}
get favorites() {
- return this.collection.createFilter<Note>((qb) =>
- qb
- .where(isFalse("dateDeleted"))
- .where(isFalse("deleted"))
- .where("favorite", "==", true)
+ return this.collection.createFilter<Note>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("favorite", "==", true),
+ this.db.options?.batchSize
);
}
get locked() {
- return this.collection.createFilter<Note>((qb) =>
- qb
- .where(isFalse("dateDeleted"))
- .where(isFalse("deleted"))
- .where("locked", "==", true)
+ return this.collection.createFilter<Note>(
+ (qb) =>
+ qb
+ .where(isFalse("dateDeleted"))
+ .where(isFalse("deleted"))
+ .where("locked", "==", true),
+ this.db.options?.batchSize
);
}
@@ -299,21 +311,19 @@ export class Notes implements ICollection {
dateCreated: undefined,
dateModified: undefined
});
- if (!duplicateId) return;
+ if (!duplicateId) continue;
- for (const notebook of await this.db.relations
+ for (const relation of await this.db.relations
.to(note, "notebook")
.get()) {
await this.db.relations.add(
- { type: "notebook", id: notebook },
+ { type: "notebook", id: relation.fromId },
{
id: duplicateId,
type: "note"
}
);
}
-
- return duplicateId;
}
}
diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts
index 599680ef7..46a54b3e6 100644
--- a/packages/core/src/collections/relations.ts
+++ b/packages/core/src/collections/relations.ts
@@ -19,18 +19,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { makeId } from "../utils/id";
import { ICollection } from "./collection";
-import {
- Relation,
- ItemMap,
- ItemReference,
- ValueOf,
- MaybeDeletedItem
-} from "../types";
+import { Relation, ItemMap, ItemReference, ValueOf, ItemType } from "../types";
import Database from "../api";
-import { SQLCollection } from "../database/sql-collection";
-import { DatabaseAccessor, DatabaseSchema, isFalse } from "../database";
+import { FilteredSelector, SQLCollection } from "../database/sql-collection";
+import { DatabaseSchema, isFalse } from "../database";
import { SelectQueryBuilder } from "kysely";
+type ItemReferences = { type: ItemType; ids: string[] };
export class Relations implements ICollection {
name = "relations";
readonly collection: SQLCollection<"relations", Relation>;
@@ -39,6 +34,7 @@ export class Relations implements ICollection {
}
async init() {
+ await this.buildCache();
// return this.collection.init();
}
@@ -55,18 +51,71 @@ export class Relations implements ICollection {
});
}
+ from(
+ reference: ItemReference | ItemReferences,
+ types: (keyof RelatableTable)[]
+ ): RelationsArray<keyof RelatableTable>;
from<TType extends keyof RelatableTable>(
- reference: ItemReference,
+ reference: ItemReference | ItemReferences,
type: TType
+ ): RelationsArray<TType>;
+ from<TType extends keyof RelatableTable = keyof RelatableTable>(
+ reference: ItemReference | ItemReferences,
+ type: TType | keyof RelatableTable[]
) {
- return new RelationsArray(this.db, reference, type, "from");
+ return new RelationsArray(
+ this.db,
+ reference,
+ Array.isArray(type) ? type : [type],
+ "from"
+ );
}
+ to(
+ reference: ItemReference | ItemReferences,
+ types: (keyof RelatableTable)[]
+ ): RelationsArray<keyof RelatableTable>;
to<TType extends keyof RelatableTable>(
- reference: ItemReference,
+ reference: ItemReference | ItemReferences,
type: TType
+ ): RelationsArray<TType>;
+ to<TType extends keyof RelatableTable = keyof RelatableTable>(
+ reference: ItemReference | ItemReferences,
+ type: TType | keyof RelatableTable[]
) {
- return new RelationsArray(this.db, reference, type, "to");
+ return new RelationsArray(
+ this.db,
+ reference,
+ Array.isArray(type) ? type : [type],
+ "to"
+ );
+ }
+
+ fromCache: Map<string, string[]> = new Map();
+ toCache: Map<string, string[]> = new Map();
+ async buildCache() {
+ console.time("cache build");
+ this.fromCache.clear();
+ this.toCache.clear();
+
+ console.time("query");
+ const relations = await this.db
+ .sql()
+ .selectFrom("relations")
+ .select(["toId", "fromId"])
+ .$narrowType<{ toId: string; fromId: string }>()
+ .execute();
+ console.timeEnd("query");
+ for (const { fromId, toId } of relations) {
+ const fromIds = this.fromCache.get(fromId) || [];
+ fromIds.push(toId);
+ this.fromCache.set(fromId, fromIds);
+
+ const toIds = this.toCache.get(toId) || [];
+ toIds.push(fromId);
+ this.toCache.set(toId, toIds);
+ }
+ console.timeEnd("cache build");
}
// get raw() {
@@ -146,32 +195,40 @@ const TABLE_MAP = {
type RelatableTable = typeof TABLE_MAP;
class RelationsArray<TType extends keyof RelatableTable> {
- private table: ValueOf<RelatableTable> = TABLE_MAP[this.type];
+ private table: ValueOf<RelatableTable> = TABLE_MAP[this.types[0]];
constructor(
private readonly db: Database,
- private readonly reference: ItemReference,
- private readonly type: TType,
+ private readonly reference: ItemReference | ItemReferences,
+ private readonly types: TType[],
private readonly direction: "from" | "to"
) {}
- async resolve(limit?: number): Promise<ItemMap[TType][]> {
- const items = await this.db
- .sql()
- .selectFrom(this.table)
- .where("id", "in", (b) =>
- b
- .selectFrom("relations")
- .$call((eb) =>
- this.buildRelationsQuery()(
- eb as SelectQueryBuilder<DatabaseSchema, "relations", unknown>
+ get selector() {
+ return new FilteredSelector<ItemMap[TType]>(
+ this.table,
+ this.db
+ .sql()
+ .selectFrom<keyof DatabaseSchema>(this.table)
+ .where("id", "in", (b) =>
+ b
+ .selectFrom("relations")
+ .$call((eb) =>
+ this.buildRelationsQuery()(
+ eb as SelectQueryBuilder<DatabaseSchema, "relations", unknown>
+ )
)
- )
- )
+ )
+ // TODO: check if we need to index deleted field.
+ .where(isFalse("deleted")),
+ this.db.options?.batchSize
+ );
+ }
+
+ async resolve(limit?: number) {
+ const items = await this.selector.filter
.$if(limit !== undefined && limit > 0, (b) => b.limit(limit!))
.selectAll()
- // TODO: check if we need to index deleted field.
- .where(isFalse("deleted"))
.execute();
return items as unknown as ItemMap[TType][];
}
@@ -196,12 +253,20 @@ class RelationsArray<TType extends keyof RelatableTable> {
}
async get() {
- const ids = await this.db
+ const relations = await this.db
.sql()
.selectFrom("relations")
.$call(this.buildRelationsQuery())
+ .clearSelect()
+ .select(["fromId", "toId", "fromType", "toType"])
+ .$narrowType<{
+ fromId: string;
+ toId: string;
+ fromType: keyof ItemMap;
+ toType: keyof ItemMap;
+ }>()
.execute();
- return ids.map((i) => i.id);
+ return relations;
}
async count() {
@@ -216,13 +281,13 @@ class RelationsArray<TType extends keyof RelatableTable> {
return result.count;
}
- async has(id: string) {
+ async has(...ids: string[]) {
const result = await this.db
.sql()
.selectFrom("relations")
.$call(this.buildRelationsQuery())
.clearSelect()
- .where(this.direction === "from" ? "toId" : "fromId", "==", id)
+ .where(this.direction === "from" ? "toId" : "fromId", "in", ids)
.select((b) => b.fn.count<number>("id").as("count"))
.executeTakeFirst();
if (!result) return false;
@@ -240,15 +305,26 @@ class RelationsArray<TType extends keyof RelatableTable> {
) => {
if (this.direction === "to") {
return builder
- .where("fromType", "==", this.type)
+ .where(
+ "fromType",
+ this.types.length > 1 ? "in" : "==",
+ this.types.length > 1 ? this.types : this.types[0]
+ )
.where("toType", "==", this.reference.type)
- .where("toId", "==", this.reference.id)
+ .where(
+ "toId",
+ isItemReferences(this.reference) ? "in" : "==",
+ isItemReferences(this.reference)
+ ? this.reference.ids
+ : this.reference.id
+ )
.$if(
- this.type === "note" && this.db.trash.cache.notes.length > 0,
+ this.types.includes("note" as TType) &&
+ this.db.trash.cache.notes.length > 0,
(b) => b.where("fromId", "not in", this.db.trash.cache.notes)
)
.$if(
- this.type === "notebook" &&
+ this.types.includes("notebook" as TType) &&
this.db.trash.cache.notebooks.length > 0,
(b) => b.where("fromId", "not in", this.db.trash.cache.notebooks)
)
@@ -256,15 +332,26 @@ class RelationsArray<TType extends keyof RelatableTable> {
.$narrowType<{ id: string }>();
} else {
return builder
- .where("toType", "==", this.type)
+ .where(
+ "toType",
+ this.types.length > 1 ? "in" : "==",
+ this.types.length > 1 ? this.types : this.types[0]
+ )
.where("fromType", "==", this.reference.type)
- .where("fromId", "==", this.reference.id)
+ .where(
+ "fromId",
+ isItemReferences(this.reference) ? "in" : "==",
+ isItemReferences(this.reference)
+ ? this.reference.ids
+ : this.reference.id
+ )
.$if(
- this.type === "note" && this.db.trash.cache.notes.length > 0,
+ this.types.includes("note" as TType) &&
+ this.db.trash.cache.notes.length > 0,
(b) => b.where("toId", "not in", this.db.trash.cache.notes)
)
.$if(
- this.type === "notebook" &&
+ this.types.includes("notebook" as TType) &&
this.db.trash.cache.notebooks.length > 0,
(b) => b.where("toId", "not in", this.db.trash.cache.notebooks)
)
@@ -274,3 +361,9 @@ class RelationsArray<TType extends keyof RelatableTable> {
};
}
}
+
+function isItemReferences(
+ ref: ItemReference | ItemReferences
+): ref is ItemReferences {
+ return "ids" in ref;
+}
diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts
index e7c570c50..dddc28111 100644
--- a/packages/core/src/collections/reminders.ts
+++ b/packages/core/src/collections/reminders.ts
@@ -28,6 +28,7 @@ import { ICollection } from "./collection";
import { Reminder } from "../types";
import Database from "../api";
import { SQLCollection } from "../database/sql-collection";
+import { isFalse } from "../database";
dayjs.extend(isTomorrow);
dayjs.extend(isSameOrBefore);
@@ -84,9 +85,12 @@ export class Reminders implements ICollection {
// return this.collection.raw();
// }
- // get all() {
- // return this.collection.items();
- // }
+ get all() {
+ return this.collection.createFilter<Reminder>(
+ (qb) => qb.where(isFalse("deleted")),
+ this.db.options?.batchSize
+ );
+ }
exists(itemId: string) {
return this.collection.exists(itemId);
diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts
index 6fae38a99..b5509b139 100644
--- a/packages/core/src/collections/session-content.ts
+++ b/packages/core/src/collections/session-content.ts
@@ -52,17 +52,17 @@ export class SessionContent implements ICollection {
locked: TLocked
) {
if (!sessionId || !content) return;
- const data =
- locked || isCipher(content.data)
- ? content.data
- : await this.db.compressor().compress(content.data);
+ // const data =
+ // locked || isCipher(content.data)
+ // ? content.data
+ // : await this.db.compressor().compress(content.data);
await this.collection.upsert({
type: "sessioncontent",
id: makeSessionContentId(sessionId),
- data,
+ data: content.data,
contentType: content.type,
- compressed: !locked,
+ compressed: false,
localOnly: true,
locked,
dateCreated: Date.now(),
diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts
index bd7c8e62b..1f8356aa4 100644
--- a/packages/core/src/collections/shortcuts.ts
+++ b/packages/core/src/collections/shortcuts.ts
@@ -18,17 +18,20 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../api";
-import { isFalse } from "../database";
-import { SQLCollection } from "../database/sql-collection";
+import { SQLCachedCollection } from "../database/sql-cached-collection";
import { Shortcut } from "../types";
import { ICollection } from "./collection";
const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
export class Shortcuts implements ICollection {
name = "shortcuts";
- readonly collection: SQLCollection<"shortcuts", Shortcut>;
+ readonly collection: SQLCachedCollection<"shortcuts", Shortcut>;
constructor(private readonly db: Database) {
- this.collection = new SQLCollection(db.sql, "shortcuts", db.eventManager);
+ this.collection = new SQLCachedCollection(
+ db.sql,
+ "shortcuts",
+ db.eventManager
+ );
}
init() {
@@ -82,33 +85,25 @@ export class Shortcuts implements ICollection {
// }
get all() {
- return this.collection.createFilter<Shortcut>((qb) =>
- qb.where(isFalse("deleted"))
- );
+ return this.collection.items();
}
- async get() {
- // return this.all.reduce((prev, shortcut) => {
- // const {
- // item: { id }
- // } = shortcut;
- // let item: Notebook | Topic | Tag | null | undefined = null;
- // switch (shortcut.item.type) {
- // case "notebook": {
- // const notebook = this.db.notebooks.notebook(id);
- // item = notebook ? notebook.data : null;
- // break;
- // }
- // case "tag":
- // item = this.db.tags.tag(id);
- // break;
- // }
- // if (item) prev.push(item);
- // return prev;
- // }, [] as (Notebook | Topic | Tag)[]);
+ async resolved() {
+ const tagIds: string[] = [];
+ const notebookIds: string[] = [];
+ for (const shortcut of this.all) {
+ if (shortcut.itemType === "notebook") notebookIds.push(shortcut.itemId);
+ else if (shortcut.itemType === "tag") tagIds.push(shortcut.itemId);
+ }
+ return [
+ ...(notebookIds.length > 0
+ ? await this.db.notebooks.all.items(notebookIds)
+ : []),
+ ...(tagIds.length > 0 ? await this.db.tags.all.items(tagIds) : [])
+ ];
}
- async exists(id: string) {
+ exists(id: string) {
return this.collection.exists(id);
}
diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts
index f78bf48eb..8093a0645 100644
--- a/packages/core/src/collections/tags.ts
+++ b/packages/core/src/collections/tags.ts
@@ -71,8 +71,9 @@ export class Tags implements ICollection {
// }
get all() {
- return this.collection.createFilter<Tag>((qb) =>
- qb.where(isFalse("deleted"))
+ return this.collection.createFilter<Tag>(
+ (qb) => qb.where(isFalse("deleted")),
+ this.db.options?.batchSize
);
}
diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.ts
index 7284dc2f3..6818017e8 100644
--- a/packages/core/src/database/backup.ts
+++ b/packages/core/src/database/backup.ts
@@ -206,7 +206,7 @@ export default class Backup {
collection: DatabaseCollection<T, B>,
state: BackupState
) {
- for await (const item of collection.stream()) {
+ for await (const item of collection.stream() as any) {
const data = JSON.stringify(item);
state.buffer.push(data);
state.bufferLength += data.length;
diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.ts
index 7e6ba538f..796ae609c 100644
--- a/packages/core/src/database/cached-collection.ts
+++ b/packages/core/src/database/cached-collection.ts
@@ -29,6 +29,9 @@ import { StorageAccessor } from "../interfaces";
import EventManager from "../utils/event-manager";
import { chunkedIterate } from "../utils/array";
+/**
+ * @deprecated only kept here for migration purposes
+ */
export class CachedCollection<
TCollectionType extends CollectionType,
T extends ItemMap[Collections[TCollectionType]]
diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts
index 6fc478d08..4490b2a63 100644
--- a/packages/core/src/database/index.ts
+++ b/packages/core/src/database/index.ts
@@ -20,11 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import {
Migrator,
Kysely,
- SqliteAdapter,
- SqliteIntrospector,
- SqliteQueryCompiler,
sql,
- Driver,
KyselyPlugin,
PluginTransformQueryArgs,
PluginTransformResultArgs,
@@ -37,7 +33,8 @@ import {
Transaction,
ColumnType,
ExpressionBuilder,
- ReferenceExpression
+ ReferenceExpression,
+ Dialect
} from "kysely";
import {
Attachment,
@@ -119,9 +116,8 @@ export interface DatabaseCollection<T, IsAsync extends boolean> {
put(items: (T | undefined)[]): Promise<void>;
update(ids: string[], partial: Partial<T>): Promise<void>;
ids(options: GroupOptions): AsyncOrSyncResult<IsAsync, string[]>;
- items(
- ids: string[],
- sortOptions?: GroupOptions
+ records(
+ ids: string[]
): AsyncOrSyncResult<
IsAsync,
Record<string, MaybeDeletedItem<T> | undefined>
@@ -198,14 +194,17 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {
}
};
-export async function createDatabase(driver: Driver) {
+export type SQLiteOptions = {
+ dialect: Dialect;
+ journalMode?: "WAL" | "MEMORY" | "OFF" | "PERSIST" | "TRUNCATE" | "DELETE";
+ synchronous?: "normal" | "extra" | "full" | "off";
+ lockingMode?: "normal" | "exclusive";
+ cacheSize?: number;
+ pageSize?: number;
+};
+export async function createDatabase(options: SQLiteOptions) {
const db = new Kysely<DatabaseSchema>({
- dialect: {
- createAdapter: () => new SqliteAdapter(),
- createDriver: () => driver,
- createIntrospector: (db) => new SqliteIntrospector(db),
- createQueryCompiler: () => new SqliteQueryCompiler()
- },
+ dialect: options.dialect,
plugins: [new SqliteBooleanPlugin()]
});
@@ -214,8 +213,28 @@ export async function createDatabase(driver: Driver) {
provider: new NNMigrationProvider()
});
- await sql`PRAGMA journal_mode = WAL`.execute(db);
- await sql`PRAGMA synchronous = normal`.execute(db);
+ await sql`PRAGMA journal_mode = ${sql.raw(
+ options.journalMode || "WAL"
+ )}`.execute(db);
+
+ await sql`PRAGMA synchronous = ${sql.raw(
+ options.synchronous || "normal"
+ )}`.execute(db);
+
+ if (options.pageSize)
+ await sql`PRAGMA page_size = ${sql.raw(
+ options.pageSize.toString()
+ )}`.execute(db);
+
+ if (options.cacheSize)
+ await sql`PRAGMA cache_size = ${sql.raw(
+ options.cacheSize.toString()
+ )}`.execute(db);
+
+ if (options.lockingMode)
+ await sql`PRAGMA locking_mode = ${sql.raw(options.lockingMode)}`.execute(
+ db
+ );
await migrator.migrateToLatest();
@@ -240,6 +259,8 @@ export class SqliteBooleanPlugin implements KyselyPlugin {
args: PluginTransformResultArgs
): Promise<QueryResult<UnknownRow>> {
for (const row of args.result.rows) {
+ if (typeof row !== "object") continue;
+
for (const key in row) {
if (BooleanProperties.has(key as BooleanFields)) {
row[key] = row[key] === 1 ? true : false;
diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.ts
index 8189e0990..3f5fbe389 100644
--- a/packages/core/src/database/migrations.ts
+++ b/packages/core/src/database/migrations.ts
@@ -281,7 +281,6 @@ async function createFTS5Table(
const ref_ai = sql.raw(table + "_ai");
const ref_ad = sql.raw(table + "_ad");
const ref_au = sql.raw(table + "_au");
-
const indexed_cols = sql.raw(indexedColumns.join(", "));
const unindexed_cols =
unindexedColumns.length > 0
@@ -289,11 +288,9 @@ async function createFTS5Table(
: sql.raw("");
const new_indexed_cols = sql.raw(indexedColumns.join(", new."));
const old_indexed_cols = sql.raw(indexedColumns.join(", old."));
-
await sql`CREATE VIRTUAL TABLE ${ref_fts} USING fts5(
id UNINDEXED, ${unindexed_cols} ${indexed_cols}, content='${sql.raw(table)}'
)`.execute(db);
-
insertConditions = [
"(new.deleted is null or new.deleted == 0)",
...insertConditions
@@ -304,13 +301,11 @@ async function createFTS5Table(
BEGIN
INSERT INTO ${ref_fts}(rowid, id, ${indexed_cols}) VALUES (new.rowid, new.id, new.${new_indexed_cols});
END;`.execute(db);
-
await sql`CREATE TRIGGER ${ref_ad} AFTER DELETE ON ${ref}
BEGIN
INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})
VALUES ('delete', old.rowid, old.id, old.${old_indexed_cols});
END;`.execute(db);
-
await sql`CREATE TRIGGER ${ref_au} AFTER UPDATE ON ${ref}
BEGIN
INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})
diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.ts
index da1ef920a..5183e9951 100644
--- a/packages/core/src/database/sql-cached-collection.ts
+++ b/packages/core/src/database/sql-cached-collection.ts
@@ -28,7 +28,7 @@ export class SQLCachedCollection<
> implements DatabaseCollection<T, false>
{
private collection: SQLCollection<TCollectionType, T>;
- private cache = new Map<string, MaybeDeletedItem<T>>();
+ private cache = new Map<string, MaybeDeletedItem<T> | undefined>();
// private cachedItems?: T[];
constructor(
@@ -41,6 +41,8 @@ export class SQLCachedCollection<
async init() {
await this.collection.init();
+ const records = await this.collection.records([]);
+ this.cache = new Map(Object.entries(records));
// const data = await this.collection.indexer.readMulti(
// this.collection.indexer.indices
// );
@@ -114,10 +116,7 @@ export class SQLCachedCollection<
return Array.from(this.cache.keys());
}
- items(
- ids: string[],
- _sortOptions?: GroupOptions
- ): Record<string, MaybeDeletedItem<T> | undefined> {
+ records(ids: string[]): Record<string, MaybeDeletedItem<T> | undefined> {
const items: Record<string, MaybeDeletedItem<T> | undefined> = {};
for (const id of ids) {
items[id] = this.cache.get(id);
@@ -125,13 +124,30 @@ export class SQLCachedCollection<
return items;
}
+ items(ids?: string[]): T[] {
+ const items: T[] = [];
+ if (ids) {
+ for (const id of ids) {
+ const item = this.cache.get(id);
+ if (!item || isDeleted(item)) continue;
+ items.push(item);
+ }
+ } else {
+ for (const [_key, value] of this.cache) {
+ if (!value || isDeleted(value)) continue;
+ items.push(value);
+ }
+ }
+ return items;
+ }
+
*unsynced(
after: number,
chunkSize: number
): IterableIterator<MaybeDeletedItem<T>[]> {
let chunk: MaybeDeletedItem<T>[] = [];
for (const [_key, value] of this.cache) {
- if (value.dateModified && value.dateModified > after) {
+ if (value && value.dateModified && value.dateModified > after) {
chunk.push(value);
if (chunk.length === chunkSize) {
yield chunk;
@@ -144,7 +160,7 @@ export class SQLCachedCollection<
*stream(): IterableIterator<T> {
for (const [_key, value] of this.cache) {
- if (!value.deleted) yield value as T;
+ if (value && !value.deleted) yield value as T;
}
}
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
index 347a08c4e..76957a4c5 100644
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -27,7 +27,7 @@ import {
SQLiteItem,
isFalse
} from ".";
-import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely";
+import { ExpressionOrFactory, SelectQueryBuilder, SqlBool, sql } from "kysely";
import { VirtualizedGrouping } from "../utils/virtualized-grouping";
import { groupArray } from "../utils/grouping";
@@ -165,13 +165,13 @@ export class SQLCollection<
return ids.map((id) => id.id);
}
- async items(
+ async records(
ids: string[]
): Promise<Record<string, MaybeDeletedItem<T> | undefined>> {
const results = await this.db()
.selectFrom<keyof DatabaseSchema>(this.type)
.selectAll()
- .where("id", "in", ids)
+ .$if(ids.length > 0, (eb) => eb.where("id", "in", ids))
.execute();
const items: Record<string, MaybeDeletedItem<T>> = {};
for (const item of results) {
@@ -229,9 +229,10 @@ export class SQLCollection<
selector: (
qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>
) => SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>,
- batchSize = 50
+ batchSize?: number
) {
return new FilteredSelector<T>(
+ this.type,
this.db().selectFrom<keyof DatabaseSchema>(this.type).$call(selector),
batchSize
);
@@ -240,12 +241,13 @@ export class SQLCollection<
export class FilteredSelector<T extends Item> {
constructor(
+ readonly type: keyof DatabaseSchema,
readonly filter: SelectQueryBuilder<
DatabaseSchema,
keyof DatabaseSchema,
unknown
>,
- readonly batchSize: number
+ readonly batchSize: number = 500
) {}
async ids(sortOptions?: GroupOptions) {
@@ -269,6 +271,15 @@ export class FilteredSelector<T extends Item> {
.execute()) as T[];
}
+ async records(ids?: string[], sortOptions?: GroupOptions) {
+ const results = await this.items(ids, sortOptions);
+ const items: Record<string, T> = {};
+ for (const item of results) {
+ items[item.id] = item as T;
+ }
+ return items;
+ }
+
async has(id: string) {
const { count } =
(await this.filter
@@ -307,7 +318,14 @@ export class FilteredSelector<T extends Item> {
}
async grouped(options: GroupOptions) {
- const ids = await this.ids(options);
+ console.time("getting items");
+ const items = await this.filter
+ .$call(this.buildSortExpression(options))
+ .select(["id", options.sortBy, "type"])
+ .execute();
+ console.timeEnd("getting items");
+ console.log(items.length);
+ const ids = groupArray(items, options);
return new VirtualizedGrouping<T>(
ids,
this.batchSize,
@@ -321,8 +339,8 @@ export class FilteredSelector<T extends Item> {
items[item.id] = item as T;
}
return items;
- },
- (ids, items) => groupArray(ids, items, options)
+ }
+ //(ids, items) => groupArray(ids, items, options)
);
}
@@ -331,9 +349,20 @@ export class FilteredSelector<T extends Item> {
qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, T>
) => {
return qb
- .orderBy("conflicted desc")
- .orderBy("pinned desc")
- .orderBy(options.sortBy, options.sortDirection);
+ .$if(this.type === "notes", (eb) => eb.orderBy("conflicted desc"))
+ .$if(this.type === "notes" || this.type === "notebooks", (eb) =>
+ eb.orderBy("pinned desc")
+ )
+ .$if(options.sortBy === "title", (eb) =>
+ eb.orderBy(
+ sql`${sql.raw(options.sortBy)} COLLATE NOCASE ${sql.raw(
+ options.sortDirection
+ )}`
+ )
+ )
+ .$if(options.sortBy !== "title", (eb) =>
+ eb.orderBy(options.sortBy, options.sortDirection)
+ );
};
}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
new file mode 100644
index 000000000..a5dc39b27
--- /dev/null
+++ b/packages/core/src/index.ts
@@ -0,0 +1,21 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export * from "./types";
+export { VirtualizedGrouping } from "./utils/virtualized-grouping";
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index f0d13021b..757f6fc88 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -39,6 +39,8 @@ export type GroupingKey =
| "reminders";
export type ValueOf<T> = T[keyof T];
+export type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
+export type RequiredBy<T, K extends keyof T> = Partial<Omit<T, K>> & Pick<T, K>; // Pick<, K> & Omit<T, K>;
export type GroupHeader = {
type: "header";
@@ -443,6 +445,6 @@ export function isTrashItem(item: MaybeDeletedItem<Item>): item is TrashItem {
return !isDeleted(item) && item.type === "trash";
}
-export function isGroupHeader(item: GroupHeader | Item): item is GroupHeader {
+export function isGroupHeader(item: any): item is GroupHeader {
return item.type === "header";
}
diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts
index e85e77d6d..0f38de458 100644
--- a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts
+++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts
@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import { expect, test, vi } from "vitest";
+import { test, vi } from "vitest";
import { VirtualizedGrouping } from "../virtualized-grouping";
function item<T>(value: T) {
@@ -123,40 +123,3 @@ test("reloading ids should clear all cached batches", async (t) => {
t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
});
-
-test("merge groups if last & first groups are the same (sequential)", async (t) => {
- const mocked = createMock();
- const grouping = new VirtualizedGrouping<string>(
- ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
- 3,
- mocked,
- (ids) => [{ title: "Hello", id: ids[0] }]
- );
- expect((await grouping.item("1"))?.group?.title).toBe("Hello");
- expect((await grouping.item("4"))?.group).toBeUndefined();
-});
-
-test("merge groups if last & first groups are the same (random)", async (t) => {
- const mocked = createMock();
- const grouping = new VirtualizedGrouping<string>(
- ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
- 3,
- mocked,
- (ids) => [{ title: "Hello", id: ids[0] }]
- );
- expect((await grouping.item("1"))?.group?.title).toBe("Hello");
- expect((await grouping.item("7"))?.group).toBeUndefined();
-});
-
-test("merge groups if last & first groups are the same (reverse)", async (t) => {
- const mocked = createMock();
- const grouping = new VirtualizedGrouping<string>(
- ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
- 3,
- mocked,
- (ids) => [{ title: "Hello", id: ids[0] }]
- );
- expect((await grouping.item("7"))?.group?.title).toBe("Hello");
- expect((await grouping.item("1"))?.group?.title).toBe("Hello");
- expect((await grouping.item("7"))?.group).toBeUndefined();
-});
diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.ts
index 853e811da..12a7e6223 100644
--- a/packages/core/src/utils/grouping.ts
+++ b/packages/core/src/utils/grouping.ts
@@ -18,15 +18,23 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { isReminderActive } from "../collections/reminders";
-import { GroupOptions, Item } from "../types";
+import { GroupHeader, GroupOptions, ItemType } from "../types";
import { getWeekGroupFromTimestamp, MONTHS_FULL } from "./date";
-import { VirtualizedGroupHeader } from "./virtualized-grouping";
+type PartialGroupableItem = {
+ id: string;
+ type?: ItemType | null;
+ dateDeleted?: number | null;
+ title?: string | null;
+ filename?: string | null;
+ dateEdited?: number | null;
+ dateCreated?: number | null;
+};
type EvaluateKeyFunction<T> = (item: T) => string;
-export const getSortValue = <T extends Item>(
+export const getSortValue = (
options: GroupOptions,
- item: T
+ item: PartialGroupableItem
) => {
if (
options.sortBy === "dateDeleted" &&
@@ -43,18 +51,20 @@ export const getSortValue = <T extends Item>(
const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;
const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;
-function getKeySelector(options: GroupOptions): EvaluateKeyFunction<Item> {
- return (item: Item) => {
+function getKeySelector(
+ options: GroupOptions
+): EvaluateKeyFunction<PartialGroupableItem> {
+ return (item) => {
if ("pinned" in item && item.pinned) return "Pinned";
else if ("conflicted" in item && item.conflicted) return "Conflicted";
const date = new Date();
if (item.type === "reminder")
- return isReminderActive(item) ? "Active" : "Inactive";
+ return "Active"; // isReminderActive(item) ? "Active" : "Inactive";
else if (options.sortBy === "title")
return getFirstCharacter(getTitle(item));
else {
- const value = getSortValue(options, item);
+ const value = getSortValue(options, item) || 0;
switch (options.groupBy) {
case "none":
return "All";
@@ -80,36 +90,42 @@ function getKeySelector(options: GroupOptions): EvaluateKeyFunction<Item> {
}
export function groupArray(
- ids: string[],
- items: Record<string, Item>,
+ items: PartialGroupableItem[],
options: GroupOptions = {
groupBy: "default",
sortBy: "dateEdited",
sortDirection: "desc"
}
-): VirtualizedGroupHeader[] {
- const groups = new Map<string, VirtualizedGroupHeader>([
- ["Conflicted", { title: "Conflicted", id: "" }],
- ["Pinned", { title: "Pinned", id: "" }],
- ["Active", { title: "Active", id: "" }],
- ["Inactive", { title: "Inactive", id: "" }]
+): (string | GroupHeader)[] {
+ const groups = new Map<string, string[]>([
+ ["Conflicted", []],
+ ["Pinned", []]
]);
const keySelector = getKeySelector(options);
- for (const id of ids) {
- const item = items[id];
- if (!item) continue;
-
+ for (const item of items) {
const groupTitle = keySelector(item);
- const group = groups.get(groupTitle) || {
- title: groupTitle,
- id: ""
- };
- if (group.id === "") group.id = id;
+ const group = groups.get(groupTitle) || [];
+ group.push(item.id);
groups.set(groupTitle, group);
}
- return Array.from(groups.values());
+ return flattenGroups(groups);
+}
+
+function flattenGroups(groups: Map<string, string[]>) {
+ const items: (string | GroupHeader)[] = [];
+ groups.forEach((groupItems, groupTitle) => {
+ if (groupItems.length <= 0) return;
+ items.push({
+ title: groupTitle,
+ id: groupTitle.toLowerCase(),
+ type: "header"
+ });
+ items.push(...groupItems);
+ });
+
+ return items;
}
function getFirstCharacter(str: string) {
@@ -119,10 +135,6 @@ function getFirstCharacter(str: string) {
return str[0].toUpperCase();
}
-function getTitle(item: Item): string {
- return item.type === "attachment"
- ? item.filename
- : "title" in item
- ? item.title
- : "Unknown";
+function getTitle(item: PartialGroupableItem): string {
+ return item.filename || item.title || "Unknown";
}
diff --git a/packages/core/src/utils/object-id.ts b/packages/core/src/utils/object-id.ts
index 2355a602c..76a45ea64 100644
--- a/packages/core/src/utils/object-id.ts
+++ b/packages/core/src/utils/object-id.ts
@@ -30,3 +30,14 @@ export function createObjectId(date = Date.now()): string {
function swap16(val: number) {
return ((val & 0xff) << 16) | (val & 0xff00) | ((val >> 16) & 0xff);
}
+
+export function getObjectIdTimestamp(id: string) {
+ const timestamp = new Date();
+ const time =
+ id.charCodeAt(3) |
+ (id.charCodeAt(2) << 8) |
+ (id.charCodeAt(1) << 16) |
+ (id.charCodeAt(0) << 24);
+ timestamp.setTime(Math.floor(time) * 1000);
+ return timestamp;
+}
diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts
index afdbdbc38..bc8a2212b 100644
--- a/packages/core/src/utils/virtualized-grouping.ts
+++ b/packages/core/src/utils/virtualized-grouping.ts
@@ -17,51 +17,57 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-export type VirtualizedGroupHeader = {
- title: string;
- id: string;
-};
+import { GroupHeader, isGroupHeader } from "../types";
+
+type BatchOperator<T> = (
+ ids: string[],
+ items: Record<string, T>
+) => Promise<Record<string, unknown>>;
+type Batch<T> = { items: Record<string, T>; data?: Record<string, unknown> };
export class VirtualizedGrouping<T> {
- private cache: Map<number, Record<string, T>> = new Map();
- private groups: Map<number, VirtualizedGroupHeader[]> = new Map();
+ private cache: Map<number, Batch<T>> = new Map();
+ private pending: Map<number, Promise<Batch<T>>> = new Map();
+ groups: GroupHeader[] = [];
constructor(
- public ids: string[],
+ public ids: (string | GroupHeader)[],
private readonly batchSize: number,
- private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,
- private readonly groupItems: (
- ids: string[],
- items: Record<string, T>
- ) => VirtualizedGroupHeader[] = () => []
+ private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>
) {
this.ids = ids;
+ this.groups = ids.filter((i) => isGroupHeader(i)) as GroupHeader[];
+ }
+
+ getKey(index: number) {
+ const item = this.ids[index];
+ if (isGroupHeader(item)) return item.id;
+ return item;
}
/**
* Get item from cache or request the appropriate batch for caching
* and load it from there.
*/
- async item(id: string) {
+ item(id: string): Promise<T | undefined>;
+ item(
+ id: string,
+ operate: BatchOperator<T>
+ ): Promise<{ item: T; data: unknown } | undefined>;
+ async item(id: string, operate?: BatchOperator<T>) {
const index = this.ids.indexOf(id);
if (index <= -1) return;
const batchIndex = Math.floor(index / this.batchSize);
- const batch = this.cache.get(batchIndex) || (await this.load(batchIndex));
- const groups = this.groups.get(batchIndex);
+ const { items, data } =
+ this.cache.get(batchIndex) || (await this.loadBatch(batchIndex, operate));
- const group = groups?.find((g) => g.id === id);
- if (group)
- return {
- group: { type: "header", id: group.title, title: group.title },
- item: batch[id]
- };
- return { item: batch[id] };
+ return operate ? { item: items[id], data: data?.[id] } : items[id];
}
/**
* Reload the cache
*/
- refresh(ids: string[]) {
+ refresh(ids: (string | GroupHeader)[]) {
this.ids = ids;
this.cache.clear();
}
@@ -70,42 +76,40 @@ export class VirtualizedGrouping<T> {
*
* @param index
*/
- private async load(batch: number) {
- const start = batch * this.batchSize;
+ private async load(batchIndex: number, operate?: BatchOperator<T>) {
+ const start = batchIndex * this.batchSize;
const end = start + this.batchSize;
- const batchIds = this.ids.slice(start, end);
+ const batchIds = this.ids
+ .slice(start, end)
+ .filter((id) => typeof id === "string") as string[];
const items = await this.fetchItems(batchIds);
- const groups = this.groupItems(batchIds, items);
-
- const lastBatchIndex = this.last;
- const prevGroups = this.groups.get(lastBatchIndex);
- if (prevGroups && prevGroups.length > 0 && groups.length > 0) {
- const lastGroup = prevGroups[prevGroups.length - 1];
- if (lastGroup.title === groups[0].title) {
- // if user is moving downwards, we remove the last group from the
- // current batch, otherwise we remove the first group from the previous
- // batch.
- lastBatchIndex < batch ? groups.pop() : prevGroups.shift();
- }
- }
-
- this.cache.set(batch, items);
- this.groups.set(batch, groups);
+ console.time("operate");
+ const batch = {
+ items,
+ data: operate ? await operate(batchIds, items) : undefined
+ };
+ console.timeEnd("operate");
+ this.cache.set(batchIndex, batch);
this.clear();
- return items;
+ return batch;
+ }
+
+ private loadBatch(batch: number, operate?: BatchOperator<T>) {
+ if (this.pending.has(batch)) return this.pending.get(batch)!;
+ console.time("loading batch");
+ const promise = this.load(batch, operate);
+ this.pending.set(batch, promise);
+ return promise.finally(() => {
+ console.timeEnd("loading batch");
+ this.pending.delete(batch);
+ });
}
private clear() {
if (this.cache.size <= 2) return;
for (const [key] of this.cache) {
this.cache.delete(key);
- this.groups.delete(key);
if (this.cache.size === 2) break;
}
}
-
- private get last() {
- const keys = Array.from(this.cache.keys());
- return keys[keys.length - 1];
- }
}