Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: fix types & syncing issues

Abdullah Atta5b537ab4707b
diff --git a/packages/core/__e2e__/sync.test.js b/packages/core/__e2e__/sync.test.js
index b2b7fc9d2..55172e815 100644
--- a/packages/core/__e2e__/sync.test.js
+++ b/packages/core/__e2e__/sync.test.js
@@ -27,7 +27,7 @@ import { delay } from "../__tests__/utils";
import { test, expect, vitest } from "vitest";
import { login } from "./utils";
-const TEST_TIMEOUT = 30 * 1000;
+const TEST_TIMEOUT = 60 * 1000;
test(
"case 1: device A & B should only download the changes from device C (no uploading)",
@@ -311,12 +311,16 @@ test(
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy();
+ expect(
+ deviceB.notebooks.topics(id).topic("Topic 2").dateModified >
+ deviceA.notebooks.topics(id).topic("Topic 1").dateModified
+ ).toBeTruthy();
+ expect(
+ deviceB.notebooks.notebook(id).dateModified >
+ deviceA.notebooks.notebook(id).dateModified
+ ).toBeTruthy();
- await syncAndWait(deviceA, deviceB, false);
-
- // await delay(1000);
-
- // await syncAndWait(deviceB, deviceB, false);
+ await syncAndWait(deviceB, deviceA, false);
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 1")).toBeTruthy();
@@ -360,15 +364,27 @@ test(
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(1);
+ ctx.onTestFailed(() => {
+ console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+ console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+
+ deviceB.notes.topicReferences.rebuild();
+ deviceA.notes.topicReferences.rebuild();
+
+ console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+ console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+ });
await syncAndWait(deviceB, deviceA, false);
- await syncAndWait(deviceA, deviceB, false);
- expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
- expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
+ expect(deviceA.notes.note(noteB)).toBeDefined();
+ expect(deviceB.notes.note(noteA)).toBeDefined();
expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
+ expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
+ expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
+
await cleanup(deviceA, deviceB);
},
TEST_TIMEOUT
@@ -456,6 +472,7 @@ function syncAndWait(deviceA, deviceB, force = false) {
console.log("sync completed.");
resolve();
});
+
console.log(
"waiting for sync...",
"Device A:",
diff --git a/packages/core/__tests__/trash.test.js b/packages/core/__tests__/trash.test.js
index 01a3c785a..10b9a0706 100644
--- a/packages/core/__tests__/trash.test.js
+++ b/packages/core/__tests__/trash.test.js
@@ -45,7 +45,7 @@ test("permanently delete a note", () =>
expect(await note.content()).toBeDefined();
await db.trash.delete(db.trash.all[0].id);
expect(db.trash.all).toHaveLength(0);
- const content = await db.content.raw(note.data.contentId);
+ const content = await db.content.get(note.data.contentId);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(noteId);
@@ -229,7 +229,7 @@ test("clear trash should delete note content", () =>
expect(db.trash.all).toHaveLength(0);
- const content = await db.content.raw(note.contentId);
+ const content = await db.content.get(note.contentId);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(note.id);
diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.ts
index 8123500aa..e2e2bd5c3 100644
--- a/packages/core/src/api/sync/collector.ts
+++ b/packages/core/src/api/sync/collector.ts
@@ -19,103 +19,131 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { Cipher, SerializedKey } from "@notesnook/crypto";
import Database from "..";
-import { CURRENT_DATABASE_VERSION } from "../../common";
+import { CURRENT_DATABASE_VERSION, EV, EVENTS } from "../../common";
+import { logger } from "../../logger";
+import { SYNC_COLLECTIONS_MAP, SyncItem, SyncTransferItem } from "./types";
import { Item, MaybeDeletedItem } from "../../types";
-export type SyncableItemType =
- | "note"
- | "shortcut"
- | "notebook"
- | "content"
- | "attachment"
- | "reminder"
- | "relation"
- | "color"
- | "tag"
- | "settings";
-export type CollectedResult = {
- items: (MaybeDeletedItem<Item> | Cipher)[];
- types: (SyncableItemType | "vaultKey")[];
-};
-
-export type SyncItem = {
- id: string;
- v: number;
-} & Cipher;
-
+const ASYNC_COLLECTIONS_MAP = {
+ content: "content"
+} as const;
class Collector {
- private lastSyncedTimestamp = 0;
- private key?: SerializedKey;
+ logger = logger.scope("SyncCollector");
constructor(private readonly db: Database) {}
- async collect(lastSyncedTimestamp: number, isForceSync?: boolean) {
- await this.db.notes.init();
-
- this.lastSyncedTimestamp = lastSyncedTimestamp;
- this.key = await this.db.user.getEncryptionKey();
- const vaultKey = await this.db.vault.getKey();
-
- const collections = {
- note: this.db.notes.raw,
- shortcut: this.db.shortcuts.raw,
- notebook: this.db.notebooks.raw,
- content: await this.db.content.all(),
- attachment: this.db.attachments.syncable,
- reminder: this.db.reminders.raw,
- relation: this.db.relations.raw,
- color: this.db.colors.raw,
- tag: this.db.tags.raw,
- settings: [this.db.settings.raw]
- };
-
- const result: CollectedResult = {
- items: [],
- types: []
- };
- for (const type in collections) {
- this.collectInternal(
- type as SyncableItemType,
- collections[type as SyncableItemType],
- result,
- isForceSync
- );
+ async *collect(
+ chunkSize: number,
+ lastSyncedTimestamp: number,
+ isForceSync = false
+ ): AsyncGenerator<SyncTransferItem, void, unknown> {
+ const key = await this.db.user.getEncryptionKey();
+ if (!key || !key.key || !key.salt) {
+ EV.publish(EVENTS.userSessionExpired);
+ throw new Error("User encryption key not generated. Please relogin.");
}
- if (vaultKey) {
- result.items.push(vaultKey);
- result.types.push("vaultKey");
+ const settings = await this.prepareChunk(
+ [this.db.settings.raw],
+ lastSyncedTimestamp,
+ isForceSync,
+ key
+ );
+ if (settings) yield { items: settings, type: "settings" };
+
+ const attachments = await this.prepareChunk(
+ this.db.attachments.syncable,
+ lastSyncedTimestamp,
+ isForceSync,
+ key
+ );
+ if (attachments) yield { items: attachments, type: "attachment" };
+
+ for (const itemType in ASYNC_COLLECTIONS_MAP) {
+ const collectionKey =
+ ASYNC_COLLECTIONS_MAP[itemType as keyof typeof ASYNC_COLLECTIONS_MAP];
+ const collection = this.db[collectionKey].collection;
+ for await (const chunk of collection.iterate(chunkSize)) {
+ const items = await this.prepareChunk(
+ chunk.map((item) => item[1]),
+ lastSyncedTimestamp,
+ isForceSync,
+ key
+ );
+ if (!items) continue;
+ yield { items, type: itemType as keyof typeof ASYNC_COLLECTIONS_MAP };
+ }
}
- return result;
+ for (const itemType in SYNC_COLLECTIONS_MAP) {
+ const collectionKey =
+ SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];
+ const collection = this.db[collectionKey].collection;
+ for (const chunk of collection.iterateSync(chunkSize)) {
+ const items = await this.prepareChunk(
+ chunk,
+ lastSyncedTimestamp,
+ isForceSync,
+ key
+ );
+ if (!items) continue;
+ yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };
+ }
+ }
}
- private serialize(item: MaybeDeletedItem<Item>) {
- if (!this.key) throw new Error("No encryption key found.");
- return this.db.storage().encrypt(this.key, JSON.stringify(item));
+ async prepareChunk(
+ chunk: MaybeDeletedItem<Item>[],
+ lastSyncedTimestamp: number,
+ isForceSync: boolean,
+ key: SerializedKey
+ ) {
+ const { ids, items } = filterSyncableItems(
+ chunk,
+ lastSyncedTimestamp,
+ isForceSync
+ );
+ if (!ids.length) return;
+ const ciphers = await this.db.storage().encryptMulti(key, items);
+ return toPushItem(ids, ciphers);
}
+}
+export default Collector;
- encrypt(array: MaybeDeletedItem<Item>[]) {
- if (!array.length) return [];
- return Promise.all(array.map(this.map, this));
+function toPushItem(ids: string[], ciphers: Cipher<"base64">[]) {
+ if (ids.length !== ciphers.length)
+ throw new Error("ids.length must be equal to ciphers.length");
+
+ const items: SyncItem[] = [];
+ for (let i = 0; i < ids.length; ++i) {
+ const id = ids[i];
+ const cipher = ciphers[i];
+ items.push({ ...cipher, v: CURRENT_DATABASE_VERSION, id });
}
+ return items;
+}
- private collectInternal(
- itemType: SyncableItemType,
- items: MaybeDeletedItem<Item>[],
- result: CollectedResult,
- isForceSync?: boolean
- ) {
- if (!items || !items.length) return;
+function filterSyncableItems(
+ items: MaybeDeletedItem<Item>[],
+ lastSyncedTimestamp: number,
+ isForceSync = false
+): { items: string[]; ids: string[] } {
+ if (!items || !items.length) return { items: [], ids: [] };
- for (const item of items) {
- if (!item) continue;
+ const ids = [];
+ const syncableItems = [];
+ for (const item of items) {
+ if (!item) continue;
- const isSyncable = !item.synced || isForceSync;
- const isUnsynced =
- item.dateModified > this.lastSyncedTimestamp || isForceSync;
+ const isSyncable = !item.synced || isForceSync;
+ const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;
- if (isUnsynced && isSyncable) {
- result.items.push(
+ // synced is a local only property
+ delete item.synced;
+
+ if (isUnsynced && isSyncable) {
+ ids.push(item.id);
+ syncableItems.push(
+ JSON.stringify(
"localOnly" in item && item.localOnly
? {
id: item.id,
@@ -124,21 +152,9 @@ class Collector {
deleteReason: "localOnly"
}
: item
- );
- result.types.push(itemType);
- }
+ )
+ );
}
}
-
- private async map(item: MaybeDeletedItem<Item>) {
- // synced is a local only property
- delete item.synced;
-
- return {
- id: item.id,
- v: CURRENT_DATABASE_VERSION,
- ...(await this.serialize(item))
- };
- }
+ return { items: syncableItems, ids };
}
-export default Collector;
diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts
index 73c1d9fc6..ab74fb685 100644
--- a/packages/core/src/api/sync/index.ts
+++ b/packages/core/src/api/sync/index.ts
@@ -26,11 +26,7 @@ import {
} from "../../common";
import Constants from "../../utils/constants";
import TokenManager from "../token-manager";
-import Collector, {
- CollectedResult,
- SyncableItemType,
- SyncItem
-} from "./collector";
+import Collector from "./collector";
import * as signalr from "@microsoft/signalr";
import Merger from "./merger";
import Conflicts from "./conflicts";
@@ -40,6 +36,14 @@ import { Mutex } from "async-mutex";
import Database from "..";
import { migrateItem } from "../../migrations";
import { SerializedKey } from "@notesnook/crypto";
+import {
+ ItemMap,
+ MaybeDeletedItem,
+ Note,
+ Notebook,
+ TrashOrItem
+} from "../../types";
+import { SyncableItemType, SyncTransferItem } from "./types";
const ITEM_TYPE_TO_COLLECTION_TYPE = {
note: "notes",
@@ -48,7 +52,9 @@ const ITEM_TYPE_TO_COLLECTION_TYPE = {
attachment: "attachments",
relation: "relations",
reminder: "reminders",
- shortcut: "shortcuts"
+ shortcut: "shortcuts",
+ tag: "tags",
+ color: "colors"
};
export type SyncOptions = {
@@ -57,11 +63,6 @@ export type SyncOptions = {
serverLastSynced?: number;
};
-type SyncTransferItem = {
- items: SyncItem[];
- type: SyncableItemType;
-};
-
export default class SyncManager {
sync = new Sync(this.db);
constructor(private readonly db: Database) {}
@@ -169,6 +170,11 @@ class Sync {
}, 15000) as unknown as number;
const key = await this.db.user.getEncryptionKey();
+ if (!key || !key.key || !key.salt) {
+ EV.publish(EVENTS.userSessionExpired);
+ throw new Error("User encryption key not generated. Please relogin.");
+ }
+
const dbLastSynced = await this.db.lastSynced();
await this.processChunk(chunk, key, dbLastSynced, true);
});
@@ -208,7 +214,7 @@ class Sync {
if (
(options.type === "send" || options.type === "full") &&
- (await this.send(lastSynced, options.force, newLastSynced))
+ (await this.send(lastSynced, newLastSynced, options.force))
) {
this.logger.info("New data sent");
await this.stop(newLastSynced);
@@ -278,7 +284,7 @@ class Sync {
serverResponse.vaultKey.salt !== null &&
serverResponse.vaultKey.length > 0
) {
- await this.db.vault._setKey(serverResponse.vaultKey);
+ await this.db.vault.setKey(serverResponse.vaultKey);
}
this.connection.off("SendItems");
@@ -328,7 +334,7 @@ class Sync {
// return true;
}
- async stop(lastSynced) {
+ async stop(lastSynced: number) {
// refresh topic references
this.db.notes.topicReferences.rebuild();
// refresh monographs on sync completed
@@ -366,7 +372,7 @@ class Sync {
/**
* @private
*/
- async onPushCompleted(lastSynced) {
+ async onPushCompleted(lastSynced: number) {
// refresh topic references
this.db.notes.topicReferences.rebuild();
@@ -395,12 +401,14 @@ class Sync {
)
);
- let items = [];
- if (this.merger.isSyncCollection(chunk.type)) {
- items = deserialized.map((item) =>
- this.merger.mergeItemSync(item, chunk.type, dbLastSynced)
- );
- } else if (chunk.type === "content") {
+ const itemType = chunk.type;
+ let items: (
+ | MaybeDeletedItem<
+ ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>
+ >
+ | undefined
+ )[] = [];
+ if (itemType === "content") {
const localItems = await this.db.content.multi(
chunk.items.map((i) => i.id)
);
@@ -409,28 +417,33 @@ class Sync {
this.merger.mergeContent(item, localItems[item.id], dbLastSynced)
)
);
+ } else if (itemType === "settings") {
+ await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced);
+ return;
} else {
- items = await Promise.all(
- deserialized.map((item) =>
- this.merger.mergeItem(item, chunk.type, dbLastSynced)
- )
- );
+ items = this.merger.isSyncCollection(itemType)
+ ? deserialized.map((item) =>
+ this.merger.mergeItemSync(item, itemType, dbLastSynced)
+ )
+ : await Promise.all(
+ deserialized.map((item) =>
+ this.merger.mergeItem(item, itemType, dbLastSynced)
+ )
+ );
}
+ const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType];
+ await this.db[collectionType].collection.setItems(items as any);
+
if (
notify &&
- (chunk.type === "content" || chunk.type === "note") &&
+ (itemType === "note" || itemType === "content") &&
items.length > 0
) {
items.forEach((item) =>
this.db.eventManager.publish(EVENTS.syncItemMerged, item)
);
}
-
- const collectionType = this.itemTypeToCollection[chunk.type];
- if (collectionType && this.db[collectionType]) {
- await this.db[collectionType]._collection.setItems(items);
- }
}
private async pushItem(item: SyncTransferItem, newLastSynced: number) {
@@ -477,7 +490,11 @@ function promiseTimeout(ms: number, promise: Promise<unknown>) {
return Promise.race([promise, timeout]);
}
-async function deserializeItem(decryptedItem, version, database) {
+async function deserializeItem(
+ decryptedItem: string,
+ version: number,
+ database: Database
+) {
const deserialized = JSON.parse(decryptedItem);
deserialized.remote = true;
deserialized.synced = true;
diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.ts
index 8e44b8bb2..511be0bdb 100644
--- a/packages/core/src/api/sync/merger.ts
+++ b/packages/core/src/api/sync/merger.ts
@@ -17,292 +17,215 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import { migrateItem } from "../../migrations";
import { set } from "../../utils/set";
import { logger } from "../../logger";
import { isHTMLEqual } from "../../utils/html-diff";
-import { EV, EVENTS } from "../../common";
import Database from "..";
-import { SyncItem, SyncableItemType } from "./collector";
-import { Item, ItemMap, MaybeDeletedItem, isDeleted } from "../../types";
-import { SerializedKey } from "@notesnook/crypto";
-import { isCipher } from "../../database/crypto";
-
-type Conflict<P extends SyncableItemType> = (
- local: MaybeDeletedItem<ItemMap[P]>,
- remote: MaybeDeletedItem<ItemMap[P]>
-) => Promise<void>;
-
-type Set<P extends SyncableItemType> = (
- item: MaybeDeletedItem<ItemMap[P]>
-) => Promise<void>;
-
-type Get<P extends SyncableItemType> = (
- id: string
-) =>
- | MaybeDeletedItem<ItemMap[P]>
- | undefined
- | Promise<MaybeDeletedItem<ItemMap[P]> | undefined>;
-
-type MergeDefinition = {
- [P in SyncableItemType]: {
- threshold?: number;
- get?: Get<P>;
- set: Set<P>;
- conflict?: Conflict<P>;
- };
-};
+import { SYNC_COLLECTIONS_MAP } from "./types";
+import {
+ Attachment,
+ ContentItem,
+ Item,
+ ItemMap,
+ MaybeDeletedItem,
+ Note,
+ Notebook,
+ SettingsItem,
+ TrashOrItem,
+ isDeleted
+} from "../../types";
class Merger {
- private mergeDefinition: MergeDefinition;
- private logger = logger.scope("Merger");
- private lastSynced = 0;
- private key?: SerializedKey;
- constructor(private readonly db: Database) {
- this.mergeDefinition = {
- settings: {
- threshold: 1000,
- get: () => this.db.settings.raw,
- set: (item) => this.db.settings.merge(item),
- conflict: (_local, remote) => this.db.settings.merge(remote)
- },
- note: {
- get: (id) => this.db.notes.note(id)?.data,
- set: (item) => this.db.notes.merge(item)
- },
- shortcut: {
- get: (id) => this.db.shortcuts.shortcut(id),
- set: (item) => this.db.shortcuts.merge(item)
- },
- reminder: {
- get: (id) => this.db.reminders.reminder(id),
- set: (item) => this.db.reminders.merge(item)
- },
- relation: {
- get: (id) => this.db.relations.relation(id),
- set: (item) => this.db.relations.merge(item)
- },
- tag: {
- get: (id) => this.db.tags.tag(id),
- set: (item) => this.db.tags.merge(item)
- },
- color: {
- get: (id) => this.db.colors.color(id),
- set: (item) => this.db.colors.merge(item)
- },
- notebook: {
- threshold: 1000,
- get: (id) => this.db.notebooks.notebook(id)?.data,
- set: (item) => this.db.notebooks.merge(item),
- conflict: (_local, remote) => this.db.notebooks.merge(remote)
- },
- content: {
- threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,
- get: (id) => this.db.content.raw(id),
- set: async (item) => {
- await this.db.content.merge(item);
- },
- conflict: async (local, remote) => {
- if (isDeleted(local) || isDeleted(remote)) {
- if (remote.dateModified > local.dateModified)
- await db.content.merge(remote);
- return;
- }
-
- const note = this.db.notes.note(local.noteId);
- if (!note || !note.data) return;
+ logger = logger.scope("Merger");
+ constructor(private readonly db: Database) {}
- // if hashes are equal do nothing
- if (
- !note.locked &&
- (!remote ||
- !local ||
- !local.data ||
- !remote.data ||
- isHTMLEqual(local.data, remote.data))
- )
- return;
-
- if (note.locked) {
- // if note is locked or content is deleted we keep the most recent version.
- if (remote.dateModified > local.dateModified)
- await this.db.content.merge({ ...remote, id: local.id });
- } else {
- // otherwise we trigger the conflicts
- await this.db.content.merge({ ...local, conflicted: remote });
- await this.db.notes.add({ id: local.noteId, conflicted: true });
- await this.db.storage().write("hasConflicts", true);
- }
- }
- },
- attachment: {
- set: async (remoteAttachment) => {
- if (isDeleted(remoteAttachment)) {
- await this.db.attachments.merge(remoteAttachment);
- return;
- }
+ isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP {
+ return type in SYNC_COLLECTIONS_MAP;
+ }
- const localAttachment = this.db.attachments.attachment(
- remoteAttachment.metadata.hash
- );
- if (
- localAttachment &&
- localAttachment.dateUploaded !== remoteAttachment.dateUploaded
- ) {
- const noteIds = localAttachment.noteIds.slice();
- const isRemoved = await this.db.attachments.remove(
- localAttachment.metadata.hash,
- true
- );
- if (!isRemoved)
- throw new Error(
- "Conflict could not be resolved in one of the attachments."
- );
- remoteAttachment.noteIds = set.union(
- remoteAttachment.noteIds,
- noteIds
- );
- }
- await this.db.attachments.merge(remoteAttachment);
+ isConflicted(
+ localItem: MaybeDeletedItem<Item>,
+ remoteItem: MaybeDeletedItem<Item>,
+ lastSynced: number,
+ conflictThreshold: number
+ ) {
+ const isResolved =
+ "dateResolved" in localItem &&
+ localItem.dateResolved === remoteItem.dateModified;
+ const isModified =
+ // the local item is modified if it was changed/modified after the last
+ // sync i.e. it wasn't synced yet.
+ // However, in case a sync is interrupted the local item's date modified
+ // will be ahead of last sync. In that case, we also have to check if the
+ // synced flag is false (it is only false if a user makes edits on the
+ // local device).
+ localItem.dateModified > lastSynced && !localItem.synced;
+ if (isModified && !isResolved) {
+ // If time difference between local item's edits & remote item's edits
+ // is less than threshold, we shouldn't trigger a merge conflict; instead
+ // we will keep the most recently changed item.
+ const timeDiff =
+ Math.max(remoteItem.dateModified, localItem.dateModified) -
+ Math.min(remoteItem.dateModified, localItem.dateModified);
+
+ if (timeDiff < conflictThreshold) {
+ if (remoteItem.dateModified > localItem.dateModified) {
+ return "merge";
}
+ return;
}
- };
- }
-
- async _migrate(deserialized: Item, version: number) {
- // it is a locked note, bail out.
- if (isCipher(deserialized) && deserialized.alg && deserialized.cipher)
- return deserialized;
-
- return migrateItem(deserialized, version, deserialized.type, this.db);
- }
-
- async _deserialize(item: SyncItem, migrate = true) {
- if (!this.key) throw new Error("User encryption key not found.");
- const decrypted = await this.db.storage().decrypt(this.key, item);
- if (!decrypted) {
- throw new Error("Decrypted item cannot be undefined or empty.");
+ return "conflict";
+ } else if (!isResolved) {
+ return "merge";
}
-
- const deserialized = JSON.parse(decrypted);
- deserialized.remote = true;
- deserialized.synced = true;
- if (!migrate) return deserialized;
- await this._migrate(deserialized, item.v);
- return deserialized;
}
- async _mergeItem<TItemType extends SyncableItemType>(
- syncItem: SyncItem,
- get: Get<TItemType>,
- add: Set<TItemType>
+ mergeItemSync<TType extends keyof typeof SYNC_COLLECTIONS_MAP>(
+ remoteItem: MaybeDeletedItem<
+ ItemMap[TType] | TrashOrItem<Note> | TrashOrItem<Notebook>
+ >,
+ type: TType,
+ lastSynced: number
) {
- const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
- ItemMap[TItemType]
- >;
- const localItem = await get(remoteItem.id);
- if (!localItem || remoteItem.dateModified > localItem.dateModified) {
- await add(remoteItem);
- return remoteItem;
+ switch (type) {
+ case "shortcut":
+ case "reminder":
+ case "tag":
+ case "color":
+ case "note":
+ case "relation": {
+ const localItem = this.db[SYNC_COLLECTIONS_MAP[type]].collection.getRaw(
+ remoteItem.id
+ );
+ if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+ return remoteItem;
+ }
+ break;
+ }
+ // case "note": {
+ // const localItem = this.db.notes.collection.getRaw(remoteItem.id);
+ // if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+ // return this.db.notes.merge(
+ // localItem,
+ // remoteItem as MaybeDeletedItem<TrashOrItem<Note>>
+ // );
+ // }
+ // break;
+ // }
+ case "notebook": {
+ const THRESHOLD = 1000;
+ const localItem = this.db.notebooks.collection.getRaw(remoteItem.id);
+ if (
+ !localItem ||
+ this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)
+ ) {
+ return this.db.notebooks.merge(
+ localItem,
+ remoteItem as MaybeDeletedItem<TrashOrItem<Notebook>>,
+ lastSynced
+ );
+ }
+ break;
+ }
}
}
- async _mergeItemWithConflicts<TItemType extends SyncableItemType>(
- syncItem: SyncItem,
- get: Get<TItemType>,
- add: Set<TItemType>,
- markAsConflicted: Conflict<TItemType>,
- threshold: number
+ async mergeContent(
+ remoteItem: MaybeDeletedItem<ContentItem>,
+ localItem: MaybeDeletedItem<ContentItem>,
+ lastSynced: number
) {
- const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
- ItemMap[TItemType]
- >;
- const localItem = await get(remoteItem.id);
+ if (localItem && "localOnly" in localItem && localItem.localOnly) return;
- if (!localItem || isDeleted(localItem)) {
- await add(remoteItem);
+ const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
+ const conflicted =
+ localItem &&
+ this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);
+ if (!localItem || conflicted === "merge") {
return remoteItem;
- } else {
- const isResolved =
- "dateResolved" in localItem &&
- localItem.dateResolved === remoteItem.dateModified;
- const isModified =
- // the local item is modified if it was changed/modified after the last sync
- // i.e. it wasn't synced yet.
- // However, in case a sync is interrupted the local item's date modified will
- // be ahead of last sync. In that case, we also have to check if the synced flag
- // is false (it is only false if a user makes edits on the local device).
- localItem.dateModified > this.lastSynced && !localItem.synced;
- if (isModified && !isResolved) {
- // If time difference between local item's edits & remote item's edits
- // is less than threshold, we shouldn't trigger a merge conflict; instead
- // we will keep the most recently changed item.
- const timeDiff =
- Math.max(remoteItem.dateModified, localItem.dateModified) -
- Math.min(remoteItem.dateModified, localItem.dateModified);
-
- if (timeDiff < threshold) {
- if (remoteItem.dateModified > localItem.dateModified) {
- await add(remoteItem);
- return remoteItem;
- }
- return;
- }
+ } else if (conflicted === "conflict") {
+ if (isDeleted(localItem) || isDeleted(remoteItem)) {
+ if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
+ return;
+ }
- this.logger.info("Conflict detected", {
- itemId: remoteItem.id,
- isResolved,
- isModified,
- timeDiff,
- remote: remoteItem.dateModified,
- local: localItem.dateModified,
- lastSynced: this.lastSynced
+ const note = this.db.notes.collection.get(localItem.noteId);
+ if (!note) return;
+
+ // if hashes are equal do nothing
+ if (
+ !note.locked &&
+ (!remoteItem ||
+ !remoteItem ||
+ !localItem.data ||
+ !remoteItem.data ||
+ isHTMLEqual(localItem.data, remoteItem.data))
+ )
+ return;
+
+ if (note.locked) {
+ // if note is locked or content is deleted we keep the most recent version.
+ if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
+ } else {
+ // otherwise we trigger the conflicts
+ await this.db.notes.add({
+ id: localItem.noteId,
+ conflicted: true
});
-
- await markAsConflicted(localItem, remoteItem);
- } else if (!isResolved) {
- await add(remoteItem);
- return remoteItem;
+ await this.db.storage().write("hasConflicts", true);
+ return {
+ ...localItem,
+ conflicted: remoteItem
+ };
}
}
}
- async mergeItem<TItemType extends SyncableItemType>(
- type: TItemType | "vaultKey",
- item: SyncItem
+ async mergeItem(
+ remoteItem: SettingsItem | MaybeDeletedItem<Attachment>,
+ type: "settings" | "attachment",
+ lastSynced: number
) {
- this.lastSynced = await this.db.lastSynced();
-
- if (!this.key) this.key = await this.db.user.getEncryptionKey();
- if (!this.key || !this.key.key || !this.key.salt) {
- EV.publish(EVENTS.userSessionExpired);
- throw new Error("User encryption key not generated. Please relogin.");
- }
-
- if (type === "vaultKey") {
- await this.db.vault.setKey(await this._deserialize(item, false));
- return;
- }
-
- const definition = this.mergeDefinition[type];
- if (definition.conflict && definition.get && definition.threshold) {
- return await this._mergeItemWithConflicts<TItemType>(
- item,
- definition.get,
- definition.set,
- definition.conflict,
- definition.threshold
- );
- } else if (definition.get && definition.set) {
- return await this._mergeItem<TItemType>(
- item,
- definition.get,
- definition.set
- );
- } else if (!definition.get && !!definition.set) {
- const remote = await this._deserialize(item);
- await definition.set(remote);
+ switch (type) {
+ case "settings": {
+ if (isDeleted(remoteItem) || remoteItem.type !== "settings") return;
+
+ const localItem = this.db.settings.raw;
+ if (
+ !localItem ||
+ this.isConflicted(localItem, remoteItem, lastSynced, 1000)
+ ) {
+ await this.db.settings.merge(remoteItem, lastSynced);
+ }
+ break;
+ }
+ case "attachment": {
+ if (isDeleted(remoteItem)) {
+ return this.db.attachments.merge(undefined, remoteItem);
+ }
+ if (remoteItem.type !== "attachment") return;
+
+ const localAttachment = this.db.attachments.attachment(
+ remoteItem.metadata.hash
+ );
+ if (
+ localAttachment &&
+ localAttachment.dateUploaded !== remoteItem.dateUploaded
+ ) {
+ const noteIds = localAttachment.noteIds.slice();
+ const isRemoved = await this.db.attachments.remove(
+ localAttachment.metadata.hash,
+ true
+ );
+ if (!isRemoved)
+ throw new Error(
+ "Conflict could not be resolved in one of the attachments."
+ );
+ remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds);
+ }
+ return this.db.attachments.merge(undefined, remoteItem);
+ }
}
}
}
diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.ts
new file mode 100644
index 000000000..4d53748e5
--- /dev/null
+++ b/packages/core/src/api/sync/types.ts
@@ -0,0 +1,52 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Cipher } from "@notesnook/crypto";
+
+export type SyncableItemType =
+ | "note"
+ | "shortcut"
+ | "notebook"
+ | "content"
+ | "attachment"
+ | "reminder"
+ | "relation"
+ | "color"
+ | "tag"
+ | "settings";
+
+export type SyncItem = {
+ id: string;
+ v: number;
+} & Cipher<"base64">;
+
+export const SYNC_COLLECTIONS_MAP = {
+ note: "notes",
+ notebook: "notebooks",
+ shortcut: "shortcuts",
+ reminder: "reminders",
+ relation: "relations",
+ tag: "tags",
+ color: "colors"
+} as const;
+
+export type SyncTransferItem = {
+ items: SyncItem[];
+ type: SyncableItemType;
+};
diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.ts
index 55a7a4067..5abc4bd50 100644
--- a/packages/core/src/api/vault.ts
+++ b/packages/core/src/api/vault.ts
@@ -209,7 +209,7 @@ export default class Vault {
return await this.lockNote(note, await this.getVaultPassword());
}
- async exists(vaultKey?: Cipher) {
+ async exists(vaultKey?: Cipher<"base64">) {
if (!vaultKey) vaultKey = await this.getKey();
return vaultKey && isCipher(vaultKey);
}
@@ -370,10 +370,10 @@ export default class Vault {
}
async getKey() {
- return await this.db.storage().read<Cipher>("vaultKey");
+ return await this.db.storage().read<Cipher<"base64">>("vaultKey");
}
- async setKey(vaultKey: Cipher) {
+ async setKey(vaultKey: Cipher<"base64">) {
await this.db.storage().write("vaultKey", vaultKey);
}
}
diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts
index 0d48284be..1dcfda2b9 100644
--- a/packages/core/src/collections/attachments.ts
+++ b/packages/core/src/collections/attachments.ts
@@ -43,7 +43,7 @@ import Database from "../api";
export class Attachments implements ICollection {
name = "attachments";
key: Cipher<"base64"> | null = null;
- private readonly collection: CachedCollection<"attachments", Attachment>;
+ readonly collection: CachedCollection<"attachments", Attachment>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
@@ -109,7 +109,7 @@ export class Attachments implements ICollection {
}
merge(
- localAttachment: MaybeDeletedItem<Attachment>,
+ localAttachment: MaybeDeletedItem<Attachment> | undefined,
remoteAttachment: MaybeDeletedItem<Attachment>
) {
if (isDeleted(remoteAttachment)) return remoteAttachment;
@@ -323,7 +323,10 @@ export class Attachments implements ICollection {
return (
outputType === "base64"
- ? dataurl.fromObject({ type: attachment.metadata.type, data })
+ ? dataurl.fromObject({
+ type: attachment.metadata.type,
+ data
+ })
: data
) as Output<TOutputFormat>;
}
diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts
index a1710a86b..ded823f59 100644
--- a/packages/core/src/collections/colors.ts
+++ b/packages/core/src/collections/colors.ts
@@ -36,7 +36,7 @@ export const DefaultColors: Record<string, string> = {
export class Colors implements ICollection {
name = "colors";
- private readonly collection: CachedCollection<"colors", Color>;
+ readonly collection: CachedCollection<"colors", Color>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts
index ef119c9d8..efd437b6f 100644
--- a/packages/core/src/collections/content.ts
+++ b/packages/core/src/collections/content.ts
@@ -49,7 +49,7 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
export class Content implements ICollection {
name = "content";
- private readonly collection: IndexedCollection<"content", ContentItem>;
+ readonly collection: IndexedCollection<"content", ContentItem>;
constructor(private readonly db: Database) {
this.collection = new IndexedCollection(
db.storage,
@@ -130,7 +130,7 @@ export class Content implements ICollection {
async get(id: string) {
const content = await this.raw(id);
if (!content || isDeleted(content)) return;
- return content.data;
+ return content;
}
async raw(id: string) {
diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts
index 1fd5d5f1d..2142ca14d 100644
--- a/packages/core/src/collections/notebooks.ts
+++ b/packages/core/src/collections/notebooks.ts
@@ -27,7 +27,6 @@ import {
MaybeDeletedItem,
Notebook,
Topic,
- TrashItem,
TrashOrItem,
isDeleted,
isTrashItem
@@ -52,15 +51,23 @@ export class Notebooks implements ICollection {
return this.collection.init();
}
- async merge(remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>) {
+ merge(
+ localNotebook: MaybeDeletedItem<TrashOrItem<Notebook>> | undefined,
+ remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>,
+ lastSyncedTimestamp: number
+ ) {
if (isDeleted(remoteNotebook) || isTrashItem(remoteNotebook))
- return await this.collection.add(remoteNotebook);
+ return remoteNotebook;
- const id = remoteNotebook.id;
- const localNotebook = this.collection.get(id);
+ if (
+ localNotebook &&
+ (isTrashItem(localNotebook) || isDeleted(localNotebook))
+ ) {
+ if (localNotebook.dateModified > remoteNotebook.dateModified) return;
+ return remoteNotebook;
+ }
if (localNotebook && localNotebook.topics?.length) {
- const lastSyncedTimestamp = await this.db.lastSynced();
let isChanged = false;
// merge new and old topics
for (const oldTopic of localNotebook.topics) {
@@ -93,7 +100,7 @@ export class Notebooks implements ICollection {
}
remoteNotebook.remote = !isChanged;
}
- return await this.collection.add(remoteNotebook);
+ return remoteNotebook;
}
async add(
diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts
index 8eaff927e..dcbb2ff57 100644
--- a/packages/core/src/collections/notes.ts
+++ b/packages/core/src/collections/notes.ts
@@ -27,13 +27,7 @@ import { Tiptap } from "../content-types/tiptap";
import { EMPTY_CONTENT, isUnencryptedContent } from "./content";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { buildFromTemplate } from "../utils/templates";
-import {
- Note,
- TrashOrItem,
- isTrashItem,
- MaybeDeletedItem,
- isDeleted
-} from "../types";
+import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types";
import Database from "../api";
import { CachedCollection } from "../database/cached-collection";
import { ICollection } from "./collection";
@@ -67,17 +61,6 @@ export class Notes implements ICollection {
this.topicReferences.rebuild();
}
- async merge(remoteNote: MaybeDeletedItem<TrashOrItem<Note>>) {
- if (!remoteNote) return;
-
- const id = remoteNote.id;
- const localNote = this.collection.get(id);
-
- if (localNote && localNote.localOnly) return;
-
- return await this.collection.add(remoteNote);
- }
-
async add(
item: Partial<Note & { content: NoteContent<false>; sessionId: string }>
): Promise<string | undefined> {
@@ -430,6 +413,9 @@ export class Notes implements ICollection {
this.topicReferences.rebuild();
}
+ /**
+ * @internal
+ */
async _clearAllNotebookReferences(notebookId: string) {
const notes = this.db.notes.all;
@@ -482,7 +468,7 @@ class NoteIdCache {
for (const note of notes) {
const { notebooks } = note;
- if (!notebooks) continue;
+ if (!notebooks) return;
for (const notebook of notebooks) {
for (const topic of notebook.topics) {
diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts
index b5050ed56..9ba70a8c2 100644
--- a/packages/core/src/collections/relations.ts
+++ b/packages/core/src/collections/relations.ts
@@ -29,7 +29,7 @@ type RelationsArray<TType extends keyof ItemMap> = Relation[] & {
export class Relations implements ICollection {
name = "relations";
- private readonly collection: CachedCollection<"relations", Relation>;
+ readonly collection: CachedCollection<"relations", Relation>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts
index d0178c231..ff3e2e41b 100644
--- a/packages/core/src/collections/reminders.ts
+++ b/packages/core/src/collections/reminders.ts
@@ -36,7 +36,7 @@ dayjs.extend(isToday);
export class Reminders implements ICollection {
name = "reminders";
- private readonly collection: CachedCollection<"reminders", Reminder>;
+ readonly collection: CachedCollection<"reminders", Reminder>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts
index 16d5942b7..0936ab6f7 100644
--- a/packages/core/src/collections/session-content.ts
+++ b/packages/core/src/collections/session-content.ts
@@ -27,7 +27,7 @@ import Database from "../api";
import { ContentType, SessionContentItem, isDeleted } from "../types";
export type NoteContent<TLocked extends boolean> = {
- data: TLocked extends true ? Cipher : string;
+ data: TLocked extends true ? Cipher<"base64"> : string;
type: ContentType;
};
diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts
index ecd4a8ea7..e032e71aa 100644
--- a/packages/core/src/collections/shortcuts.ts
+++ b/packages/core/src/collections/shortcuts.ts
@@ -25,7 +25,7 @@ import { ICollection } from "./collection";
const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
export class Shortcuts implements ICollection {
name = "shortcuts";
- private readonly collection: CachedCollection<"shortcuts", Shortcut>;
+ readonly collection: CachedCollection<"shortcuts", Shortcut>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts
index 730c4fb07..92b90ba56 100644
--- a/packages/core/src/collections/tags.ts
+++ b/packages/core/src/collections/tags.ts
@@ -25,7 +25,7 @@ import { ICollection } from "./collection";
export class Tags implements ICollection {
name = "tags";
- private readonly collection: CachedCollection<"tags", Tag>;
+ readonly collection: CachedCollection<"tags", Tag>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(db.storage, "tags", db.eventManager);
}
diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.ts
index 12a85df87..1c6a13726 100644
--- a/packages/core/src/database/cached-collection.ts
+++ b/packages/core/src/database/cached-collection.ts
@@ -28,7 +28,7 @@ import {
} from "../types";
import { StorageAccessor } from "../interfaces";
import EventManager from "../utils/event-manager";
-import { toChunks } from "../utils/array";
+import { chunkedIterate } from "../utils/array";
export class CachedCollection<
TCollectionType extends CollectionType,
@@ -109,6 +109,11 @@ export class CachedCollection<
return item;
}
+ getRaw(id: string) {
+ const item = this.cache.get(id);
+ return item;
+ }
+
raw() {
return Array.from(this.cache.values());
}
@@ -140,10 +145,7 @@ export class CachedCollection<
}
*iterateSync(chunkSize: number) {
- const chunks = toChunks(Array.from(this.cache.values()), chunkSize);
- for (const chunk of chunks) {
- yield chunk;
- }
+ yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);
}
invalidateCache() {
diff --git a/packages/core/src/database/crypto.ts b/packages/core/src/database/crypto.ts
index 1128e0b91..4673c2d51 100644
--- a/packages/core/src/database/crypto.ts
+++ b/packages/core/src/database/crypto.ts
@@ -31,7 +31,7 @@ export class Crypto {
}
}
-export function isCipher(item: any): item is Cipher {
+export function isCipher(item: any): item is Cipher<"base64"> {
return (
typeof item === "object" &&
"cipher" in item &&
diff --git a/packages/core/src/database/indexed-collection.ts b/packages/core/src/database/indexed-collection.ts
index ac9b1c151..99168e291 100644
--- a/packages/core/src/database/indexed-collection.ts
+++ b/packages/core/src/database/indexed-collection.ts
@@ -108,7 +108,7 @@ export class IndexedCollection<
return Object.fromEntries(data);
}
- setItems(items) {
+ setItems(items: (MaybeDeletedItem<T> | undefined)[]) {
const entries = items.reduce((array, item) => {
if (!item) return array;
@@ -120,7 +120,7 @@ export class IndexedCollection<
array.push([item.id, item]);
return array;
- }, []);
+ }, [] as [string, MaybeDeletedItem<T>][]);
return this.indexer.writeMulti(entries);
}
diff --git a/packages/core/src/database/indexer.ts b/packages/core/src/database/indexer.ts
index 8625b083b..8d8813ad5 100644
--- a/packages/core/src/database/indexer.ts
+++ b/packages/core/src/database/indexer.ts
@@ -86,13 +86,15 @@ export default class Indexer<T> {
* @param {any[]} items
* @returns
*/
- async writeMulti(items) {
- const entries = items.map(([id, item]) => {
- if (!this.indices.includes(id)) this.indices.push(id);
- return [this.makeId(id), item];
- });
+ async writeMulti(items: [string, MaybeDeletedItem<T>][]) {
+ const entries: [string, MaybeDeletedItem<T> | string[]][] = items.map(
+ ([id, item]) => {
+ if (!this.indices.includes(id)) this.indices.push(id);
+ return [this.makeId(id), item];
+ }
+ );
entries.push([this.type, this.indices]);
- await super.writeMulti(entries);
+ await this.storage().writeMulti(entries);
}
async migrateIndices() {
diff --git a/packages/core/src/utils/array.js b/packages/core/src/utils/array.ts
similarity index 64%
rename from packages/core/src/utils/array.js
rename to packages/core/src/utils/array.ts
index 4c92b0cdb..9f20fd5b0 100644
--- a/packages/core/src/utils/array.js
+++ b/packages/core/src/utils/array.ts
@@ -17,48 +17,58 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-export function findItemAndDelete(array, predicate) {
+export function findItemAndDelete<T>(
+ array: T[],
+ predicate: (item: T) => boolean
+) {
return deleteAtIndex(array, array.findIndex(predicate));
}
-export function addItem(array, item) {
+export function addItem<T>(array: T[], item: T) {
const index = array.indexOf(item);
if (index > -1) return false;
array.push(item);
return true;
}
-export function deleteItem(array, item) {
+export function deleteItem<T>(array: T[], item: T) {
return deleteAtIndex(array, array.indexOf(item));
}
-export function deleteItems(array, ...items) {
- for (let item of items) {
+export function deleteItems<T>(array: T[], ...items: T[]) {
+ for (const item of items) {
deleteItem(array, item);
}
}
-export function findById(array, id) {
+export function findById<T extends { id: string }>(array: T[], id: string) {
if (!array) return false;
return array.find((item) => item.id === id);
}
-export function hasItem(array, item) {
+export function hasItem<T>(array: T[], item: T) {
if (!array) return false;
return array.indexOf(item) > -1;
}
-function deleteAtIndex(array, index) {
+function deleteAtIndex<T>(array: T[], index: number) {
if (index === -1) return false;
array.splice(index, 1);
return true;
}
-export function toChunks(array, chunkSize) {
- let chunks = [];
+export function toChunks<T>(array: T[], chunkSize: number) {
+ const chunks: T[][] = [];
for (let i = 0; i < array.length; i += chunkSize) {
const chunk = array.slice(i, i + chunkSize);
chunks.push(chunk);
}
return chunks;
}
+
+export function* chunkedIterate<T>(array: T[], chunkSize: number) {
+ for (let i = 0; i < array.length; i += chunkSize) {
+ const chunk = array.slice(i, i + chunkSize);
+ yield chunk;
+ }
+}