A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: migrate attachments to use relations for linking to notes
Abdullah Attacc6055883939
diff --git a/packages/core/__tests__/backup.test.js b/packages/core/__tests__/backup.test.jsindex 15b0e3949..f257205be 100644--- a/packages/core/__tests__/backup.test.js+++ b/packages/core/__tests__/backup.test.js@@ -174,6 +174,8 @@ describe.each([db.attachments.all.every((v) => v.dateModified > 0 && !v.dateEdited)).toBeTruthy();+ expect(db.attachments.all.every((a) => !a.noteIds)).toBeTruthy();+if (data.data.settings.pins)expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.tsindex 511be0bdb..5a6b163c9 100644--- a/packages/core/src/api/sync/merger.ts+++ b/packages/core/src/api/sync/merger.ts@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { set } from "../../utils/set";import { logger } from "../../logger";import { isHTMLEqual } from "../../utils/html-diff";import Database from "..";@@ -201,9 +200,8 @@ class Merger {break;}case "attachment": {- if (isDeleted(remoteItem)) {- return this.db.attachments.merge(undefined, remoteItem);- }+ if (isDeleted(remoteItem)) return remoteItem;+if (remoteItem.type !== "attachment") return;const localAttachment = this.db.attachments.attachment(@@ -213,7 +211,6 @@ class Merger {localAttachment &&localAttachment.dateUploaded !== remoteItem.dateUploaded) {- const noteIds = localAttachment.noteIds.slice();const isRemoved = await this.db.attachments.remove(localAttachment.metadata.hash,true@@ -222,9 +219,8 @@ class Merger {throw new Error("Conflict could not be resolved in one of the attachments.");- remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds);}- return this.db.attachments.merge(undefined, remoteItem);+ return remoteItem;}}}diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.tsindex 1dcfda2b9..e95e0ada9 100644--- a/packages/core/src/collections/attachments.ts+++ b/packages/core/src/collections/attachments.ts@@ -19,11 +19,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { ICollection } from "./collection";import { getId } from "../utils/id";-import { deleteItem, hasItem } from "../utils/array";+import { hasItem } from "../utils/array";import { EV, EVENTS } from "../common";import dataurl from "../utils/dataurl";import dayjs from "dayjs";-import { set } from "../utils/set";import {getFileNameWithExtension,isImage,@@ -32,12 +31,7 @@ import {import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";import { CachedCollection } from "../database/cached-collection";import { Output } from "../interfaces";-import {- Attachment,- AttachmentMetadata,- MaybeDeletedItem,- isDeleted-} from "../types";+import { Attachment, AttachmentMetadata, isDeleted } from "../types";import Database from "../api";export class Attachments implements ICollection {@@ -108,27 +102,6 @@ export class Attachments implements ICollection {await this.collection.init();}- merge(- localAttachment: MaybeDeletedItem<Attachment> | undefined,- remoteAttachment: MaybeDeletedItem<Attachment>- ) {- if (isDeleted(remoteAttachment)) return remoteAttachment;-- if (- localAttachment &&- !isDeleted(localAttachment) &&- localAttachment.noteIds- ) {- remoteAttachment.noteIds = set.union(- remoteAttachment.noteIds,- localAttachment.noteIds- );- remoteAttachment.remote = false;- }-- return remoteAttachment;- }-async add(item: Partial<Omit<Attachment, "key" | "metadata"> & {@@ -145,7 +118,6 @@ export class Attachments implements ICollection {(a) => a.metadata.hash === item.metadata?.hash);const id = oldAttachment?.id || getId();- const noteIds = set.union(oldAttachment?.noteIds || [], item.noteIds || []);const encryptedKey = item.key? await this.encryptKey(item.key)@@ -154,7 +126,6 @@ export class Attachments implements ICollection {...oldAttachment,...oldAttachment?.metadata,...item,- noteIds,key: encryptedKey};@@ -193,7 +164,6 @@ export class Attachments implements ICollection {return this.collection.add({type: "attachment",id,- noteIds,iv,salt,length,@@ -226,21 +196,11 @@ export class Attachments implements ICollection {return JSON.parse(plainData);}- async delete(hashOrId: string, noteId: string) {- const attachment = this.attachment(hashOrId);- if (!attachment || !deleteItem(attachment.noteIds, noteId)) return;- if (!attachment.noteIds.length) {- attachment.dateDeleted = Date.now();- EV.publish(EVENTS.attachmentDeleted, attachment);- }- return await this.collection.update(attachment);- }-async remove(hashOrId: string, localOnly: boolean) {const attachment = this.attachment(hashOrId);if (!attachment || !attachment.metadata) return false;- if (!localOnly && !(await this._canDetach(attachment)))+ if (!localOnly && !(await this.canDetach(attachment)))throw new Error("This attachment is inside a locked note.");if (@@ -261,38 +221,37 @@ export class Attachments implements ICollection {}async detach(attachment: Attachment) {- for (const noteId of attachment.noteIds) {- const note = this.db.notes.note(noteId);- if (!note || !note.data.contentId) continue;- await this.db.content.removeAttachments(note.data.contentId, [+ for (const note of this.db.relations.from(attachment, "note").resolved()) {+ if (!note || !note.contentId) continue;+ await this.db.content.removeAttachments(note.contentId, [attachment.metadata.hash]);}}- async _canDetach(attachment: Attachment) {- for (const noteId of attachment.noteIds) {- const note = this.db.notes?.note(noteId);- if (note && note.data.locked) return false;- }-- return true;+ private async canDetach(attachment: Attachment) {+ return this.db.relations+ .from(attachment, "note")+ .resolved()+ .every((note) => !note.locked);}ofNote(noteId: string,- type: "files" | "images" | "webclips" | "all"+ ...types: ("files" | "images" | "webclips" | "all")[]): Attachment[] {- let attachments: Attachment[] = [];+ const noteAttachments = this.db.relations+ .from({ type: "note", id: noteId }, "attachment")+ .resolved();- if (type === "files") attachments = this.files;- else if (type === "images") attachments = this.images;- else if (type === "webclips") attachments = this.webclips;- else if (type === "all") attachments = this.all;+ if (types.includes("all")) return noteAttachments;- return attachments.filter((attachment) =>- hasItem(attachment.noteIds, noteId)- );+ return noteAttachments.filter((a) => {+ if (isImage(a.metadata.type) && types.includes("images")) return true;+ else if (isWebClip(a.metadata.type) && types.includes("webclips"))+ return true;+ else if (types.includes("files")) return true;+ });}exists(hash: string) {@@ -387,12 +346,11 @@ export class Attachments implements ICollection {}async downloadMedia(noteId: string, hashesToLoad?: string[]) {- const attachments = this.media.filter(- (attachment) =>- !!attachment.metadata &&- hasItem(attachment.noteIds, noteId) &&- (!hashesToLoad || hasItem(hashesToLoad, attachment.metadata?.hash))- );+ let attachments = this.ofNote(noteId, "images", "webclips");+ if (hashesToLoad)+ attachments = attachments.filter((a) =>+ hasItem(hashesToLoad, a.metadata.hash)+ );await this.db.fs().queueDownloads(attachments.map((a) => ({diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex efd437b6f..10b8fcc50 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -20,7 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { ICollection } from "./collection";import { getId } from "../utils/id";import { getContentFromData } from "../content-types";-import { hasItem } from "../utils/array";import { ResolveHashes } from "../content-types/tiptap";import { isCipher } from "../database/crypto";import {@@ -238,16 +237,15 @@ export class Content implements ICollection {async extractAttachments(contentItem: UnencryptedContentItem) {if (contentItem.localOnly) return contentItem;- const allAttachments = this.db.attachments?.all;const content = getContentFromData(contentItem.type, contentItem.data);if (!content) return contentItem;const { data, hashes } = await content.extractAttachments(this.db.attachments.save);- const noteAttachments = allAttachments.filter((attachment) =>- hasItem(attachment.noteIds, contentItem.noteId)- );+ const noteAttachments = this.db.relations+ .from({ type: "note", id: contentItem.noteId }, "attachment")+ .resolved();const toDelete = noteAttachments.filter((attachment) => {return hashes.every((hash) => hash !== attachment.metadata.hash);@@ -258,17 +256,25 @@ export class Content implements ICollection {});for (const attachment of toDelete) {- await this.db.attachments.delete(- attachment.metadata.hash,- contentItem.noteId+ await this.db.relations.unlink(+ {+ id: contentItem.noteId,+ type: "note"+ },+ attachment);}for (const hash of toAdd) {- await this.db.attachments.add({- noteIds: [contentItem.noteId],- metadata: { hash }- });+ const attachment = this.db.attachments.attachment(hash);+ if (!attachment) continue;+ await this.db.relations.add(+ {+ id: contentItem.noteId,+ type: "note"+ },+ attachment+ );}if (toAdd.length > 0) {diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex dcbb2ff57..22f5ded5c 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -302,11 +302,7 @@ export class Notes implements ICollection {await this.db.relations.unlinkAll(item, "tag");await this.db.relations.unlinkAll(item, "color");-- const attachments = this.db.attachments.ofNote(itemData.id, "all");- for (const attachment of attachments) {- await this.db.attachments.delete(attachment.metadata.hash, itemData.id);- }+ await this.db.relations.unlinkAll(item, "attachment");if (moveToTrash && !isTrashItem(itemData))await this.db.trash.add(itemData);diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex 9ba70a8c2..2366e56e0 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -168,6 +168,12 @@ export class Relations implements ICollection {item = notebook.data;break;}+ case "attachment": {+ const attachment = this.db.attachments.attachment(reference.id);+ if (!attachment) continue;+ item = attachment;+ break;+ }}if (item) items.push(item);}diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.tsindex 1c6a13726..410a6202e 100644--- a/packages/core/src/database/cached-collection.ts+++ b/packages/core/src/database/cached-collection.ts@@ -20,9 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { IndexedCollection } from "./indexed-collection";import MapStub from "../utils/map";import {- BaseItem,CollectionType,Collections,+ ItemMap,MaybeDeletedItem,isDeleted} from "../types";@@ -32,7 +32,7 @@ import { chunkedIterate } from "../utils/array";export class CachedCollection<TCollectionType extends CollectionType,- T extends BaseItem<Collections[TCollectionType]>+ T extends ItemMap[Collections[TCollectionType]]> {private collection: IndexedCollection<TCollectionType, T>;private cache = new Map<string, MaybeDeletedItem<T>>();diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex 8bc856542..f26a0f1a9 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -249,6 +249,16 @@ const migrations: Migration[] = [delete item.tags;delete item.color;return true;+ },+ attachment: async (item, db) => {+ for (const noteId of item.noteIds || []) {+ await db.relations.add(+ { type: "attachment", id: item.id },+ { type: "note", id: noteId }+ );+ }+ delete item.noteIds;+ return true;}}},diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex c7fd1bfb9..be4f54627 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -178,7 +178,6 @@ export type AttachmentMetadata = {};export interface Attachment extends BaseItem<"attachment"> {- noteIds: string[];iv: string;salt: string;length: number;@@ -189,6 +188,11 @@ export interface Attachment extends BaseItem<"attachment"> {dateUploaded?: number;failed?: string;dateDeleted?: number;++ /**+ * @deprecated only kept here for migration purposes+ */+ noteIds?: string[];}export interface Color extends BaseItem<"color"> {