Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: optimize attachment caching in offline mode

Abdullah Attafbcb8675ca5f
diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts
index 3f911ccf0..6350bfcb7 100644
--- a/packages/core/src/api/sync/index.ts
+++ b/packages/core/src/api/sync/index.ts
@@ -121,6 +121,8 @@ class Sync {
syncConnectionMutex = new Mutex();
connection?: signalr.HubConnection;
devices = new SyncDevices(this.db.kv, this.db.tokenManager);
+ private conflictedNoteIds: string[] = [];
+ private uncachedAttachments: DownloadableFile[] = [];
constructor(private readonly db: Database) {
EV.subscribe(EVENTS.userLoggedOut, async () => {
@@ -130,7 +132,7 @@ class Sync {
}
async start(options: SyncOptions) {
- this.createConnection();
+ this.createConnection(options);
if (!this.connection) return;
if (!(await checkSyncStatus(SYNC_CHECK_IDS.sync))) {
@@ -151,7 +153,7 @@ class Sync {
this.logger.info("Initialized sync", { deviceId });
if (options.type === "fetch" || options.type === "full") {
- await this.fetch(deviceId);
+ await this.fetch(deviceId, options);
this.logger.info("Data fetched");
}
@@ -188,24 +190,31 @@ class Sync {
return { deviceId };
}
- async fetch(deviceId: string) {
+ async fetch(deviceId: string, options: SyncOptions) {
await this.checkConnection();
await this.connection?.invoke("RequestFetch", deviceId);
- await this.db
- .sql()
- .updateTable("notes")
- .where("id", "in", (eb) =>
- eb
- .selectFrom("content")
- .select("noteId as id")
- .where("conflicted", "is not", null)
- .where("conflicted", "!=", false)
- .$castTo<string | null>()
- )
- .set({ conflicted: true })
- .execute();
+ if (this.conflictedNoteIds.length > 0) {
+ await this.db
+ .sql()
+ .updateTable("notes")
+ .where("id", "in", this.conflictedNoteIds)
+ .set({ conflicted: true })
+ .execute();
+ this.conflictedNoteIds = [];
+ }
+
+ if (this.uncachedAttachments.length > 0 && options.offlineMode) {
+ await this.db
+ .fs()
+ .queueDownloads(
+ this.uncachedAttachments,
+ "download-uncached-attachments",
+ { readOnDownload: false }
+ );
+ this.uncachedAttachments = [];
+ }
}
async send(deviceId: string, isForceSync?: boolean) {
@@ -246,21 +255,6 @@ class Sync {
this.logger.info("Stopping sync");
await this.db.setLastSynced(Date.now());
this.db.eventManager.publish(EVENTS.syncCompleted);
-
- if (options.offlineMode) {
- const attachments = await this.db.attachments.linked
- .fields(["attachments.id", "attachments.hash", "attachments.chunkSize"])
- .items();
-
- await this.db.fs().queueDownloads(
- attachments.map((a) => ({
- filename: a.hash,
- chunkSize: a.chunkSize
- })),
- "download-all-attachments",
- { readOnDownload: false }
- );
- }
}
async cancel() {
@@ -291,7 +285,11 @@ class Sync {
this.db.eventManager.publish(EVENTS.databaseSyncRequested, true, false);
}
- async processChunk(chunk: SyncTransferItem, key: SerializedKey) {
+ async processChunk(
+ chunk: SyncTransferItem,
+ key: SerializedKey,
+ options: SyncOptions
+ ) {
const itemType = chunk.type;
const decrypted = await this.db.storage().decryptMulti(key, chunk.items);
@@ -332,12 +330,24 @@ class Sync {
);
}
- if ((itemType === "note" || itemType === "content") && items.length > 0) {
+ if (itemType === "note" || itemType === "content") {
items.forEach((item) =>
this.db.eventManager.publish(EVENTS.syncItemMerged, item)
);
+
+ for (const item of items)
+ if (!item?.deleted && item?.type === "tiptap" && !item.conflicted)
+ this.conflictedNoteIds.push(item.noteId);
}
+ if (itemType === "attachment" && options.offlineMode)
+ for (const item of items)
+ if (!item?.deleted && item?.type === "attachment")
+ this.uncachedAttachments.push({
+ filename: item.hash,
+ chunkSize: item.chunkSize
+ });
+
await collection.put(items as any);
}
@@ -346,7 +356,7 @@ class Sync {
return (await this.connection?.invoke("PushItems", deviceId, item)) === 1;
}
- private createConnection() {
+ private createConnection(options: SyncOptions) {
if (this.connection) return;
const tokenManager = new TokenManager(this.db.kv);
@@ -410,7 +420,7 @@ class Sync {
const key = await this.getKey();
if (!key) return false;
- await this.processChunk(chunk, key);
+ await this.processChunk(chunk, key, options);
sendSyncProgressEvent(this.db.eventManager, `download`, chunk.count);
diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts
index 4e8912f24..9094431a4 100644
--- a/packages/core/src/collections/attachments.ts
+++ b/packages/core/src/collections/attachments.ts
@@ -355,6 +355,20 @@ export class Attachments implements ICollection {
});
}
+ async cacheAttachments(attachments?: FilteredSelector<Attachment>) {
+ const items = await (attachments || this.db.attachments.linked)
+ .fields(["attachments.id", "attachments.hash", "attachments.chunkSize"])
+ .items();
+ await this.db.fs().queueDownloads(
+ items.map((a) => ({
+ filename: a.hash,
+ chunkSize: a.chunkSize
+ })),
+ "download-all-attachments",
+ { readOnDownload: false }
+ );
+ }
+
async save(
data: string,
mimeType: string,
diff --git a/packages/core/src/database/fs.ts b/packages/core/src/database/fs.ts
index 042881b63..eefd85c26 100644
--- a/packages/core/src/database/fs.ts
+++ b/packages/core/src/database/fs.ts
@@ -56,6 +56,10 @@ export class FileStorage {
groupId: string,
eventData?: Record<string, unknown>
) {
+ const newFiles = await this.fs.bulkExists(files.map((f) => f.filename));
+ files = files.filter((f) => newFiles.includes(f.filename));
+ if (files.length <= 0) return;
+
let current = 0;
const token = await this.tokenManager.getAccessToken();
const total = files.length;
diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.ts
index e0ea6fa09..4604a4268 100644
--- a/packages/core/src/interfaces.ts
+++ b/packages/core/src/interfaces.ts
@@ -108,6 +108,7 @@ export interface IFileStorage {
requestOptions?: RequestOptions
): Promise<boolean>;
exists(filename: string): Promise<boolean>;
+ bulkExists(filenames: string[]): Promise<string[]>;
getUploadedFileSize(filename: string): Promise<number>;
clearFileStorage(): Promise<void>;
hashBase64(data: string): Promise<{ hash: string; type: string }>;