π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
refractor(locker):rename-remote-cache-and-split-remote-read
Aswin Asok1efd45b7b19d
diff --git a/web/apps/locker/src/services/download.ts b/web/apps/locker/src/services/download.tsindex 8def4e4e7f..d4564de372 100644--- a/web/apps/locker/src/services/download.ts+++ b/web/apps/locker/src/services/download.ts@@ -3,8 +3,8 @@ import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";import log from "ente-base/log";import { customAPIOrigin } from "ente-base/origins";import { createStreamDecryptor, decryptBox } from "ente-locker-wasm";-import { getEncryptedFileRecord, getLockerCacheSnapshot } from "./remote-cache";-import { decryptCollectionKey } from "./remote-read";+import { getEncryptedFileRecord, getLockerCacheSnapshot } from "./locker-cache";+import { decryptCollectionKey } from "./sync/decrypt";const DOWNLOAD_URL_REVOKE_DELAY_MS = 30_000;diff --git a/web/apps/locker/src/services/file-links.ts b/web/apps/locker/src/services/file-links.tsindex c81166ba98..9d8c06ef3d 100644--- a/web/apps/locker/src/services/file-links.ts+++ b/web/apps/locker/src/services/file-links.ts@@ -3,8 +3,8 @@ import { apiURL } from "ente-base/origins";import { openFileLinkSecret, prepareFileLink } from "ente-locker-wasm";import { z } from "zod";import { ensureAuthenticatedSession } from "./authenticated-session";-import { getEncryptedFileRecord } from "./remote-cache";-import { decryptFileKeyForRecord } from "./remote-read";+import { getEncryptedFileRecord } from "./locker-cache";+import { decryptFileKeyForRecord } from "./sync/decrypt";const RemoteFileShareLink = z.object({linkID: z.union([z.string(), z.number().transform(String)]),diff --git a/web/apps/locker/src/services/remote-cache.ts b/web/apps/locker/src/services/locker-cache.tssimilarity index 100%rename from web/apps/locker/src/services/remote-cache.tsrename to web/apps/locker/src/services/locker-cache.tsdiff --git a/web/apps/locker/src/services/locker-db.ts b/web/apps/locker/src/services/locker-db.tsindex 58db0036c0..8af459a989 100644--- a/web/apps/locker/src/services/locker-db.ts+++ b/web/apps/locker/src/services/locker-db.ts@@ -4,7 +4,7 @@ import { deleteDB, openDB, type DBSchema, type IDBPDatabase } from "idb";import type {EncryptedCollectionRecord,EncryptedFileRecord,-} from "./remote-cache";+} from "./locker-cache";const LOCKER_DB_NAME_PREFIX = "ente-locker";const LOCKER_DB_VERSION = 2;diff --git a/web/apps/locker/src/services/logout.ts b/web/apps/locker/src/services/logout.tsindex a929137201..d40da6387e 100644--- a/web/apps/locker/src/services/logout.ts+++ b/web/apps/locker/src/services/logout.ts@@ -2,8 +2,8 @@ import { savedLocalUser } from "ente-accounts/services/accounts-db";import { accountLogout } from "ente-accounts/services/logout";import log from "ente-base/log";import { clearAuthenticatedSession } from "./authenticated-session";+import { clearLockerCache } from "./locker-cache";import { clearLockerDB } from "./locker-db";-import { clearLockerCache } from "./remote-cache";export const lockerLogout = async () => {const ignoreError = (label: string, error: unknown) =>diff --git a/web/apps/locker/src/services/remote-collection-sharing.ts b/web/apps/locker/src/services/remote-collection-sharing.tsindex 466cf2381c..2790447703 100644--- a/web/apps/locker/src/services/remote-collection-sharing.ts+++ b/web/apps/locker/src/services/remote-collection-sharing.ts@@ -7,12 +7,12 @@ import { z } from "zod";import {getCollectionRecord,updateCollectionShareesInCache,-} from "./remote-cache";-import { decryptCollectionKey } from "./remote-read";+} from "./locker-cache";import {RemoteCollectionUserSchema,toLockerCollectionParticipant,} from "./remote-types";+import { decryptCollectionKey } from "./sync/decrypt";const RemoteShareesResponse = z.object({sharees: z.array(RemoteCollectionUserSchema),diff --git a/web/apps/locker/src/services/remote-collections.ts b/web/apps/locker/src/services/remote-collections.tsindex 58a7e57fed..4afeaf6999 100644--- a/web/apps/locker/src/services/remote-collections.ts+++ b/web/apps/locker/src/services/remote-collections.ts@@ -2,9 +2,10 @@ import { ensureLocalUser } from "ente-accounts/services/user";import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";import { apiURL } from "ente-base/origins";import { encryptBox, encryptBoxBytes, generateKey } from "ente-locker-wasm";-import { findCollectionByType, getCollectionRecord } from "./remote-cache";-import { decryptCollectionKey, fetchLockerData } from "./remote-read";+import { findCollectionByType, getCollectionRecord } from "./locker-cache";import { RemoteCollectionCreateResponseSchema } from "./remote-types";+import { decryptCollectionKey } from "./sync/decrypt";+import { fetchLockerData } from "./sync/sync";const ensureCollectionWithType = async (name: string,diff --git a/web/apps/locker/src/services/remote.ts b/web/apps/locker/src/services/remote.tsindex 585c865df0..923d530383 100644--- a/web/apps/locker/src/services/remote.ts+++ b/web/apps/locker/src/services/remote.ts@@ -16,7 +16,7 @@ import {getCollectionRecord,getEncryptedFileRecord,updateCachedPubMagicMetadata,-} from "./remote-cache";+} from "./locker-cache";import {deleteCollectionKeepingFilesWithDeps,type EncryptedCollectionFileItem,@@ -27,17 +27,17 @@ import {ensureFavoritesCollection,ensureUncategorizedCollection,} from "./remote-collections";-import {- decryptCollectionKey,- fetchLockerTrash,- loadPersistedLockerState,- syncLockerState,-} from "./remote-read";import { RemoteIDResponseSchema } from "./remote-types";import {type LockerUploadProgress,uploadLockerFileWithDeps,} from "./remote-uploads";+import { decryptCollectionKey } from "./sync/decrypt";+import {+ fetchLockerTrash,+ loadPersistedLockerState,+ syncLockerState,+} from "./sync/sync";export {fetchCollectionSharees,shareCollection,diff --git a/web/apps/locker/src/services/remote-read.ts b/web/apps/locker/src/services/sync/decrypt.tssimilarity index 56%rename from web/apps/locker/src/services/remote-read.tsrename to web/apps/locker/src/services/sync/decrypt.tsindex c9c0229873..0fcaf4b696 100644--- a/web/apps/locker/src/services/remote-read.ts+++ b/web/apps/locker/src/services/sync/decrypt.ts@@ -3,9 +3,7 @@ import type {LockerCollectionParticipant,LockerItem,} from "@/types";-import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";import log from "ente-base/log";-import { apiURL } from "ente-base/origins";import {decryptBox,decryptBoxBytes,@@ -13,124 +11,19 @@ import {encryptBoxBytes,openCollectionKey,} from "ente-locker-wasm";-import { z } from "zod";-import { ensureAuthenticatedSession } from "./authenticated-session";-import { fromInfoTypeWireValue } from "./info-type-wire";-import {- type StoredTrashFileRecord,- deleteCollectionSinceTime,- deleteFileRecords,- deleteFileRecordsForCollection,- deleteTrashFileRecords,- loadLockerSnapshotFromDB,- saveCollectionRecords,- saveCollectionSinceTime,- saveCollectionsSinceTime,- saveFileRecords,- saveTrashFileRecords,- saveTrashSinceTime,-} from "./locker-db";+import type { z } from "zod";+import { ensureAuthenticatedSession } from "../authenticated-session";+import { fromInfoTypeWireValue } from "../info-type-wire";import {type EncryptedCollectionRecord,type EncryptedFileRecord,type LockerCollectionPayload,type LockerEncryptedCache,getLockerCacheSnapshot,- replaceLockerCache,- setEncryptedFileRecord,-} from "./remote-cache";-import {- RemoteCollectionUserSchema,- toLockerCollectionParticipant,-} from "./remote-types";--const RemoteMagicMetadata = z.object({- version: z.number(),- count: z.number().optional(),- data: z.string(),- header: z.string(),-});--const RemoteCollection = z.object({- id: z.number(),- owner: RemoteCollectionUserSchema,- encryptedKey: z.string(),- keyDecryptionNonce: z.string().nullish(),- encryptedName: z.string().nullish(),- nameDecryptionNonce: z.string().nullish(),- name: z.string().nullish(),- type: z.string(),- sharees: z.array(RemoteCollectionUserSchema).nullish(),- publicURLs: z.array(z.unknown()).nullish(),- updationTime: z.number(),- isDeleted: z.boolean().nullish(),- magicMetadata: RemoteMagicMetadata.nullish(),- pubMagicMetadata: RemoteMagicMetadata.nullish(),- sharedMagicMetadata: RemoteMagicMetadata.nullish(),-});--type RemoteCollection = z.infer<typeof RemoteCollection>;--const CollectionsResponse = z.object({- collections: z.array(RemoteCollection),-});--const RemoteEncryptedMetadata = z.object({- encryptedData: z.string(),- decryptionHeader: z.string(),-});--const RemoteFileObjectAttributes = z.object({ decryptionHeader: z.string() });--const RemoteFile = z.object({- id: z.number(),- collectionID: z.number(),- ownerID: z.number().optional(),- encryptedKey: z.string(),- keyDecryptionNonce: z.string(),- file: RemoteFileObjectAttributes,- thumbnail: RemoteFileObjectAttributes.optional(),- metadata: RemoteEncryptedMetadata,- magicMetadata: z- .object({- version: z.number(),- count: z.number().optional(),- data: z.string(),- header: z.string(),- })- .nullish(),- pubMagicMetadata: z- .object({- version: z.number(),- count: z.number().optional(),- data: z.string(),- header: z.string(),- })- .nullish(),- updationTime: z.number(),- isDeleted: z.boolean(),- info: z.object({ fileSize: z.number().optional() }).nullish(),-});--type RemoteFile = z.infer<typeof RemoteFile>;--const FileDiffResponse = z.object({- diff: z.array(RemoteFile),- hasMore: z.boolean(),-});--const RemoteTrashItem = z.object({- file: RemoteFile,- isDeleted: z.boolean(),- isRestored: z.boolean(),- updatedAt: z.number(),- deleteBy: z.number(),-});--const TrashDiffResponse = z.object({- diff: z.array(RemoteTrashItem),- hasMore: z.boolean(),-});+} from "../locker-cache";+import type { StoredTrashFileRecord } from "../locker-db";+import { toLockerCollectionParticipant } from "../remote-types";+import type { RemoteCollection, RemoteFile, RemoteTrashItem } from "./schemas";interface DecryptAllDataResult {collections: LockerCollection[];@@ -138,24 +31,13 @@ interface DecryptAllDataResult {totalCollectionCount: number;}-interface LockerTrashData {+export interface LockerTrashData {items: LockerItem[];lastUpdatedAt: number;}-interface LockerHydratedState {- collections: LockerCollection[];- trashItems: LockerItem[];- trashLastUpdatedAt: number;- collectionsSinceTime: number;- trashSinceTime: number;-}--interface LockerPersistedState extends LockerHydratedState {- hasPersistedState: boolean;-}-const COLLECTION_PAYLOAD_VERSION = 1;+const collectionTextDecoder = new TextDecoder();const describeCryptoError = (error: unknown) =>@@ -171,7 +53,9 @@ const toEpochMicroseconds = (timestamp: unknown) => {return timestamp < 100_000_000_000_000 ? timestamp * 1000 : timestamp;};-const buildEncryptedFileRecord = (file: RemoteFile): EncryptedFileRecord => ({+export const buildEncryptedFileRecord = (+ file: RemoteFile,+): EncryptedFileRecord => ({id: file.id,collectionID: file.collectionID,ownerID: file.ownerID ?? undefined,@@ -338,7 +222,7 @@ const encryptCollectionPayload = async (};};-const toEncryptedCollectionRecord = (+export const toEncryptedCollectionRecord = (collection: RemoteCollection,): Promise<EncryptedCollectionRecord> => {const record: EncryptedCollectionRecord = {@@ -408,71 +292,7 @@ const decryptCollectionDetails = async (return { owner: payload.owner, sharees: payload.sharees, name };};-const buildLockerCache = (- collections: Map<number, EncryptedCollectionRecord>,- files: EncryptedFileRecord[],- trashFiles: StoredTrashFileRecord[],-): LockerEncryptedCache => {- const nextFiles = new Map<number, Map<number, EncryptedFileRecord>>();- for (const record of files) {- setEncryptedFileRecord(nextFiles, record);- }- for (const record of trashFiles) {- setEncryptedFileRecord(nextFiles, record);- }-- return { collections, files: nextFiles };-};--const fetchEncryptedCollections = async (sinceTime: number) => {- const response = await fetch(- await apiURL("/collections/v2", { sinceTime }),- { headers: await authenticatedRequestHeaders() },- );- ensureOk(response);- const { collections } = CollectionsResponse.parse(await response.json());- return collections;-};--interface CollectionFileDiff {- recordsToSave: EncryptedFileRecord[];- fileKeysToDelete: [number, number][];- sinceTime: number;-}--const fetchEncryptedFilesForCollection = async (- collectionID: number,- initialSinceTime: number,-): Promise<CollectionFileDiff> => {- let sinceTime = initialSinceTime;- let hasMore = true;- const recordsToSave: EncryptedFileRecord[] = [];- const fileKeysToDelete: [number, number][] = [];-- while (hasMore) {- const response = await fetch(- await apiURL("/collections/v2/diff", { collectionID, sinceTime }),- { headers: await authenticatedRequestHeaders() },- );- ensureOk(response);- const parsed = FileDiffResponse.parse(await response.json());-- for (const file of parsed.diff) {- sinceTime = Math.max(sinceTime, file.updationTime);- if (file.isDeleted) {- fileKeysToDelete.push([file.id, collectionID]);- } else {- recordsToSave.push(buildEncryptedFileRecord(file));- }- }-- hasMore = parsed.hasMore;- }-- return { recordsToSave, fileKeysToDelete, sinceTime };-};--const decryptStoredTrash = async (+export const decryptStoredTrash = async (cache: LockerEncryptedCache,trashFiles: StoredTrashFileRecord[],lastUpdatedAt: number,@@ -510,7 +330,7 @@ const decryptStoredTrash = async (return { items: trashItems, lastUpdatedAt };};-const buildStoredTrashFileRecord = (+export const buildStoredTrashFileRecord = (entry: z.infer<typeof RemoteTrashItem>,): StoredTrashFileRecord => ({...buildEncryptedFileRecord(entry.file),@@ -645,7 +465,7 @@ const decryptFileToLockerItem = async (}};-const decryptAllData = async (+export const decryptAllData = async (cache: LockerEncryptedCache,): Promise<DecryptAllDataResult> => {const activeCollectionRecords = [...cache.collections.values()].filter(@@ -715,219 +535,3 @@ const decryptAllData = async (result.sort((a, b) => a.name.localeCompare(b.name));return { collections: result, failedCollectionIDs, totalCollectionCount };};--const withoutFailedCollections = (- cache: LockerEncryptedCache,- failedCollectionIDs: number[],-): LockerEncryptedCache => {- if (failedCollectionIDs.length === 0) {- return cache;- }-- const failedCollectionIDSet = new Set(failedCollectionIDs);- return {- collections: new Map(- [...cache.collections.entries()].filter(- ([collectionID]) => !failedCollectionIDSet.has(collectionID),- ),- ),- files: new Map(- [...cache.files.entries()]- .map(- ([fileID, records]): [- number,- Map<number, EncryptedFileRecord>,- ] => [- fileID,- new Map(- [...records.entries()].filter(- ([collectionID]) =>- !failedCollectionIDSet.has(collectionID),- ),- ),- ],- )- .filter(([, records]) => records.size > 0),- ),- };-};--const hydrateLockerState = async (- collections: Map<number, EncryptedCollectionRecord>,- files: EncryptedFileRecord[],- trashFiles: StoredTrashFileRecord[],- trashLastUpdatedAt: number,-): Promise<LockerHydratedState> => {- const activeCache = buildLockerCache(collections, files, []);-- const decrypted = await decryptAllData(activeCache);- if (- decrypted.totalCollectionCount > 0 &&- decrypted.collections.length === 0- ) {- throw new Error(- `Failed to decrypt all ${decrypted.totalCollectionCount} locker collections`,- );- }-- const hydratedCache = withoutFailedCollections(- buildLockerCache(collections, files, trashFiles),- decrypted.failedCollectionIDs,- );- replaceLockerCache(hydratedCache);-- if (decrypted.failedCollectionIDs.length > 0) {- log.warn(- `Decrypted ${decrypted.collections.length}/${decrypted.totalCollectionCount} locker collections`,- );- }-- const trash = await decryptStoredTrash(- hydratedCache,- trashFiles,- trashLastUpdatedAt,- );-- return {- collections: decrypted.collections,- trashItems: trash.items,- trashLastUpdatedAt: trash.lastUpdatedAt,- collectionsSinceTime: 0,- trashSinceTime: 0,- };-};--export const loadPersistedLockerState =- async (): Promise<LockerPersistedState> => {- const snapshot = await loadLockerSnapshotFromDB();- const hydrated = await hydrateLockerState(- snapshot.collections,- snapshot.files,- snapshot.trashFiles,- snapshot.trashSinceTime,- );-- return {- ...hydrated,- collectionsSinceTime: snapshot.collectionsSinceTime,- trashSinceTime: snapshot.trashSinceTime,- hasPersistedState: snapshot.hasPersistedState,- };- };--export const syncLockerState = async (): Promise<LockerHydratedState> => {- const snapshot = await loadLockerSnapshotFromDB();- const collectionChanges = await fetchEncryptedCollections(- snapshot.collectionsSinceTime,- );-- let latestCollectionsSinceTime = snapshot.collectionsSinceTime;- const changedCollections: EncryptedCollectionRecord[] = [];- const deletedCollectionIDs: number[] = [];-- for (const change of collectionChanges) {- latestCollectionsSinceTime = Math.max(- latestCollectionsSinceTime,- change.updationTime,- );- const record = await toEncryptedCollectionRecord(change);- changedCollections.push(record);- if (record.isDeleted) {- deletedCollectionIDs.push(record.id);- }- }-- if (changedCollections.length > 0) {- await saveCollectionRecords(changedCollections);- }- for (const collectionID of deletedCollectionIDs) {- await deleteFileRecordsForCollection(collectionID);- await deleteCollectionSinceTime(collectionID);- }- await saveCollectionsSinceTime(latestCollectionsSinceTime);-- const postCollectionSnapshot = await loadLockerSnapshotFromDB();- for (const collection of postCollectionSnapshot.collections.values()) {- if (collection.isDeleted) {- continue;- }-- const savedSinceTime =- postCollectionSnapshot.collectionSinceTimeByID.get(collection.id) ??- 0;- if (savedSinceTime >= collection.updationTime) {- continue;- }-- const diff = await fetchEncryptedFilesForCollection(- collection.id,- savedSinceTime,- );- if (diff.recordsToSave.length > 0) {- await saveFileRecords(diff.recordsToSave);- }- if (diff.fileKeysToDelete.length > 0) {- await deleteFileRecords(diff.fileKeysToDelete);- }-- await saveCollectionSinceTime(- collection.id,- Math.max(diff.sinceTime, collection.updationTime),- );- }-- let trashSinceTime = postCollectionSnapshot.trashSinceTime;- let hasMore = true;- while (hasMore) {- const response = await fetch(- await apiURL("/trash/v2/diff", { sinceTime: trashSinceTime }),- { headers: await authenticatedRequestHeaders() },- );- ensureOk(response);- const parsed = TrashDiffResponse.parse(await response.json());-- const recordsToSave: StoredTrashFileRecord[] = [];- const fileIDsToDelete: number[] = [];- for (const entry of parsed.diff) {- trashSinceTime = Math.max(trashSinceTime, entry.updatedAt);- if (entry.isDeleted || entry.isRestored) {- fileIDsToDelete.push(entry.file.id);- } else {- recordsToSave.push(buildStoredTrashFileRecord(entry));- }- }-- if (recordsToSave.length > 0) {- await saveTrashFileRecords(recordsToSave);- }- if (fileIDsToDelete.length > 0) {- await deleteTrashFileRecords(fileIDsToDelete);- }-- hasMore = parsed.hasMore;- }-- await saveTrashSinceTime(trashSinceTime);-- const nextSnapshot = await loadLockerSnapshotFromDB();- const hydrated = await hydrateLockerState(- nextSnapshot.collections,- nextSnapshot.files,- nextSnapshot.trashFiles,- nextSnapshot.trashSinceTime,- );-- return {- ...hydrated,- collectionsSinceTime: nextSnapshot.collectionsSinceTime,- trashSinceTime: nextSnapshot.trashSinceTime,- };-};--export const fetchLockerData = async (): Promise<LockerCollection[]> =>- (await syncLockerState()).collections;--export const fetchLockerTrash = async (): Promise<LockerTrashData> => {- const state = await syncLockerState();- return { items: state.trashItems, lastUpdatedAt: state.trashLastUpdatedAt };-};diff --git a/web/apps/locker/src/services/sync/schemas.ts b/web/apps/locker/src/services/sync/schemas.tsnew file mode 100644index 0000000000..d6caf300c8--- /dev/null+++ b/web/apps/locker/src/services/sync/schemas.ts@@ -0,0 +1,90 @@+import { z } from "zod";+import { RemoteCollectionUserSchema } from "../remote-types";++const RemoteMagicMetadata = z.object({+ version: z.number(),+ count: z.number().optional(),+ data: z.string(),+ header: z.string(),+});++export const RemoteCollection = z.object({+ id: z.number(),+ owner: RemoteCollectionUserSchema,+ encryptedKey: z.string(),+ keyDecryptionNonce: z.string().nullish(),+ encryptedName: z.string().nullish(),+ nameDecryptionNonce: z.string().nullish(),+ name: z.string().nullish(),+ type: z.string(),+ sharees: z.array(RemoteCollectionUserSchema).nullish(),+ publicURLs: z.array(z.unknown()).nullish(),+ updationTime: z.number(),+ isDeleted: z.boolean().nullish(),+ magicMetadata: RemoteMagicMetadata.nullish(),+ pubMagicMetadata: RemoteMagicMetadata.nullish(),+ sharedMagicMetadata: RemoteMagicMetadata.nullish(),+});++export type RemoteCollection = z.infer<typeof RemoteCollection>;++export const CollectionsResponse = z.object({+ collections: z.array(RemoteCollection),+});++const RemoteEncryptedMetadata = z.object({+ encryptedData: z.string(),+ decryptionHeader: z.string(),+});++const RemoteFileObjectAttributes = z.object({ decryptionHeader: z.string() });++export const RemoteFile = z.object({+ id: z.number(),+ collectionID: z.number(),+ ownerID: z.number().optional(),+ encryptedKey: z.string(),+ keyDecryptionNonce: z.string(),+ file: RemoteFileObjectAttributes,+ thumbnail: RemoteFileObjectAttributes.optional(),+ metadata: RemoteEncryptedMetadata,+ magicMetadata: z+ .object({+ version: z.number(),+ count: z.number().optional(),+ data: z.string(),+ header: z.string(),+ })+ .nullish(),+ pubMagicMetadata: z+ .object({+ version: z.number(),+ count: z.number().optional(),+ data: z.string(),+ header: z.string(),+ })+ .nullish(),+ updationTime: z.number(),+ isDeleted: z.boolean(),+ info: z.object({ fileSize: z.number().optional() }).nullish(),+});++export type RemoteFile = z.infer<typeof RemoteFile>;++export const FileDiffResponse = z.object({+ diff: z.array(RemoteFile),+ hasMore: z.boolean(),+});++export const RemoteTrashItem = z.object({+ file: RemoteFile,+ isDeleted: z.boolean(),+ isRestored: z.boolean(),+ updatedAt: z.number(),+ deleteBy: z.number(),+});++export const TrashDiffResponse = z.object({+ diff: z.array(RemoteTrashItem),+ hasMore: z.boolean(),+});diff --git a/web/apps/locker/src/services/sync/sync.ts b/web/apps/locker/src/services/sync/sync.tsnew file mode 100644index 0000000000..c15e487c64--- /dev/null+++ b/web/apps/locker/src/services/sync/sync.ts@@ -0,0 +1,330 @@+import type { LockerCollection, LockerItem } from "@/types";+import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";+import log from "ente-base/log";+import { apiURL } from "ente-base/origins";+import {+ type EncryptedCollectionRecord,+ type EncryptedFileRecord,+ type LockerEncryptedCache,+ replaceLockerCache,+ setEncryptedFileRecord,+} from "../locker-cache";+import {+ type StoredTrashFileRecord,+ deleteCollectionSinceTime,+ deleteFileRecords,+ deleteFileRecordsForCollection,+ deleteTrashFileRecords,+ loadLockerSnapshotFromDB,+ saveCollectionRecords,+ saveCollectionSinceTime,+ saveCollectionsSinceTime,+ saveFileRecords,+ saveTrashFileRecords,+ saveTrashSinceTime,+} from "../locker-db";+import {+ type LockerTrashData,+ buildEncryptedFileRecord,+ buildStoredTrashFileRecord,+ decryptAllData,+ decryptStoredTrash,+ toEncryptedCollectionRecord,+} from "./decrypt";+import {+ CollectionsResponse,+ FileDiffResponse,+ TrashDiffResponse,+} from "./schemas";++interface LockerHydratedState {+ collections: LockerCollection[];+ trashItems: LockerItem[];+ trashLastUpdatedAt: number;+ collectionsSinceTime: number;+ trashSinceTime: number;+}++interface LockerPersistedState extends LockerHydratedState {+ hasPersistedState: boolean;+}++const buildLockerCache = (+ collections: Map<number, EncryptedCollectionRecord>,+ files: EncryptedFileRecord[],+ trashFiles: StoredTrashFileRecord[],+): LockerEncryptedCache => {+ const nextFiles = new Map<number, Map<number, EncryptedFileRecord>>();+ for (const record of files) {+ setEncryptedFileRecord(nextFiles, record);+ }+ for (const record of trashFiles) {+ setEncryptedFileRecord(nextFiles, record);+ }++ return { collections, files: nextFiles };+};++const fetchEncryptedCollections = async (sinceTime: number) => {+ const response = await fetch(+ await apiURL("/collections/v2", { sinceTime }),+ { headers: await authenticatedRequestHeaders() },+ );+ ensureOk(response);+ const { collections } = CollectionsResponse.parse(await response.json());+ return collections;+};++interface CollectionFileDiff {+ recordsToSave: EncryptedFileRecord[];+ fileKeysToDelete: [number, number][];+ sinceTime: number;+}++const fetchEncryptedFilesForCollection = async (+ collectionID: number,+ initialSinceTime: number,+): Promise<CollectionFileDiff> => {+ let sinceTime = initialSinceTime;+ let hasMore = true;+ const recordsToSave: EncryptedFileRecord[] = [];+ const fileKeysToDelete: [number, number][] = [];++ while (hasMore) {+ const response = await fetch(+ await apiURL("/collections/v2/diff", { collectionID, sinceTime }),+ { headers: await authenticatedRequestHeaders() },+ );+ ensureOk(response);+ const parsed = FileDiffResponse.parse(await response.json());++ for (const file of parsed.diff) {+ sinceTime = Math.max(sinceTime, file.updationTime);+ if (file.isDeleted) {+ fileKeysToDelete.push([file.id, collectionID]);+ } else {+ recordsToSave.push(buildEncryptedFileRecord(file));+ }+ }++ hasMore = parsed.hasMore;+ }++ return { recordsToSave, fileKeysToDelete, sinceTime };+};++const withoutFailedCollections = (+ cache: LockerEncryptedCache,+ failedCollectionIDs: number[],+): LockerEncryptedCache => {+ if (failedCollectionIDs.length === 0) {+ return cache;+ }++ const failedCollectionIDSet = new Set(failedCollectionIDs);+ return {+ collections: new Map(+ [...cache.collections.entries()].filter(+ ([collectionID]) => !failedCollectionIDSet.has(collectionID),+ ),+ ),+ files: new Map(+ [...cache.files.entries()]+ .map(+ ([fileID, records]): [+ number,+ Map<number, EncryptedFileRecord>,+ ] => [+ fileID,+ new Map(+ [...records.entries()].filter(+ ([collectionID]) =>+ !failedCollectionIDSet.has(collectionID),+ ),+ ),+ ],+ )+ .filter(([, records]) => records.size > 0),+ ),+ };+};++const hydrateLockerState = async (+ collections: Map<number, EncryptedCollectionRecord>,+ files: EncryptedFileRecord[],+ trashFiles: StoredTrashFileRecord[],+ trashLastUpdatedAt: number,+): Promise<LockerHydratedState> => {+ const activeCache = buildLockerCache(collections, files, []);++ const decrypted = await decryptAllData(activeCache);+ if (+ decrypted.totalCollectionCount > 0 &&+ decrypted.collections.length === 0+ ) {+ throw new Error(+ `Failed to decrypt all ${decrypted.totalCollectionCount} locker collections`,+ );+ }++ const hydratedCache = withoutFailedCollections(+ buildLockerCache(collections, files, trashFiles),+ decrypted.failedCollectionIDs,+ );+ replaceLockerCache(hydratedCache);++ if (decrypted.failedCollectionIDs.length > 0) {+ log.warn(+ `Decrypted ${decrypted.collections.length}/${decrypted.totalCollectionCount} locker collections`,+ );+ }++ const trash = await decryptStoredTrash(+ hydratedCache,+ trashFiles,+ trashLastUpdatedAt,+ );++ return {+ collections: decrypted.collections,+ trashItems: trash.items,+ trashLastUpdatedAt: trash.lastUpdatedAt,+ collectionsSinceTime: 0,+ trashSinceTime: 0,+ };+};++export const loadPersistedLockerState =+ async (): Promise<LockerPersistedState> => {+ const snapshot = await loadLockerSnapshotFromDB();+ const hydrated = await hydrateLockerState(+ snapshot.collections,+ snapshot.files,+ snapshot.trashFiles,+ snapshot.trashSinceTime,+ );++ return {+ ...hydrated,+ collectionsSinceTime: snapshot.collectionsSinceTime,+ trashSinceTime: snapshot.trashSinceTime,+ hasPersistedState: snapshot.hasPersistedState,+ };+ };++export const syncLockerState = async (): Promise<LockerHydratedState> => {+ const snapshot = await loadLockerSnapshotFromDB();+ const collectionChanges = await fetchEncryptedCollections(+ snapshot.collectionsSinceTime,+ );++ let latestCollectionsSinceTime = snapshot.collectionsSinceTime;+ const changedCollections: EncryptedCollectionRecord[] = [];+ const deletedCollectionIDs: number[] = [];++ for (const change of collectionChanges) {+ latestCollectionsSinceTime = Math.max(+ latestCollectionsSinceTime,+ change.updationTime,+ );+ const record = await toEncryptedCollectionRecord(change);+ changedCollections.push(record);+ if (record.isDeleted) {+ deletedCollectionIDs.push(record.id);+ }+ }++ if (changedCollections.length > 0) {+ await saveCollectionRecords(changedCollections);+ }+ for (const collectionID of deletedCollectionIDs) {+ await deleteFileRecordsForCollection(collectionID);+ await deleteCollectionSinceTime(collectionID);+ }+ await saveCollectionsSinceTime(latestCollectionsSinceTime);++ const postCollectionSnapshot = await loadLockerSnapshotFromDB();+ for (const collection of postCollectionSnapshot.collections.values()) {+ if (collection.isDeleted) {+ continue;+ }++ const savedSinceTime =+ postCollectionSnapshot.collectionSinceTimeByID.get(collection.id) ??+ 0;+ if (savedSinceTime >= collection.updationTime) {+ continue;+ }++ const diff = await fetchEncryptedFilesForCollection(+ collection.id,+ savedSinceTime,+ );+ if (diff.recordsToSave.length > 0) {+ await saveFileRecords(diff.recordsToSave);+ }+ if (diff.fileKeysToDelete.length > 0) {+ await deleteFileRecords(diff.fileKeysToDelete);+ }++ await saveCollectionSinceTime(+ collection.id,+ Math.max(diff.sinceTime, collection.updationTime),+ );+ }++ let trashSinceTime = postCollectionSnapshot.trashSinceTime;+ let hasMore = true;+ while (hasMore) {+ const response = await fetch(+ await apiURL("/trash/v2/diff", { sinceTime: trashSinceTime }),+ { headers: await authenticatedRequestHeaders() },+ );+ ensureOk(response);+ const parsed = TrashDiffResponse.parse(await response.json());++ const recordsToSave: StoredTrashFileRecord[] = [];+ const fileIDsToDelete: number[] = [];+ for (const entry of parsed.diff) {+ trashSinceTime = Math.max(trashSinceTime, entry.updatedAt);+ if (entry.isDeleted || entry.isRestored) {+ fileIDsToDelete.push(entry.file.id);+ } else {+ recordsToSave.push(buildStoredTrashFileRecord(entry));+ }+ }++ if (recordsToSave.length > 0) {+ await saveTrashFileRecords(recordsToSave);+ }+ if (fileIDsToDelete.length > 0) {+ await deleteTrashFileRecords(fileIDsToDelete);+ }++ hasMore = parsed.hasMore;+ }++ await saveTrashSinceTime(trashSinceTime);++ const nextSnapshot = await loadLockerSnapshotFromDB();+ const hydrated = await hydrateLockerState(+ nextSnapshot.collections,+ nextSnapshot.files,+ nextSnapshot.trashFiles,+ nextSnapshot.trashSinceTime,+ );++ return {+ ...hydrated,+ collectionsSinceTime: nextSnapshot.collectionsSinceTime,+ trashSinceTime: nextSnapshot.trashSinceTime,+ };+};++export const fetchLockerData = async (): Promise<LockerCollection[]> =>+ (await syncLockerState()).collections;++export const fetchLockerTrash = async (): Promise<LockerTrashData> => {+ const state = await syncLockerState();+ return { items: state.trashItems, lastUpdatedAt: state.trashLastUpdatedAt };+};diff --git a/web/apps/locker/tests/download.test.ts b/web/apps/locker/tests/download.test.tsindex 938406efd2..97d6f63320 100644--- a/web/apps/locker/tests/download.test.ts+++ b/web/apps/locker/tests/download.test.ts@@ -18,11 +18,11 @@ const {customOrigin: vi.fn(),fetchFile: vi.fn(),}));-vi.mock("../src/services/remote-cache", () => ({+vi.mock("../src/services/locker-cache", () => ({getEncryptedFileRecord: getRecord,getLockerCacheSnapshot: getSnapshot,}));-vi.mock("../src/services/remote-read", () => ({ decryptCollectionKey }));+vi.mock("../src/services/sync/decrypt", () => ({ decryptCollectionKey }));vi.mock("ente-locker-wasm", () => ({decryptBox,createStreamDecryptor: createDecryptor,diff --git a/web/apps/locker/tests/file-links.test.ts b/web/apps/locker/tests/file-links.test.tsindex b0ef8f684a..fd1ffddc49 100644--- a/web/apps/locker/tests/file-links.test.ts+++ b/web/apps/locker/tests/file-links.test.ts@@ -13,10 +13,10 @@ const { getRecord, decryptKey, session, prepare, openSecret } = vi.hoisted(openSecret: vi.fn(),}),);-vi.mock("../src/services/remote-cache", () => ({+vi.mock("../src/services/locker-cache", () => ({getEncryptedFileRecord: getRecord,}));-vi.mock("../src/services/remote-read", () => ({+vi.mock("../src/services/sync/decrypt", () => ({decryptFileKeyForRecord: decryptKey,}));vi.mock("../src/services/authenticated-session", () => ({diff --git a/web/apps/locker/tests/item-metadata.test.ts b/web/apps/locker/tests/item-metadata.test.tsindex a212226b5a..feb389cde7 100644--- a/web/apps/locker/tests/item-metadata.test.ts+++ b/web/apps/locker/tests/item-metadata.test.ts@@ -1,10 +1,10 @@import { afterEach, beforeEach, expect, test, vi } from "vitest";-import { updateFileItem, updateInfoItem } from "../src/services/remote";import {getEncryptedFileRecord,replaceLockerCache,type EncryptedFileRecord,-} from "../src/services/remote-cache";+} from "../src/services/locker-cache";+import { updateFileItem, updateInfoItem } from "../src/services/remote";const { encryptBlob, decryptMetadataJSON } = vi.hoisted(() => ({encryptBlob: vi.fn(),@@ -25,7 +25,7 @@ vi.mock("ente-locker-wasm", () => ({encryptBlob,}));vi.mock("../src/services/authenticated-session", () => ({}));-vi.mock("../src/services/remote-read", () => ({+vi.mock("../src/services/sync/decrypt", () => ({decryptCollectionKey: () => "collection-key",}));diff --git a/web/apps/locker/tests/sync.test.ts b/web/apps/locker/tests/sync.test.tsnew file mode 100644index 0000000000..a814881f64--- /dev/null+++ b/web/apps/locker/tests/sync.test.ts@@ -0,0 +1,256 @@+import { afterEach, beforeEach, expect, test, vi } from "vitest";+import {+ getLockerCacheSnapshot,+ replaceLockerCache,+ type EncryptedCollectionRecord,+ type EncryptedFileRecord,+} from "../src/services/locker-cache";+import * as db from "../src/services/locker-db";+import {+ loadPersistedLockerState,+ syncLockerState,+} from "../src/services/sync/sync";++vi.mock("../src/services/locker-db", () => ({+ loadLockerSnapshotFromDB: vi.fn(),+ saveCollectionRecords: vi.fn(),+ saveCollectionsSinceTime: vi.fn(),+ saveCollectionSinceTime: vi.fn(),+ deleteCollectionSinceTime: vi.fn(),+ saveFileRecords: vi.fn(),+ deleteFileRecords: vi.fn(),+ deleteFileRecordsForCollection: vi.fn(),+ saveTrashFileRecords: vi.fn(),+ deleteTrashFileRecords: vi.fn(),+ saveTrashSinceTime: vi.fn(),+}));+const { openCollectionKey, decryptMetadataJSON } = vi.hoisted(() => ({+ openCollectionKey: vi.fn(),+ decryptMetadataJSON: vi.fn(),+}));+vi.mock("../src/services/authenticated-session", () => ({+ ensureAuthenticatedSession: () => "session",+}));+vi.mock("ente-locker-wasm", () => ({+ openCollectionKey,+ decryptBox: () => "file-key",+ decryptMetadataJSON,+ decryptBoxBytes: () =>+ new TextEncoder().encode(+ JSON.stringify({ name: "Saved", owner: { id: 1 }, sharees: [] }),+ ),+ encryptBoxBytes: () => ({ encryptedData: "payload", nonce: "nonce" }),+}));+vi.mock("ente-base/http", () => ({+ authenticatedRequestHeaders: () => ({ "X-Auth-Token": "test" }),+ ensureOk: (response: Response) => {+ if (!response.ok) throw new Error("Request failed");+ },+}));+vi.mock("ente-base/origins", () => ({+ apiURL: (path: string, params: Record<string, number>) =>+ `${path}?${new URLSearchParams(Object.entries(params).map(([key, value]) => [key, String(value)]))}`,+}));+vi.mock("ente-base/log", () => ({+ default: { error: vi.fn(), warn: vi.fn() },+}));++const collection = (id: number): EncryptedCollectionRecord => ({+ id,+ ownerID: 1,+ encryptedKey: `key-${id}`,+ keyDecryptionNonce: "nonce",+ encryptedName: undefined,+ nameDecryptionNonce: undefined,+ payloadEncryptedData: "payload",+ payloadDecryptionNonce: "nonce",+ type: "folder",+ isDeleted: false,+ updationTime: 20,+});+const file = (id: number, collectionID: number): EncryptedFileRecord => ({+ id,+ collectionID,+ encryptedKey: "key",+ keyDecryptionNonce: "nonce",+ fileDecryptionHeader: "header",+ hasObject: true,+ metadata: { encryptedData: "metadata", decryptionHeader: "header" },+ pubMagicMetadata: { version: 1, data: "public", header: "header" },+ updationTime: 30,+});+const snapshot = () => ({+ collections: new Map([+ [1, collection(1)],+ [2, collection(2)],+ ]),+ files: [file(10, 1), file(10, 2)],+ trashFiles: [{ ...file(11, 1), updatedAt: 40, deleteBy: 100 }],+ collectionsSinceTime: 5,+ collectionSinceTimeByID: new Map([+ [1, 10],+ [2, 20],+ ]),+ trashSinceTime: 40,+ hasPersistedState: true,+});+const fetchMock = vi.fn<typeof fetch>();+const response = (body: unknown) =>+ new Response(JSON.stringify(body), { status: 200 });++beforeEach(() => {+ vi.resetAllMocks();+ vi.stubGlobal("fetch", fetchMock);+ replaceLockerCache({ collections: new Map(), files: new Map() });+ vi.mocked(db.loadLockerSnapshotFromDB).mockResolvedValue(snapshot());+ openCollectionKey.mockResolvedValue("collection-key");+ decryptMetadataJSON.mockImplementation(+ (value: { encryptedData: string }) =>+ value.encryptedData === "public"+ ? { editedName: "Renamed" }+ : { title: "Original", creationTime: 1_700_000_000_000 },+ );+});+afterEach(() => vi.unstubAllGlobals());++test("persisted hydration preserves memberships, metadata, trash and cursors without fetching", async () => {+ const result = await loadPersistedLockerState();+ expect(result).toMatchObject({+ collectionsSinceTime: 5,+ trashSinceTime: 40,+ trashLastUpdatedAt: 40,+ hasPersistedState: true,+ });+ expect(result.collections).toHaveLength(2);+ expect(result.collections[0]?.items[0]).toMatchObject({+ id: 10,+ collectionIDs: [1, 2],+ data: { name: "Renamed" },+ createdAt: 1_700_000_000_000_000,+ });+ expect(result.trashItems[0]).toMatchObject({+ id: 11,+ updatedAt: 40,+ deleteBy: 100,+ });+ expect(getLockerCacheSnapshot().files.get(10)?.size).toBe(2);+ expect(getLockerCacheSnapshot().files.has(11)).toBe(true);+ expect(fetchMock).not.toHaveBeenCalled();+});++test("partial decryption failure excludes only the failed collection and its cache records", async () => {+ openCollectionKey.mockImplementation((_session, _owner, key) => {+ if (key === "key-2") throw new Error("Bad key");+ return "collection-key";+ });+ const result = await loadPersistedLockerState();+ expect(result.collections.map((entry) => entry.id)).toEqual([1]);+ expect([...getLockerCacheSnapshot().collections.keys()]).toEqual([1]);+ expect([...getLockerCacheSnapshot().files.get(10)!.keys()]).toEqual([1]);+});++test("total decryption failure leaves the existing cache intact", async () => {+ const existing = {+ collections: new Map([[9, collection(9)]]),+ files: new Map(),+ };+ replaceLockerCache(existing);+ openCollectionKey.mockRejectedValue(new Error("Bad key"));+ await expect(loadPersistedLockerState()).rejects.toThrow(+ "Failed to decrypt all 2 locker collections",+ );+ expect(getLockerCacheSnapshot()).toEqual(existing);+});++test("incremental sync paginates changed collections and trash, skipping up-to-date collections", async () => {+ const remoteFile = {+ id: 12,+ collectionID: 1,+ encryptedKey: "key",+ keyDecryptionNonce: "nonce",+ file: { decryptionHeader: "header" },+ metadata: { encryptedData: "metadata", decryptionHeader: "header" },+ updationTime: 30,+ isDeleted: false,+ };+ fetchMock+ .mockResolvedValueOnce(response({ collections: [] }))+ .mockResolvedValueOnce(response({ diff: [remoteFile], hasMore: true }))+ .mockResolvedValueOnce(+ response({+ diff: [+ {+ ...remoteFile,+ id: 13,+ updationTime: 35,+ isDeleted: true,+ },+ ],+ hasMore: false,+ }),+ )+ .mockResolvedValueOnce(+ response({+ diff: [+ {+ file: remoteFile,+ isDeleted: false,+ isRestored: false,+ updatedAt: 45,+ deleteBy: 100,+ },+ ],+ hasMore: true,+ }),+ )+ .mockResolvedValueOnce(+ response({+ diff: [+ {+ file: { ...remoteFile, id: 11 },+ isDeleted: false,+ isRestored: true,+ updatedAt: 50,+ deleteBy: 100,+ },+ ],+ hasMore: false,+ }),+ );+ await syncLockerState();+ expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([+ "/collections/v2?sinceTime=5",+ "/collections/v2/diff?collectionID=1&sinceTime=10",+ "/collections/v2/diff?collectionID=1&sinceTime=30",+ "/trash/v2/diff?sinceTime=40",+ "/trash/v2/diff?sinceTime=45",+ ]);+ expect(db.saveFileRecords).toHaveBeenCalledWith([+ expect.objectContaining({ id: 12, collectionID: 1, hasObject: true }),+ ]);+ expect(db.deleteFileRecords).toHaveBeenCalledWith([[13, 1]]);+ expect(db.saveCollectionSinceTime).toHaveBeenCalledExactlyOnceWith(1, 35);+ expect(db.saveTrashFileRecords).toHaveBeenCalledWith([+ expect.objectContaining({ id: 12, updatedAt: 45, deleteBy: 100 }),+ ]);+ expect(db.deleteTrashFileRecords).toHaveBeenCalledWith([11]);+ expect(db.saveTrashSinceTime).toHaveBeenCalledExactlyOnceWith(50);+ expect(+ vi.mocked(db.saveFileRecords).mock.invocationCallOrder[0],+ ).toBeLessThan(+ vi.mocked(db.saveCollectionSinceTime).mock.invocationCallOrder[0]!,+ );+});++test("failed file pagination does not persist a partial diff or advance its cursor", async () => {+ fetchMock+ .mockResolvedValueOnce(response({ collections: [] }))+ .mockResolvedValueOnce(response({ diff: [], hasMore: true }))+ .mockResolvedValueOnce(new Response(null, { status: 500 }));+ const existing = getLockerCacheSnapshot();+ await expect(syncLockerState()).rejects.toThrow("Request failed");+ expect(db.saveFileRecords).not.toHaveBeenCalled();+ expect(db.saveCollectionSinceTime).not.toHaveBeenCalled();+ expect(db.saveTrashSinceTime).not.toHaveBeenCalled();+ expect(getLockerCacheSnapshot()).toEqual(existing);+});