π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Enable deferred multipart checksums for internal web uploads
Neeraj Guptad515d07d4b4c
diff --git a/web/apps/photos/src/services/upload-manager.ts b/web/apps/photos/src/services/upload-manager.tsindex 63d224e9e9..f5ba41c5dc 100644--- a/web/apps/photos/src/services/upload-manager.ts+++ b/web/apps/photos/src/services/upload-manager.ts@@ -4,6 +4,7 @@ import { ensureLocalUser } from "ente-accounts/services/user";import { isDesktop } from "ente-base/app";import { createComlinkCryptoWorker } from "ente-base/crypto";import type { CryptoWorker } from "ente-base/crypto/worker";+import { isDevBuild } from "ente-base/env";import { lowercaseExtension, nameAndExtension } from "ente-base/file-name";import log from "ente-base/log";import { ComlinkWorker } from "ente-base/worker/comlink-worker";@@ -45,6 +46,7 @@ import { FileType } from "ente-media/file-type";import { potentialFileTypeFromExtension } from "ente-media/live-photo";import { computeNormalCollectionFilesFromSaved } from "ente-new/photos/services/file";import { indexNewUpload } from "ente-new/photos/services/ml";+import { settingsSnapshot } from "ente-new/photos/services/settings";import { wait } from "ente-utils/promise";import watcher from "./watch";@@ -533,8 +535,12 @@ class UploadManager {options?: UploadItemsOptions,) {const uiService = this.uiService;+ const settings = settingsSnapshot();const uploadContext = {isCFUploadProxyDisabled: shouldDisableCFUploadProxy(),+ deferMultipartChecksums:+ settings.deferredMultipartChecksumsEnabled &&+ (settings.isInternalUser || isDevBuild),skipDuplicateAddToUploadCollection:options?.skipDuplicateAddToUploadCollection,includePartnerSharedFiles: options?.includePartnerSharedFiles,diff --git a/web/packages/base/crypto/types.ts b/web/packages/base/crypto/types.tsindex 3181dc01bb..1c45407844 100644--- a/web/packages/base/crypto/types.ts+++ b/web/packages/base/crypto/types.ts@@ -4,6 +4,8 @@ export type SodiumStateAddress = StateAddress;// Existing encrypted files depend on this chunk boundary.export const streamEncryptionChunkSize = 4 * 1024 * 1024;+// libsodium secretstream adds this many bytes per encrypted chunk.+export const streamEncryptionChunkOverhead = 17;export const deriveKeyInsufficientMemoryErrorMessage ="Failed to derive key (insufficient memory)";diff --git a/web/packages/gallery/services/upload/remote.ts b/web/packages/gallery/services/upload/remote.tsindex 63f7a32250..204070fce8 100644--- a/web/packages/gallery/services/upload/remote.ts+++ b/web/packages/gallery/services/upload/remote.ts@@ -79,7 +79,7 @@ export const fetchMultipartUploadURLsWithMetadata = async ({}: {contentLength: number;partLength: number;- partMd5s: string[];+ partMd5s?: string[];}) => {const headers = new Headers(await authenticatedRequestHeaders());headers.set("Content-Type", "application/json");diff --git a/web/packages/gallery/services/upload/upload-service.ts b/web/packages/gallery/services/upload/upload-service.tsindex 8cc95c0882..b74218f658 100644--- a/web/packages/gallery/services/upload/upload-service.ts+++ b/web/packages/gallery/services/upload/upload-service.ts@@ -2,7 +2,10 @@/* eslint-disable @typescript-eslint/ban-ts-comment */import type { BytesOrB64 } from "ente-base/crypto/types";-import { streamEncryptionChunkSize } from "ente-base/crypto/types";+import {+ streamEncryptionChunkOverhead,+ streamEncryptionChunkSize,+} from "ente-base/crypto/types";import type { CryptoWorker } from "ente-base/crypto/worker";import { ensureElectron } from "ente-base/electron";import { basename, nameAndExtension } from "ente-base/file-name";@@ -233,6 +236,21 @@ class UploadService {throw translateURLFetchErrorIfNeeded(e);});}++ async fetchMultipartUploadURLsWithoutChecksums(+ contentLength: number,+ partLength: number,+ ) {+ if (this.publicAlbumsCredentials) {+ throw new Error("Public uploads require part checksums");+ }+ return fetchMultipartUploadURLsWithMetadata({+ contentLength,+ partLength,+ }).catch((e: unknown) => {+ throw translateURLFetchErrorIfNeeded(e);+ });+ }}const uploadService = new UploadService();@@ -273,6 +291,7 @@ interface FileWithMetadata extends Omit<ThumbnailedFile, "hasStaticThumbnail"> {interface EncryptedFileStream {stream: ReadableStream<Uint8Array<ArrayBuffer>>;chunkCount: number;+ encryptedSize: number;}interface EncryptedFilePieces {@@ -453,6 +472,7 @@ const fileTooLargeErrorMessage = "File too large";interface UploadContext {isCFUploadProxyDisabled: boolean;+ deferMultipartChecksums: boolean;skipDuplicateAddToUploadCollection?: boolean;includePartnerSharedFiles?: boolean;publicAlbumsCredentials?: PublicAlbumsCredentials;@@ -1215,7 +1235,7 @@ const encryptBytesWithOptionalVerification = async (};const encryptFileStream = async (- { stream, chunkCount }: FileStream,+ { stream, chunkCount, fileSize }: FileStream,fileKey: BytesOrB64,worker: CryptoWorker,shouldVerify: boolean,@@ -1274,7 +1294,12 @@ const encryptFileStream = async (});return {decryptionHeader,- encryptedData: { stream: encryptedFileStream, chunkCount },+ encryptedData: {+ stream: encryptedFileStream,+ chunkCount,+ encryptedSize:+ fileSize + chunkCount * streamEncryptionChunkOverhead,+ },};};@@ -1428,6 +1453,9 @@ const uploadStreamUsingMultipart = async (uploadContext;const shouldSendPartChecksums =checksumEnabled || !!uploadContext.publicAlbumsCredentials;+ const deferPartChecksums =+ uploadContext.deferMultipartChecksums &&+ !uploadContext.publicAlbumsCredentials;const { stream } = dataStream;const streamReader = stream.getReader();@@ -1436,7 +1464,7 @@ const uploadStreamUsingMultipart = async (dataStream.chunkCount / multipartChunksPerPart,);- if (shouldSendPartChecksums) {+ if (shouldSendPartChecksums && !deferPartChecksums) {const parts: Uint8Array<ArrayBuffer>[] = [];const partMd5s: string[] = [];let fileSize = 0;@@ -1518,8 +1546,20 @@ const uploadStreamUsingMultipart = async (return { objectKey: multipartUploadURLs.objectKey, fileSize };}- const multipartUploadURLs =- await uploadService.fetchMultipartUploadURLs(uploadPartCount);+ const partLength = Math.min(+ dataStream.encryptedSize,+ multipartChunksPerPart *+ (streamEncryptionChunkSize + streamEncryptionChunkOverhead),+ );+ const multipartUploadURLs = deferPartChecksums+ ? await uploadService.fetchMultipartUploadURLsWithoutChecksums(+ dataStream.encryptedSize,+ partLength,+ )+ : await uploadService.fetchMultipartUploadURLs(uploadPartCount);+ if (multipartUploadURLs.partURLs.length != uploadPartCount) {+ throw new Error("Unexpected multipart upload URL count");+ }const percentPerPart = maxPercent / uploadPartCount;let fileSize = 0;@@ -1533,14 +1573,20 @@ const uploadStreamUsingMultipart = async (const partNumber = index + 1;const partData = await nextMultipartUploadPart(streamReader);fileSize += partData.length;+ const checksum = deferPartChecksums+ ? computeMd5Base64(partData)+ : undefined;const eTag = !isCFUploadProxyDisabled? await putFilePartViaWorker(partUploadURL,partData,requestRetrier,+ { contentMd5: checksum },)- : await putFilePart(partUploadURL, partData, requestRetrier);+ : await putFilePart(partUploadURL, partData, requestRetrier, {+ contentMd5: checksum,+ });if (!eTag) throw new Error(eTagMissingErrorMessage);updateUploadProgress(fileLocalID, percentPerPart * partNumber);diff --git a/web/packages/new/photos/services/settings.ts b/web/packages/new/photos/services/settings.tsindex f96fb67439..be78ab5403 100644--- a/web/packages/new/photos/services/settings.ts+++ b/web/packages/new/photos/services/settings.ts@@ -13,6 +13,7 @@ import {export interface Settings {isInternalUser: boolean;+ deferredMultipartChecksumsEnabled: boolean;mapEnabled: boolean;cfUploadProxyDisabled: boolean;castURL: string;@@ -23,6 +24,7 @@ export interface Settings {const createDefaultSettings = (): Settings => ({isInternalUser: false,+ deferredMultipartChecksumsEnabled: false,mapEnabled: false,cfUploadProxyDisabled: false,castURL: "https://cast.ente.com",@@ -85,10 +87,15 @@ const FeatureFlags = z.object({type FeatureFlags = z.infer<typeof FeatureFlags>;+const deferredMultipartChecksumsFlag = 1 << 6;+const syncSettingsSnapshotWithLocalStorage = () => {const flags = savedRemoteFeatureFlags();const settings = createDefaultSettings();settings.isInternalUser = flags?.internalUser || false;+ settings.deferredMultipartChecksumsEnabled = !!(+ (flags?.serverApiFlag ?? 0) & deferredMultipartChecksumsFlag+ );settings.mapEnabled = flags?.mapEnabled || false;settings.cfUploadProxyDisabled = savedCFProxyDisabled();if (flags?.castUrl) settings.castURL = flags.castUrl;