Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Enable deferred multipart checksums for internal web uploads

Neeraj Guptad515d07d4b4c
diff --git a/web/apps/photos/src/services/upload-manager.ts b/web/apps/photos/src/services/upload-manager.ts
index 63d224e9e9..f5ba41c5dc 100644
--- a/web/apps/photos/src/services/upload-manager.ts
+++ b/web/apps/photos/src/services/upload-manager.ts
@@ -4,6 +4,7 @@ import { ensureLocalUser } from "ente-accounts/services/user";
import { isDesktop } from "ente-base/app";
import { createComlinkCryptoWorker } from "ente-base/crypto";
import type { CryptoWorker } from "ente-base/crypto/worker";
+import { isDevBuild } from "ente-base/env";
import { lowercaseExtension, nameAndExtension } from "ente-base/file-name";
import log from "ente-base/log";
import { ComlinkWorker } from "ente-base/worker/comlink-worker";
@@ -45,6 +46,7 @@ import { FileType } from "ente-media/file-type";
import { potentialFileTypeFromExtension } from "ente-media/live-photo";
import { computeNormalCollectionFilesFromSaved } from "ente-new/photos/services/file";
import { indexNewUpload } from "ente-new/photos/services/ml";
+import { settingsSnapshot } from "ente-new/photos/services/settings";
import { wait } from "ente-utils/promise";
import watcher from "./watch";
@@ -533,8 +535,12 @@ class UploadManager {
options?: UploadItemsOptions,
) {
const uiService = this.uiService;
+ const settings = settingsSnapshot();
const uploadContext = {
isCFUploadProxyDisabled: shouldDisableCFUploadProxy(),
+ deferMultipartChecksums:
+ settings.deferredMultipartChecksumsEnabled &&
+ (settings.isInternalUser || isDevBuild),
skipDuplicateAddToUploadCollection:
options?.skipDuplicateAddToUploadCollection,
includePartnerSharedFiles: options?.includePartnerSharedFiles,
diff --git a/web/packages/base/crypto/types.ts b/web/packages/base/crypto/types.ts
index 3181dc01bb..1c45407844 100644
--- a/web/packages/base/crypto/types.ts
+++ b/web/packages/base/crypto/types.ts
@@ -4,6 +4,8 @@ export type SodiumStateAddress = StateAddress;
// Existing encrypted files depend on this chunk boundary.
export const streamEncryptionChunkSize = 4 * 1024 * 1024;
+// libsodium secretstream adds this many bytes per encrypted chunk.
+export const streamEncryptionChunkOverhead = 17;
export const deriveKeyInsufficientMemoryErrorMessage =
"Failed to derive key (insufficient memory)";
diff --git a/web/packages/gallery/services/upload/remote.ts b/web/packages/gallery/services/upload/remote.ts
index 63f7a32250..204070fce8 100644
--- a/web/packages/gallery/services/upload/remote.ts
+++ b/web/packages/gallery/services/upload/remote.ts
@@ -79,7 +79,7 @@ export const fetchMultipartUploadURLsWithMetadata = async ({
}: {
contentLength: number;
partLength: number;
- partMd5s: string[];
+ partMd5s?: string[];
}) => {
const headers = new Headers(await authenticatedRequestHeaders());
headers.set("Content-Type", "application/json");
diff --git a/web/packages/gallery/services/upload/upload-service.ts b/web/packages/gallery/services/upload/upload-service.ts
index 8cc95c0882..b74218f658 100644
--- a/web/packages/gallery/services/upload/upload-service.ts
+++ b/web/packages/gallery/services/upload/upload-service.ts
@@ -2,7 +2,10 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
import type { BytesOrB64 } from "ente-base/crypto/types";
-import { streamEncryptionChunkSize } from "ente-base/crypto/types";
+import {
+ streamEncryptionChunkOverhead,
+ streamEncryptionChunkSize,
+} from "ente-base/crypto/types";
import type { CryptoWorker } from "ente-base/crypto/worker";
import { ensureElectron } from "ente-base/electron";
import { basename, nameAndExtension } from "ente-base/file-name";
@@ -233,6 +236,21 @@ class UploadService {
throw translateURLFetchErrorIfNeeded(e);
});
}
+
+ async fetchMultipartUploadURLsWithoutChecksums(
+ contentLength: number,
+ partLength: number,
+ ) {
+ if (this.publicAlbumsCredentials) {
+ throw new Error("Public uploads require part checksums");
+ }
+ return fetchMultipartUploadURLsWithMetadata({
+ contentLength,
+ partLength,
+ }).catch((e: unknown) => {
+ throw translateURLFetchErrorIfNeeded(e);
+ });
+ }
}
const uploadService = new UploadService();
@@ -273,6 +291,7 @@ interface FileWithMetadata extends Omit<ThumbnailedFile, "hasStaticThumbnail"> {
interface EncryptedFileStream {
stream: ReadableStream<Uint8Array<ArrayBuffer>>;
chunkCount: number;
+ encryptedSize: number;
}
interface EncryptedFilePieces {
@@ -453,6 +472,7 @@ const fileTooLargeErrorMessage = "File too large";
interface UploadContext {
isCFUploadProxyDisabled: boolean;
+ deferMultipartChecksums: boolean;
skipDuplicateAddToUploadCollection?: boolean;
includePartnerSharedFiles?: boolean;
publicAlbumsCredentials?: PublicAlbumsCredentials;
@@ -1215,7 +1235,7 @@ const encryptBytesWithOptionalVerification = async (
};
const encryptFileStream = async (
- { stream, chunkCount }: FileStream,
+ { stream, chunkCount, fileSize }: FileStream,
fileKey: BytesOrB64,
worker: CryptoWorker,
shouldVerify: boolean,
@@ -1274,7 +1294,12 @@ const encryptFileStream = async (
});
return {
decryptionHeader,
- encryptedData: { stream: encryptedFileStream, chunkCount },
+ encryptedData: {
+ stream: encryptedFileStream,
+ chunkCount,
+ encryptedSize:
+ fileSize + chunkCount * streamEncryptionChunkOverhead,
+ },
};
};
@@ -1428,6 +1453,9 @@ const uploadStreamUsingMultipart = async (
uploadContext;
const shouldSendPartChecksums =
checksumEnabled || !!uploadContext.publicAlbumsCredentials;
+ const deferPartChecksums =
+ uploadContext.deferMultipartChecksums &&
+ !uploadContext.publicAlbumsCredentials;
const { stream } = dataStream;
const streamReader = stream.getReader();
@@ -1436,7 +1464,7 @@ const uploadStreamUsingMultipart = async (
dataStream.chunkCount / multipartChunksPerPart,
);
- if (shouldSendPartChecksums) {
+ if (shouldSendPartChecksums && !deferPartChecksums) {
const parts: Uint8Array<ArrayBuffer>[] = [];
const partMd5s: string[] = [];
let fileSize = 0;
@@ -1518,8 +1546,20 @@ const uploadStreamUsingMultipart = async (
return { objectKey: multipartUploadURLs.objectKey, fileSize };
}
- const multipartUploadURLs =
- await uploadService.fetchMultipartUploadURLs(uploadPartCount);
+ const partLength = Math.min(
+ dataStream.encryptedSize,
+ multipartChunksPerPart *
+ (streamEncryptionChunkSize + streamEncryptionChunkOverhead),
+ );
+ const multipartUploadURLs = deferPartChecksums
+ ? await uploadService.fetchMultipartUploadURLsWithoutChecksums(
+ dataStream.encryptedSize,
+ partLength,
+ )
+ : await uploadService.fetchMultipartUploadURLs(uploadPartCount);
+ if (multipartUploadURLs.partURLs.length != uploadPartCount) {
+ throw new Error("Unexpected multipart upload URL count");
+ }
const percentPerPart = maxPercent / uploadPartCount;
let fileSize = 0;
@@ -1533,14 +1573,20 @@ const uploadStreamUsingMultipart = async (
const partNumber = index + 1;
const partData = await nextMultipartUploadPart(streamReader);
fileSize += partData.length;
+ const checksum = deferPartChecksums
+ ? computeMd5Base64(partData)
+ : undefined;
const eTag = !isCFUploadProxyDisabled
? await putFilePartViaWorker(
partUploadURL,
partData,
requestRetrier,
+ { contentMd5: checksum },
)
- : await putFilePart(partUploadURL, partData, requestRetrier);
+ : await putFilePart(partUploadURL, partData, requestRetrier, {
+ contentMd5: checksum,
+ });
if (!eTag) throw new Error(eTagMissingErrorMessage);
updateUploadProgress(fileLocalID, percentPerPart * partNumber);
diff --git a/web/packages/new/photos/services/settings.ts b/web/packages/new/photos/services/settings.ts
index f96fb67439..be78ab5403 100644
--- a/web/packages/new/photos/services/settings.ts
+++ b/web/packages/new/photos/services/settings.ts
@@ -13,6 +13,7 @@ import {
export interface Settings {
isInternalUser: boolean;
+ deferredMultipartChecksumsEnabled: boolean;
mapEnabled: boolean;
cfUploadProxyDisabled: boolean;
castURL: string;
@@ -23,6 +24,7 @@ export interface Settings {
const createDefaultSettings = (): Settings => ({
isInternalUser: false,
+ deferredMultipartChecksumsEnabled: false,
mapEnabled: false,
cfUploadProxyDisabled: false,
castURL: "https://cast.ente.com",
@@ -85,10 +87,15 @@ const FeatureFlags = z.object({
type FeatureFlags = z.infer<typeof FeatureFlags>;
+const deferredMultipartChecksumsFlag = 1 << 6;
+
const syncSettingsSnapshotWithLocalStorage = () => {
const flags = savedRemoteFeatureFlags();
const settings = createDefaultSettings();
settings.isInternalUser = flags?.internalUser || false;
+ settings.deferredMultipartChecksumsEnabled = !!(
+ (flags?.serverApiFlag ?? 0) & deferredMultipartChecksumsFlag
+ );
settings.mapEnabled = flags?.mapEnabled || false;
settings.cfUploadProxyDisabled = savedCFProxyDisabled();
if (flags?.castUrl) settings.castURL = flags.castUrl;