Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][locker] Add multipart uploads for internal users (#13729)

aman-pilotd4e2b53a0bb8
diff --git a/mobile/apps/locker/changes/multipart-upload.md b/mobile/apps/locker/changes/multipart-upload.md
new file mode 100644
index 0000000000..bb03ca6cce
--- /dev/null
+++ b/mobile/apps/locker/changes/multipart-upload.md
@@ -0,0 +1 @@
+- (i) Added multipart uploads with retries for large files.
diff --git a/mobile/apps/locker/lib/services/feature_flag_service.dart b/mobile/apps/locker/lib/services/feature_flag_service.dart
index 0f4659397a..3d478eb8dc 100644
--- a/mobile/apps/locker/lib/services/feature_flag_service.dart
+++ b/mobile/apps/locker/lib/services/feature_flag_service.dart
@@ -16,6 +16,7 @@ class FeatureFlagService {
late FlagService _flagService;
bool get internalUser => _flagService.internalUser;
+ bool get multipartUpload => internalUser;
bool get isInternalUserDisabled =>
_preferences.getBool(_internalUserDisabled) ?? false;
diff --git a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
index 8372199ceb..2a7b869db9 100644
--- a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
+++ b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
@@ -17,12 +17,14 @@ import 'package:locker/core/errors.dart';
import 'package:locker/events/backup_updated_event.dart';
import 'package:locker/services/collections/models/collection.dart';
import 'package:locker/services/configuration.dart';
+import 'package:locker/services/feature_flag_service.dart';
import "package:locker/services/files/sync/metadata_updater_service.dart";
import 'package:locker/services/files/sync/models/file.dart';
import 'package:locker/services/files/sync/models/file_magic.dart';
import 'package:locker/services/files/upload/models/backup_item.dart';
import 'package:locker/services/files/upload/models/backup_item_status.dart';
import 'package:locker/services/files/upload/models/upload_url.dart';
+import 'package:locker/services/files/upload/multipart_uploader.dart';
import "package:locker/utils/crypto_helper.dart";
import 'package:logging/logging.dart';
import 'package:shared_preferences/shared_preferences.dart';
@@ -39,6 +41,7 @@ class FileUploader {
final _logger = Logger("FileUploader");
final _dio = Network.instance.getDio();
final _enteDio = Network.instance.enteDio;
+ final _multipartUploader = MultipartUploader();
final LinkedHashMap<String, FileUploadItem> _queue =
LinkedHashMap<String, FileUploadItem>();
final LinkedHashMap<String, BackupItem> _allBackups =
@@ -302,11 +305,17 @@ class FileUploader {
_checkIfWithinFileCountLimit();
await _checkIfWithinStorageLimit(file);
final encryptedFile = File(encryptedFilePath);
+ final useMultipart =
+ FeatureFlagService.instance.multipartUpload &&
+ sourceFileSize > MultipartUploader.partSize;
final fileAttributes = await CryptoUtil.encryptFileWithMd5(
file.path,
encryptedFilePath,
key: key,
+ multiPartChunkSizeInBytes: useMultipart
+ ? MultipartUploader.partSize
+ : null,
);
encFileSize = await encryptedFile.length();
@@ -334,7 +343,7 @@ class FileUploader {
final thumbnailMd5 = await computeMd5(encryptedThumbnailPath);
final fileMd5 = fileAttributes.fileMd5;
- if (fileMd5 == null || fileMd5.isEmpty) {
+ if (!useMultipart && (fileMd5 == null || fileMd5.isEmpty)) {
throw Exception('File MD5 hash is null or empty');
}
@@ -353,16 +362,33 @@ class FileUploader {
thumbnailMd5,
);
- final fileUploadURL = await _getUploadURL(
- contentLength: encFileSize,
- md5: fileMd5,
- );
- final fileObjectKey = await _putFile(
- fileUploadURL,
- encryptedFile,
- encFileSize,
- fileMd5,
- );
+ final String fileObjectKey;
+ if (useMultipart) {
+ final urls = MultipartUploadURLs.fromMap(
+ await _requestUploadURL('/files/multipart-upload-url', {
+ 'contentLength': encFileSize,
+ 'partLength': MultipartUploader.partSize,
+ 'partMd5s': fileAttributes.partMd5s!,
+ }),
+ );
+ fileObjectKey = await _multipartUploader.upload(
+ encryptedFile,
+ encFileSize,
+ fileAttributes.partMd5s!,
+ urls,
+ );
+ } else {
+ final fileUploadURL = await _getUploadURL(
+ contentLength: encFileSize,
+ md5: fileMd5!,
+ );
+ fileObjectKey = await _putFile(
+ fileUploadURL,
+ encryptedFile,
+ encFileSize,
+ fileMd5,
+ );
+ }
final enteFile = EnteFile.fromFile(file, fileName: fileName);
@@ -650,12 +676,21 @@ class FileUploader {
required int contentLength,
required String md5,
}) async {
+ return UploadURL.fromMap(
+ await _requestUploadURL('/files/upload-url', {
+ 'contentLength': contentLength,
+ 'contentMD5': md5,
+ }),
+ );
+ }
+
+ Future<Map<String, dynamic>> _requestUploadURL(
+ String endpoint,
+ Map<String, dynamic> data,
+ ) async {
try {
- final response = await _enteDio.post(
- "/files/upload-url",
- data: {"contentLength": contentLength, "contentMD5": md5},
- );
- return UploadURL.fromMap((response.data as Map).cast<String, dynamic>());
+ final response = await _enteDio.post(endpoint, data: data);
+ return (response.data as Map).cast<String, dynamic>();
} on DioException catch (e, s) {
if (_isFileLimitReachedResponse(e.response)) {
throw FileLimitReachedError();
diff --git a/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart b/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart
new file mode 100644
index 0000000000..ac480aadad
--- /dev/null
+++ b/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart
@@ -0,0 +1,103 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:dio/dio.dart';
+import 'package:logging/logging.dart';
+
+class MultipartUploader {
+ static const partSize = 20 * 1024 * 1024;
+ static const _maximumAttempts = 3;
+
+ final _dio = Dio(BaseOptions(connectTimeout: const Duration(seconds: 15)));
+ final _logger = Logger('MultipartUploader');
+
+ Future<String> upload(
+ File file,
+ int fileSize,
+ List<String> partMd5s,
+ MultipartUploadURLs urls,
+ ) async {
+ final partCount = (fileSize / partSize).ceil();
+ if (urls.partURLs.length != partCount || partMd5s.length != partCount) {
+ throw Exception('Multipart upload part count mismatch');
+ }
+
+ final parts = <String>[];
+ for (var i = 0; i < partCount; i++) {
+ final start = i * partSize;
+ final end = (start + partSize).clamp(0, fileSize);
+ _logger.info('Uploading part ${i + 1}/$partCount');
+ final response = await _retry(
+ () => _dio.put<String>(
+ urls.partURLs[i],
+ data: file.openRead(start, end),
+ options: Options(
+ responseType: ResponseType.plain,
+ contentType: 'application/octet-stream',
+ headers: {
+ Headers.contentLengthHeader: end - start,
+ 'Content-MD5': partMd5s[i],
+ },
+ ),
+ ),
+ );
+ final eTag = response.headers.value('etag');
+ if (eTag == null || eTag.isEmpty) {
+ throw Exception('ETAG_MISSING');
+ }
+ parts.add(
+ '<Part><PartNumber>${i + 1}</PartNumber>'
+ '<ETag>${const HtmlEscape(HtmlEscapeMode.element).convert(eTag)}</ETag></Part>',
+ );
+ }
+
+ await _retry(() async {
+ final response = await _dio.post<String>(
+ urls.completeURL,
+ data:
+ '<CompleteMultipartUpload>${parts.join()}</CompleteMultipartUpload>',
+ options: Options(
+ contentType: 'text/xml',
+ responseType: ResponseType.plain,
+ ),
+ );
+ if (!(response.data?.contains('<CompleteMultipartUploadResult') ??
+ false)) {
+ throw DioException(
+ requestOptions: response.requestOptions,
+ response: response,
+ type: DioExceptionType.badResponse,
+ message: 'Multipart upload completion failed',
+ );
+ }
+ return response;
+ });
+ return urls.objectKey;
+ }
+
+ Future<Response<String>> _retry(
+ Future<Response<String>> Function() request,
+ ) async {
+ for (var attempt = 1; ; attempt++) {
+ try {
+ return await request();
+ } on DioException {
+ if (attempt >= _maximumAttempts) rethrow;
+ _logger.info(
+ 'Upload request failed, retrying (${attempt + 1}/$_maximumAttempts)',
+ );
+ }
+ }
+ }
+}
+
+class MultipartUploadURLs {
+ final String objectKey;
+ final List<String> partURLs;
+ final String completeURL;
+
+ MultipartUploadURLs.fromMap(Map<String, dynamic> map)
+ : objectKey = map['objectKey'] as String,
+ partURLs = (map['partURLs'] as List).cast<String>(),
+ completeURL = map['completeURL'] as String;
+}