π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][locker] Add multipart uploads for internal users (#13729)
aman-pilotd4e2b53a0bb8
diff --git a/mobile/apps/locker/changes/multipart-upload.md b/mobile/apps/locker/changes/multipart-upload.mdnew file mode 100644index 0000000000..bb03ca6cce--- /dev/null+++ b/mobile/apps/locker/changes/multipart-upload.md@@ -0,0 +1 @@+- (i) Added multipart uploads with retries for large files.diff --git a/mobile/apps/locker/lib/services/feature_flag_service.dart b/mobile/apps/locker/lib/services/feature_flag_service.dartindex 0f4659397a..3d478eb8dc 100644--- a/mobile/apps/locker/lib/services/feature_flag_service.dart+++ b/mobile/apps/locker/lib/services/feature_flag_service.dart@@ -16,6 +16,7 @@ class FeatureFlagService {late FlagService _flagService;bool get internalUser => _flagService.internalUser;+ bool get multipartUpload => internalUser;bool get isInternalUserDisabled =>_preferences.getBool(_internalUserDisabled) ?? false;diff --git a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dartindex 8372199ceb..2a7b869db9 100644--- a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart+++ b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart@@ -17,12 +17,14 @@ import 'package:locker/core/errors.dart';import 'package:locker/events/backup_updated_event.dart';import 'package:locker/services/collections/models/collection.dart';import 'package:locker/services/configuration.dart';+import 'package:locker/services/feature_flag_service.dart';import "package:locker/services/files/sync/metadata_updater_service.dart";import 'package:locker/services/files/sync/models/file.dart';import 'package:locker/services/files/sync/models/file_magic.dart';import 'package:locker/services/files/upload/models/backup_item.dart';import 'package:locker/services/files/upload/models/backup_item_status.dart';import 'package:locker/services/files/upload/models/upload_url.dart';+import 'package:locker/services/files/upload/multipart_uploader.dart';import "package:locker/utils/crypto_helper.dart";import 'package:logging/logging.dart';import 'package:shared_preferences/shared_preferences.dart';@@ -39,6 +41,7 @@ class FileUploader {final _logger = Logger("FileUploader");final _dio = Network.instance.getDio();final _enteDio = Network.instance.enteDio;+ final _multipartUploader = MultipartUploader();final LinkedHashMap<String, FileUploadItem> _queue =LinkedHashMap<String, FileUploadItem>();final LinkedHashMap<String, BackupItem> _allBackups =@@ -302,11 +305,17 @@ class FileUploader {_checkIfWithinFileCountLimit();await _checkIfWithinStorageLimit(file);final encryptedFile = File(encryptedFilePath);+ final useMultipart =+ FeatureFlagService.instance.multipartUpload &&+ sourceFileSize > MultipartUploader.partSize;final fileAttributes = await CryptoUtil.encryptFileWithMd5(file.path,encryptedFilePath,key: key,+ multiPartChunkSizeInBytes: useMultipart+ ? MultipartUploader.partSize+ : null,);encFileSize = await encryptedFile.length();@@ -334,7 +343,7 @@ class FileUploader {final thumbnailMd5 = await computeMd5(encryptedThumbnailPath);final fileMd5 = fileAttributes.fileMd5;- if (fileMd5 == null || fileMd5.isEmpty) {+ if (!useMultipart && (fileMd5 == null || fileMd5.isEmpty)) {throw Exception('File MD5 hash is null or empty');}@@ -353,16 +362,33 @@ class FileUploader {thumbnailMd5,);- final fileUploadURL = await _getUploadURL(- contentLength: encFileSize,- md5: fileMd5,- );- final fileObjectKey = await _putFile(- fileUploadURL,- encryptedFile,- encFileSize,- fileMd5,- );+ final String fileObjectKey;+ if (useMultipart) {+ final urls = MultipartUploadURLs.fromMap(+ await _requestUploadURL('/files/multipart-upload-url', {+ 'contentLength': encFileSize,+ 'partLength': MultipartUploader.partSize,+ 'partMd5s': fileAttributes.partMd5s!,+ }),+ );+ fileObjectKey = await _multipartUploader.upload(+ encryptedFile,+ encFileSize,+ fileAttributes.partMd5s!,+ urls,+ );+ } else {+ final fileUploadURL = await _getUploadURL(+ contentLength: encFileSize,+ md5: fileMd5!,+ );+ fileObjectKey = await _putFile(+ fileUploadURL,+ encryptedFile,+ encFileSize,+ fileMd5,+ );+ }final enteFile = EnteFile.fromFile(file, fileName: fileName);@@ -650,12 +676,21 @@ class FileUploader {required int contentLength,required String md5,}) async {+ return UploadURL.fromMap(+ await _requestUploadURL('/files/upload-url', {+ 'contentLength': contentLength,+ 'contentMD5': md5,+ }),+ );+ }++ Future<Map<String, dynamic>> _requestUploadURL(+ String endpoint,+ Map<String, dynamic> data,+ ) async {try {- final response = await _enteDio.post(- "/files/upload-url",- data: {"contentLength": contentLength, "contentMD5": md5},- );- return UploadURL.fromMap((response.data as Map).cast<String, dynamic>());+ final response = await _enteDio.post(endpoint, data: data);+ return (response.data as Map).cast<String, dynamic>();} on DioException catch (e, s) {if (_isFileLimitReachedResponse(e.response)) {throw FileLimitReachedError();diff --git a/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart b/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dartnew file mode 100644index 0000000000..ac480aadad--- /dev/null+++ b/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart@@ -0,0 +1,103 @@+import 'dart:convert';+import 'dart:io';++import 'package:dio/dio.dart';+import 'package:logging/logging.dart';++class MultipartUploader {+ static const partSize = 20 * 1024 * 1024;+ static const _maximumAttempts = 3;++ final _dio = Dio(BaseOptions(connectTimeout: const Duration(seconds: 15)));+ final _logger = Logger('MultipartUploader');++ Future<String> upload(+ File file,+ int fileSize,+ List<String> partMd5s,+ MultipartUploadURLs urls,+ ) async {+ final partCount = (fileSize / partSize).ceil();+ if (urls.partURLs.length != partCount || partMd5s.length != partCount) {+ throw Exception('Multipart upload part count mismatch');+ }++ final parts = <String>[];+ for (var i = 0; i < partCount; i++) {+ final start = i * partSize;+ final end = (start + partSize).clamp(0, fileSize);+ _logger.info('Uploading part ${i + 1}/$partCount');+ final response = await _retry(+ () => _dio.put<String>(+ urls.partURLs[i],+ data: file.openRead(start, end),+ options: Options(+ responseType: ResponseType.plain,+ contentType: 'application/octet-stream',+ headers: {+ Headers.contentLengthHeader: end - start,+ 'Content-MD5': partMd5s[i],+ },+ ),+ ),+ );+ final eTag = response.headers.value('etag');+ if (eTag == null || eTag.isEmpty) {+ throw Exception('ETAG_MISSING');+ }+ parts.add(+ '<Part><PartNumber>${i + 1}</PartNumber>'+ '<ETag>${const HtmlEscape(HtmlEscapeMode.element).convert(eTag)}</ETag></Part>',+ );+ }++ await _retry(() async {+ final response = await _dio.post<String>(+ urls.completeURL,+ data:+ '<CompleteMultipartUpload>${parts.join()}</CompleteMultipartUpload>',+ options: Options(+ contentType: 'text/xml',+ responseType: ResponseType.plain,+ ),+ );+ if (!(response.data?.contains('<CompleteMultipartUploadResult') ??+ false)) {+ throw DioException(+ requestOptions: response.requestOptions,+ response: response,+ type: DioExceptionType.badResponse,+ message: 'Multipart upload completion failed',+ );+ }+ return response;+ });+ return urls.objectKey;+ }++ Future<Response<String>> _retry(+ Future<Response<String>> Function() request,+ ) async {+ for (var attempt = 1; ; attempt++) {+ try {+ return await request();+ } on DioException {+ if (attempt >= _maximumAttempts) rethrow;+ _logger.info(+ 'Upload request failed, retrying (${attempt + 1}/$_maximumAttempts)',+ );+ }+ }+ }+}++class MultipartUploadURLs {+ final String objectKey;+ final List<String> partURLs;+ final String completeURL;++ MultipartUploadURLs.fromMap(Map<String, dynamic> map)+ : objectKey = map['objectKey'] as String,+ partURLs = (map['partURLs'] as List).cast<String>(),+ completeURL = map['completeURL'] as String;+}