Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][locker] Remove internal multipart uploads (#14114)

aman-pilotc4afc5fd4f85
diff --git a/mobile/apps/locker/lib/services/feature_flag_service.dart b/mobile/apps/locker/lib/services/feature_flag_service.dart
index 51f547655c..3242e9cec0 100644
--- a/mobile/apps/locker/lib/services/feature_flag_service.dart
+++ b/mobile/apps/locker/lib/services/feature_flag_service.dart
@@ -16,7 +16,6 @@ class FeatureFlagService {
late FlagService _flagService;
bool get internalUser => _flagService.internalUser;
- bool get multipartUpload => internalUser;
bool get pdfViewer => internalUser;
bool get isInternalUserDisabled =>
_preferences.getBool(_internalUserDisabled) ?? false;
diff --git a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
index 2a7b869db9..8372199ceb 100644
--- a/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
+++ b/mobile/apps/locker/lib/services/files/upload/file_upload_service.dart
@@ -17,14 +17,12 @@ import 'package:locker/core/errors.dart';
import 'package:locker/events/backup_updated_event.dart';
import 'package:locker/services/collections/models/collection.dart';
import 'package:locker/services/configuration.dart';
-import 'package:locker/services/feature_flag_service.dart';
import "package:locker/services/files/sync/metadata_updater_service.dart";
import 'package:locker/services/files/sync/models/file.dart';
import 'package:locker/services/files/sync/models/file_magic.dart';
import 'package:locker/services/files/upload/models/backup_item.dart';
import 'package:locker/services/files/upload/models/backup_item_status.dart';
import 'package:locker/services/files/upload/models/upload_url.dart';
-import 'package:locker/services/files/upload/multipart_uploader.dart';
import "package:locker/utils/crypto_helper.dart";
import 'package:logging/logging.dart';
import 'package:shared_preferences/shared_preferences.dart';
@@ -41,7 +39,6 @@ class FileUploader {
final _logger = Logger("FileUploader");
final _dio = Network.instance.getDio();
final _enteDio = Network.instance.enteDio;
- final _multipartUploader = MultipartUploader();
final LinkedHashMap<String, FileUploadItem> _queue =
LinkedHashMap<String, FileUploadItem>();
final LinkedHashMap<String, BackupItem> _allBackups =
@@ -305,17 +302,11 @@ class FileUploader {
_checkIfWithinFileCountLimit();
await _checkIfWithinStorageLimit(file);
final encryptedFile = File(encryptedFilePath);
- final useMultipart =
- FeatureFlagService.instance.multipartUpload &&
- sourceFileSize > MultipartUploader.partSize;
final fileAttributes = await CryptoUtil.encryptFileWithMd5(
file.path,
encryptedFilePath,
key: key,
- multiPartChunkSizeInBytes: useMultipart
- ? MultipartUploader.partSize
- : null,
);
encFileSize = await encryptedFile.length();
@@ -343,7 +334,7 @@ class FileUploader {
final thumbnailMd5 = await computeMd5(encryptedThumbnailPath);
final fileMd5 = fileAttributes.fileMd5;
- if (!useMultipart && (fileMd5 == null || fileMd5.isEmpty)) {
+ if (fileMd5 == null || fileMd5.isEmpty) {
throw Exception('File MD5 hash is null or empty');
}
@@ -362,33 +353,16 @@ class FileUploader {
thumbnailMd5,
);
- final String fileObjectKey;
- if (useMultipart) {
- final urls = MultipartUploadURLs.fromMap(
- await _requestUploadURL('/files/multipart-upload-url', {
- 'contentLength': encFileSize,
- 'partLength': MultipartUploader.partSize,
- 'partMd5s': fileAttributes.partMd5s!,
- }),
- );
- fileObjectKey = await _multipartUploader.upload(
- encryptedFile,
- encFileSize,
- fileAttributes.partMd5s!,
- urls,
- );
- } else {
- final fileUploadURL = await _getUploadURL(
- contentLength: encFileSize,
- md5: fileMd5!,
- );
- fileObjectKey = await _putFile(
- fileUploadURL,
- encryptedFile,
- encFileSize,
- fileMd5,
- );
- }
+ final fileUploadURL = await _getUploadURL(
+ contentLength: encFileSize,
+ md5: fileMd5,
+ );
+ final fileObjectKey = await _putFile(
+ fileUploadURL,
+ encryptedFile,
+ encFileSize,
+ fileMd5,
+ );
final enteFile = EnteFile.fromFile(file, fileName: fileName);
@@ -676,21 +650,12 @@ class FileUploader {
required int contentLength,
required String md5,
}) async {
- return UploadURL.fromMap(
- await _requestUploadURL('/files/upload-url', {
- 'contentLength': contentLength,
- 'contentMD5': md5,
- }),
- );
- }
-
- Future<Map<String, dynamic>> _requestUploadURL(
- String endpoint,
- Map<String, dynamic> data,
- ) async {
try {
- final response = await _enteDio.post(endpoint, data: data);
- return (response.data as Map).cast<String, dynamic>();
+ final response = await _enteDio.post(
+ "/files/upload-url",
+ data: {"contentLength": contentLength, "contentMD5": md5},
+ );
+ return UploadURL.fromMap((response.data as Map).cast<String, dynamic>());
} on DioException catch (e, s) {
if (_isFileLimitReachedResponse(e.response)) {
throw FileLimitReachedError();
diff --git a/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart b/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart
deleted file mode 100644
index ac480aadad..0000000000
--- a/mobile/apps/locker/lib/services/files/upload/multipart_uploader.dart
+++ /dev/null
@@ -1,103 +0,0 @@
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:dio/dio.dart';
-import 'package:logging/logging.dart';
-
-class MultipartUploader {
- static const partSize = 20 * 1024 * 1024;
- static const _maximumAttempts = 3;
-
- final _dio = Dio(BaseOptions(connectTimeout: const Duration(seconds: 15)));
- final _logger = Logger('MultipartUploader');
-
- Future<String> upload(
- File file,
- int fileSize,
- List<String> partMd5s,
- MultipartUploadURLs urls,
- ) async {
- final partCount = (fileSize / partSize).ceil();
- if (urls.partURLs.length != partCount || partMd5s.length != partCount) {
- throw Exception('Multipart upload part count mismatch');
- }
-
- final parts = <String>[];
- for (var i = 0; i < partCount; i++) {
- final start = i * partSize;
- final end = (start + partSize).clamp(0, fileSize);
- _logger.info('Uploading part ${i + 1}/$partCount');
- final response = await _retry(
- () => _dio.put<String>(
- urls.partURLs[i],
- data: file.openRead(start, end),
- options: Options(
- responseType: ResponseType.plain,
- contentType: 'application/octet-stream',
- headers: {
- Headers.contentLengthHeader: end - start,
- 'Content-MD5': partMd5s[i],
- },
- ),
- ),
- );
- final eTag = response.headers.value('etag');
- if (eTag == null || eTag.isEmpty) {
- throw Exception('ETAG_MISSING');
- }
- parts.add(
- '<Part><PartNumber>${i + 1}</PartNumber>'
- '<ETag>${const HtmlEscape(HtmlEscapeMode.element).convert(eTag)}</ETag></Part>',
- );
- }
-
- await _retry(() async {
- final response = await _dio.post<String>(
- urls.completeURL,
- data:
- '<CompleteMultipartUpload>${parts.join()}</CompleteMultipartUpload>',
- options: Options(
- contentType: 'text/xml',
- responseType: ResponseType.plain,
- ),
- );
- if (!(response.data?.contains('<CompleteMultipartUploadResult') ??
- false)) {
- throw DioException(
- requestOptions: response.requestOptions,
- response: response,
- type: DioExceptionType.badResponse,
- message: 'Multipart upload completion failed',
- );
- }
- return response;
- });
- return urls.objectKey;
- }
-
- Future<Response<String>> _retry(
- Future<Response<String>> Function() request,
- ) async {
- for (var attempt = 1; ; attempt++) {
- try {
- return await request();
- } on DioException {
- if (attempt >= _maximumAttempts) rethrow;
- _logger.info(
- 'Upload request failed, retrying (${attempt + 1}/$_maximumAttempts)',
- );
- }
- }
- }
-}
-
-class MultipartUploadURLs {
- final String objectKey;
- final List<String> partURLs;
- final String completeURL;
-
- MultipartUploadURLs.fromMap(Map<String, dynamic> map)
- : objectKey = map['objectKey'] as String,
- partURLs = (map['partURLs'] as List).cast<String>(),
- completeURL = map['completeURL'] as String;
-}