Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Show upload progress and retry multipart 520s

Neeraj Gupta3fa6c638ce44
diff --git a/mobile/apps/photos/changes/backup-upload-progress.md b/mobile/apps/photos/changes/backup-upload-progress.md
new file mode 100644
index 0000000000..15b989c7f9
--- /dev/null
+++ b/mobile/apps/photos/changes/backup-upload-progress.md
@@ -0,0 +1 @@
+- Added per-file progress to Backup Status and improved multipart upload reliability.
diff --git a/mobile/apps/photos/lib/models/backup/backup_item.dart b/mobile/apps/photos/lib/models/backup/backup_item.dart
index e6ee832b5c..e990935c8b 100644
--- a/mobile/apps/photos/lib/models/backup/backup_item.dart
+++ b/mobile/apps/photos/lib/models/backup/backup_item.dart
@@ -1,44 +1,48 @@
import "package:photos/models/backup/backup_item_status.dart";
import "package:photos/models/file/file.dart";
-class _PreserveError {
- const _PreserveError();
-}
-
class BackupItem {
- static const _preserveError = _PreserveError();
-
final BackupItemStatus status;
final EnteFile file;
final int collectionID;
final Object? error;
+ final int? progressPercent;
BackupItem({
required this.status,
required this.file,
required this.collectionID,
Object? error,
- }) : error = status == BackupItemStatus.retry ? error : null;
-
- BackupItem copyWith({
- BackupItemStatus? status,
- EnteFile? file,
- int? collectionID,
- Object? error = _preserveError,
- }) {
- final nextStatus = status ?? this.status;
- final nextError = identical(error, _preserveError) ? this.error : error;
+ int? progressPercent,
+ }) : error = status == BackupItemStatus.retry ? error : null,
+ progressPercent = status == BackupItemStatus.uploading
+ ? progressPercent
+ : null;
+
+ BackupItem withStatus(BackupItemStatus status, {Object? error}) {
+ return BackupItem(
+ status: status,
+ file: file,
+ collectionID: collectionID,
+ error: error,
+ );
+ }
+
+ BackupItem withUploadProgress(int progressPercent) {
+ if (status != BackupItemStatus.uploading) {
+ throw StateError("Only an uploading backup can report progress");
+ }
return BackupItem(
- status: nextStatus,
- file: file ?? this.file,
- collectionID: collectionID ?? this.collectionID,
- error: nextStatus == BackupItemStatus.retry ? nextError : null,
+ status: status,
+ file: file,
+ collectionID: collectionID,
+ progressPercent: progressPercent,
);
}
@override
String toString() {
- return 'BackupItem(status: $status, file: $file, collectionID: $collectionID, error: $error)';
+ return 'BackupItem(status: $status, file: $file, collectionID: $collectionID, error: $error, progressPercent: $progressPercent)';
}
@override
@@ -48,7 +52,8 @@ class BackupItem {
return other.status == status &&
other.file == file &&
other.collectionID == collectionID &&
- other.error == error;
+ other.error == error &&
+ other.progressPercent == progressPercent;
}
@override
@@ -56,6 +61,7 @@ class BackupItem {
return status.hashCode ^
file.hashCode ^
collectionID.hashCode ^
- error.hashCode;
+ error.hashCode ^
+ progressPercent.hashCode;
}
}
diff --git a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
index ec7b934751..f6b1cf95e2 100644
--- a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
+++ b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
@@ -261,7 +261,13 @@ class FileUploader {
final collectionID = item.collectionID;
final stopwatch = Stopwatch()..start();
try {
- final uploadedFile = await _tryToUpload(file, collectionID, forcedUpload);
+ final uploadedFile = await _tryToUpload(
+ file,
+ collectionID,
+ forcedUpload,
+ onSendProgress: (sent, total) =>
+ _queue.updateBackupProgress(item, file.localID!, sent, total),
+ );
stopwatch.stop();
if (stopwatch.elapsed >= _longRunningUploadThreshold) {
_logger.info(
@@ -338,7 +344,19 @@ class FileUploader {
_queue.markBackupUploading(backupOwner, localID);
}
try {
- final result = await _tryToUpload(file, collectionID, true);
+ final result = await _tryToUpload(
+ file,
+ collectionID,
+ true,
+ onSendProgress: backupOwner == null
+ ? null
+ : (sent, total) => _queue.updateBackupProgress(
+ backupOwner,
+ localID,
+ sent,
+ total,
+ ),
+ );
if (backupOwner != null) {
_queue.markBackupUploaded(backupOwner, localID);
}
@@ -355,8 +373,9 @@ class FileUploader {
Future<EnteFile> _tryToUpload(
EnteFile file,
int collectionID,
- bool forcedUpload,
- ) async {
+ bool forcedUpload, {
+ ProgressCallback? onSendProgress,
+ }) async {
await checkNetworkForUpload(isForceUpload: forcedUpload);
if (!forcedUpload) {
final fileOnDisk = await FilesDB.instance.getFile(file.generatedID!);
@@ -583,6 +602,7 @@ class FileUploader {
encryptedFile,
encFileSize,
contentMd5: singlePartFileMd5,
+ onSendProgress: onSendProgress,
);
} else {
isMultipartUpload = true;
@@ -596,6 +616,7 @@ class FileUploader {
mediaUploadData.fileHash,
collectionID,
existingMultipartEncFileName,
+ onSendProgress: onSendProgress,
);
} else {
if (partMd5s == null || partMd5s.isEmpty) {
@@ -630,6 +651,7 @@ class FileUploader {
encFileSize,
fileMd5: fileMd5,
partMd5s: partMd5s,
+ onSendProgress: onSendProgress,
);
}
// in case of multipart, upload the thumbnail towards the end to avoid
diff --git a/mobile/apps/photos/lib/module/upload/service/multipart.dart b/mobile/apps/photos/lib/module/upload/service/multipart.dart
index d822397622..45ffe6c08b 100644
--- a/mobile/apps/photos/lib/module/upload/service/multipart.dart
+++ b/mobile/apps/photos/lib/module/upload/service/multipart.dart
@@ -16,6 +16,8 @@ import "package:photos/service_locator.dart";
import "package:photos/services/collections_service.dart";
class MultiPartUploader {
+ static const _maximumPartUploadAttempts = 3;
+
final Dio _s3Dio;
final UploadLocksDB _db;
final FlagService _featureFlagService;
@@ -158,8 +160,9 @@ class MultiPartUploader {
String localId,
String fileHash,
int collectionID,
- String encryptedFileName,
- ) async {
+ String encryptedFileName, {
+ ProgressCallback? onSendProgress,
+ }) async {
final multipartInfo = await _db.getCachedLinks(
localId,
fileHash,
@@ -173,7 +176,11 @@ class MultiPartUploader {
if (multipartInfo.status == MultipartStatus.pending) {
// upload individual parts and get their etags
try {
- etags = await _uploadParts(multipartInfo, encryptedFile);
+ etags = await _uploadParts(
+ multipartInfo,
+ encryptedFile,
+ onSendProgress: onSendProgress,
+ );
} on DioException catch (e) {
if (e.response?.statusCode == 404) {
_logger.severe(
@@ -189,6 +196,11 @@ class MultiPartUploader {
}
rethrow;
}
+ } else {
+ onSendProgress?.call(
+ multipartInfo.encFileSize,
+ multipartInfo.encFileSize,
+ );
}
if (multipartInfo.status != MultipartStatus.completed) {
@@ -225,6 +237,7 @@ class MultiPartUploader {
int fileSize, {
String? fileMd5,
List<String>? partMd5s,
+ ProgressCallback? onSendProgress,
}) async {
// upload individual parts and get their etags
final etags = await _uploadParts(
@@ -235,6 +248,7 @@ class MultiPartUploader {
partMd5s: partMd5s,
),
encryptedFile,
+ onSendProgress: onSendProgress,
);
// complete the multipart upload
@@ -245,8 +259,9 @@ class MultiPartUploader {
Future<Map<int, String>> _uploadParts(
MultipartInfo partInfo,
- File encryptedFile,
- ) async {
+ File encryptedFile, {
+ ProgressCallback? onSendProgress,
+ }) async {
final partsURLs = partInfo.urls.partsURLs;
final partUploadStatus = partInfo.partUploadStatus;
final partsLength = partsURLs.length;
@@ -279,6 +294,8 @@ class MultiPartUploader {
);
throw MultiPartError('multipart url count mismatch');
}
+ var completedBytes = (i * partSize).clamp(0, encFileLength).toInt();
+ onSendProgress?.call(completedBytes, encFileLength);
// Start parts upload
int count = 0;
while (i < partsLength) {
@@ -310,15 +327,42 @@ class MultiPartUploader {
headers["UPLOAD-URL"] = partURL;
}
+ var partBytesSent = 0;
try {
- final response = await _s3Dio.put(
- useUploadProxy ? "$kUploadProxyEndpoint/multipart-upload" : partURL,
- data: encryptedFile.openRead(
- i * partSize,
- isLastPart ? null : (i + 1) * partSize,
- ),
- options: Options(headers: headers),
- );
+ late final Response<dynamic> response;
+ for (var attempt = 1; ; attempt++) {
+ try {
+ response = await _s3Dio.put(
+ useUploadProxy
+ ? "$kUploadProxyEndpoint/multipart-upload"
+ : partURL,
+ data: encryptedFile.openRead(
+ i * partSize,
+ isLastPart ? null : (i + 1) * partSize,
+ ),
+ options: Options(headers: headers),
+ onSendProgress: (sent, _) {
+ if (sent > partBytesSent) {
+ partBytesSent = sent;
+ onSendProgress?.call(
+ completedBytes + partBytesSent,
+ encFileLength,
+ );
+ }
+ },
+ );
+ break;
+ } on DioException catch (error) {
+ if (error.response?.statusCode != 520 ||
+ attempt >= _maximumPartUploadAttempts) {
+ rethrow;
+ }
+ _logger.info(
+ "Multipart part PUT received HTTP 520; retrying immediately "
+ "(${attempt + 1}/$_maximumPartUploadAttempts)",
+ );
+ }
+ }
final eTag = useUploadProxy
? _extractProxyETag(response.data)
@@ -331,6 +375,8 @@ class MultiPartUploader {
etags[i] = eTag!;
await _db.updatePartStatus(partInfo.urls.objectKey, i, eTag);
+ completedBytes += fileSize;
+ onSendProgress?.call(completedBytes, encFileLength);
i++;
} on DioException catch (e) {
if (e.response?.statusCode == 400 &&
diff --git a/mobile/apps/photos/lib/module/upload/service/upload_queue.dart b/mobile/apps/photos/lib/module/upload/service/upload_queue.dart
index 0ebec45699..3ccb791f0f 100644
--- a/mobile/apps/photos/lib/module/upload/service/upload_queue.dart
+++ b/mobile/apps/photos/lib/module/upload/service/upload_queue.dart
@@ -188,6 +188,30 @@ class UploadQueue {
);
}
+ void updateBackupProgress(
+ Object owner,
+ String localID,
+ int bytesSent,
+ int totalBytes,
+ ) {
+ final item = _backupItems[localID];
+ if (!identical(_backupOwners[localID], owner) ||
+ item == null ||
+ item.status != BackupItemStatus.uploading ||
+ totalBytes <= 0) {
+ return;
+ }
+ final progressPercent = ((bytesSent * 100) ~/ totalBytes)
+ .clamp(0, 99)
+ .toInt();
+ if (progressPercent <= (item.progressPercent ?? 0)) {
+ return;
+ }
+ final updatedItem = item.withUploadProgress(progressPercent);
+ _backupItems[localID] = updatedItem;
+ _notifyBackupItemsChanged(upserts: {localID: updatedItem});
+ }
+
List<UploadQueueItem> get backgroundItems => _items.values
.where((item) => item._status == _UploadStatus.inBackground)
.toList();
@@ -250,10 +274,7 @@ class UploadQueue {
if (!identical(_backupOwners[localID], owner)) {
return null;
}
- final updatedItem = _backupItems[localID]!.copyWith(
- status: status,
- error: error,
- );
+ final updatedItem = _backupItems[localID]!.withStatus(status, error: error);
_backupItems[localID] = updatedItem;
return updatedItem;
}
diff --git a/mobile/apps/photos/lib/module/upload/service/upload_transport.dart b/mobile/apps/photos/lib/module/upload/service/upload_transport.dart
index 12a4df35fc..767471cbc7 100644
--- a/mobile/apps/photos/lib/module/upload/service/upload_transport.dart
+++ b/mobile/apps/photos/lib/module/upload/service/upload_transport.dart
@@ -60,6 +60,7 @@ class UploadTransport {
File file,
int fileSize, {
required String contentMd5,
+ ProgressCallback? onSendProgress,
}) async {
final uploadURL = await _getUploadURL(
contentLength: fileSize,
@@ -71,6 +72,7 @@ class UploadTransport {
fileSize,
contentMd5: contentMd5,
attempt: 1,
+ onSendProgress: onSendProgress,
);
}
@@ -129,6 +131,7 @@ class UploadTransport {
int fileSize, {
required String contentMd5,
required int attempt,
+ ProgressCallback? onSendProgress,
}) async {
if (contentMd5.isEmpty) {
throw StateError("Missing MD5 for checksum-protected upload");
@@ -148,8 +151,9 @@ class UploadTransport {
useUploadProxy ? "$kUploadProxyEndpoint/file-upload" : uploadURL.url,
data: file.openRead(),
options: Options(headers: headers),
- onSendProgress: (sent, total) {
+ onSendProgress: (sent, _) {
bytesSent = sent;
+ onSendProgress?.call(sent, fileSize);
},
);
_logger.info(
@@ -184,6 +188,7 @@ class UploadTransport {
fileSize,
contentMd5: contentMd5,
attempt: attempt + 1,
+ onSendProgress: onSendProgress,
);
}
_logger.info(
diff --git a/mobile/apps/photos/lib/ui/settings/backup/backup_item_card.dart b/mobile/apps/photos/lib/ui/settings/backup/backup_item_card.dart
index adb600d05e..df24f1b6e9 100644
--- a/mobile/apps/photos/lib/ui/settings/backup/backup_item_card.dart
+++ b/mobile/apps/photos/lib/ui/settings/backup/backup_item_card.dart
@@ -183,13 +183,9 @@ class _BackupItemCardState extends State<BackupItemCard> {
width: 48,
child: Center(
child: switch (widget.item.status) {
- BackupItemStatus.uploading => SizedBox(
- width: 16,
- height: 16,
- child: CircularProgressIndicator(
- strokeWidth: 2.0,
- color: colorScheme.primary700,
- ),
+ BackupItemStatus.uploading => _UploadProgressIndicator(
+ progressPercent: widget.item.progressPercent,
+ color: colorScheme.primary700,
),
BackupItemStatus.uploaded => const SizedBox(
width: 24,
@@ -234,3 +230,60 @@ class _BackupItemCardState extends State<BackupItemCard> {
);
}
}
+
+class _UploadProgressIndicator extends StatelessWidget {
+ const _UploadProgressIndicator({
+ required this.progressPercent,
+ required this.color,
+ });
+
+ static const _finalizingIndicatorValue = 0.995;
+
+ final int? progressPercent;
+ final Color color;
+
+ @override
+ Widget build(BuildContext context) {
+ final percent = progressPercent;
+ if (percent == null) {
+ return SizedBox(
+ width: 16,
+ height: 16,
+ child: CircularProgressIndicator(strokeWidth: 2, color: color),
+ );
+ }
+ return Semantics(
+ value: "$percent%",
+ excludeSemantics: true,
+ child: SizedBox(
+ width: 40,
+ height: 40,
+ child: Stack(
+ alignment: Alignment.center,
+ children: [
+ SizedBox(
+ width: 36,
+ height: 36,
+ child: CircularProgressIndicator(
+ value: percent == 99
+ ? _finalizingIndicatorValue
+ : percent / 100,
+ strokeWidth: 2,
+ color: color,
+ ),
+ ),
+ Text(
+ "$percent%",
+ textScaler: TextScaler.noScaling,
+ style: TextStyle(
+ color: color,
+ fontSize: 10,
+ fontWeight: FontWeight.w600,
+ ),
+ ),
+ ],
+ ),
+ ),
+ );
+ }
+}
diff --git a/mobile/apps/photos/test/models/backup/backup_item_test.dart b/mobile/apps/photos/test/models/backup/backup_item_test.dart
index b7f54080a8..748b8d7aef 100644
--- a/mobile/apps/photos/test/models/backup/backup_item_test.dart
+++ b/mobile/apps/photos/test/models/backup/backup_item_test.dart
@@ -4,7 +4,7 @@ import 'package:photos/models/backup/backup_item_status.dart';
import 'package:photos/models/file/file.dart';
void main() {
- test('copyWith can explicitly clear a retry error', () {
+ test('status transitions replace retry details', () {
final error = StateError('retry');
final retryItem = BackupItem(
status: BackupItemStatus.retry,
@@ -13,14 +13,14 @@ void main() {
error: error,
);
- expect(retryItem.copyWith().error, same(error));
- expect(retryItem.copyWith(error: null).error, isNull);
-
const replacementError = Object();
expect(
- retryItem.copyWith(error: replacementError).error,
+ retryItem
+ .withStatus(BackupItemStatus.retry, error: replacementError)
+ .error,
same(replacementError),
);
+ expect(retryItem.withStatus(BackupItemStatus.retry).error, isNull);
});
test('non-retry statuses cannot retain an error', () {
@@ -31,9 +31,7 @@ void main() {
error: StateError('retry'),
);
- final uploadingItem = retryItem.copyWith(
- status: BackupItemStatus.uploading,
- );
+ final uploadingItem = retryItem.withStatus(BackupItemStatus.uploading);
expect(uploadingItem.error, isNull);
});