π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Upgrade mobile FFmpeg to 9.0.2
Neeraj Guptaf9cecc40ff78
diff --git a/apple/packages/FFmpegRuntime/Package.swift b/apple/packages/FFmpegRuntime/Package.swiftnew file mode 100644index 0000000000..d4c5545a46--- /dev/null+++ b/apple/packages/FFmpegRuntime/Package.swift@@ -0,0 +1,19 @@+// swift-tools-version: 5.9++import PackageDescription++let package = Package(+ name: "FFmpegRuntime",+ platforms: [.iOS("15.1")],+ products: [+ .library(name: "FFmpegRuntime", targets: ["FFmpegRuntime"])+ ],+ targets: [+ .binaryTarget(+ name: "FFmpegRuntime",+ url:+ "https://github.com/ente/ffmpeg-packaging/releases/download/9.0.2/ios-FFmpegRuntime.xcframework.zip",+ checksum: "2c3923ac301d3deea5df47ab86d5479c5a95cb09b90eae797910498afd554844"+ )+ ]+)diff --git a/mobile/apps/photos/android/build.gradle b/mobile/apps/photos/android/build.gradleindex f4b978a79c..4d04802839 100644--- a/mobile/apps/photos/android/build.gradle+++ b/mobile/apps/photos/android/build.gradle@@ -6,10 +6,6 @@ allprojects {jcenter()mavenCentral()maven { url 'https://jitpack.io' }- // mavenLocal() // for FDroid- maven {- url "${project(':ffmpeg_kit_flutter').projectDir}/libs"- }}}diff --git a/mobile/apps/photos/ios/Podfile b/mobile/apps/photos/ios/Podfileindex 17a3461222..7e730426f6 100644--- a/mobile/apps/photos/ios/Podfile+++ b/mobile/apps/photos/ios/Podfile@@ -1,4 +1,3 @@-source 'https://github.com/ente/ffmpeg-kit-custom-repo-ios.git'platform :ios, '15.1'source 'https://cdn.cocoapods.org/'diff --git a/mobile/apps/photos/ios/Podfile.lock b/mobile/apps/photos/ios/Podfile.lockindex 9537d3a100..2339cc0f68 100644--- a/mobile/apps/photos/ios/Podfile.lock+++ b/mobile/apps/photos/ios/Podfile.lock@@ -32,10 +32,6 @@ PODS:- ente_vision (0.0.1):- Flutter- EnteOnnxRuntime (1.28.1-r1)- - ffmpeg_kit_custom (6.0.3)- - ffmpeg_kit_flutter (6.0.3):- - ffmpeg_kit_custom- - Flutter- file_saver (0.0.1):- Flutter- Firebase/CoreOnly (11.15.0):@@ -259,7 +255,6 @@ DEPENDENCIES:- ente_screen_cover (from `.symlinks/plugins/ente_screen_cover/ios`)- ente_vision (from `.symlinks/plugins/ente_vision/ios`)- EnteOnnxRuntime (from `../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec`)- - ffmpeg_kit_flutter (from `.symlinks/plugins/ffmpeg_kit_flutter/ios`)- file_saver (from `.symlinks/plugins/file_saver/ios`)- firebase_core (from `.symlinks/plugins/firebase_core/ios`)- firebase_messaging (from `.symlinks/plugins/firebase_messaging/ios`)@@ -302,8 +297,6 @@ DEPENDENCIES:- workmanager_apple (from `.symlinks/plugins/workmanager_apple/ios`)SPEC REPOS:- https://github.com/ente/ffmpeg-kit-custom-repo-ios.git:- - ffmpeg_kit_customtrunk:- Firebase- FirebaseCore@@ -357,8 +350,6 @@ EXTERNAL SOURCES::path: ".symlinks/plugins/ente_vision/ios"EnteOnnxRuntime::podspec: "../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec"- ffmpeg_kit_flutter:- :path: ".symlinks/plugins/ffmpeg_kit_flutter/ios"file_saver::path: ".symlinks/plugins/file_saver/ios"firebase_core:@@ -458,8 +449,6 @@ SPEC CHECKSUMS:ente_screen_cover: 6792566774f1c83939f9c1d03ed0a1f1ded45ae4ente_vision: 260965fb4a3bda0c0cb72c5f15eb8748fef7c897EnteOnnxRuntime: 7baa83a95717a6c9c8d0d4599f9f2c90b72d2905- ffmpeg_kit_custom: 682b4f2f1ff1f8abae5a92f6c3540f2441d5be99- ffmpeg_kit_flutter: 915b345acc97d4142e8a9a8549d177ff10f043f5file_saver: 6cdbcddd690cb02b0c1a0c225b37cd805c2bf8b6Firebase: d99ac19b909cd2c548339c2241ecd0d1599ab02efirebase_core: ece862f94b2bc72ee0edbeec7ab5c7cb09fe1ab5@@ -517,6 +506,6 @@ SPEC CHECKSUMS:wakelock_plus: e29112ab3ef0b318e58cfa5c32326458be66b556workmanager_apple: 904529ae31e97fc5be632cf628507652294a0778-PODFILE CHECKSUM: a3f986a49675b93a48b8f2861dc5afb5e8d21284+PODFILE CHECKSUM: 1e63981d1e734d82b86f2c648779cacefb592a64COCOAPODS: 1.17.0diff --git a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxprojindex fc0e7fc834..4f780b98fd 100644--- a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj+++ b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj@@ -29,6 +29,7 @@CEE6BE702D7AE7FD00E4048B /* WidgetKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6DACD83C2B755B0600BA9516 /* WidgetKit.framework */; };CEE6BE712D7AE7FD00E4048B /* SwiftUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6DACD83E2B755B0600BA9516 /* SwiftUI.framework */; };CEE6BE7C2D7AE7FE00E4048B /* EnteMemoryWidgetExtension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = CEE6BE6F2D7AE7FD00E4048B /* EnteMemoryWidgetExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; };+ D3E09F452C70B1E300F01A01 /* FFmpegRuntime in Frameworks */ = {isa = PBXBuildFile; productRef = D3E09F452C70B1E300F01A03 /* FFmpegRuntime */; };DA6BE5E826B3BC8600656280 /* BuildFile in Resources */ = {isa = PBXBuildFile; };/* End PBXBuildFile section */@@ -225,6 +226,7 @@isa = PBXFrameworksBuildPhase;buildActionMask = 2147483647;files = (+ D3E09F452C70B1E300F01A01 /* FFmpegRuntime in Frameworks */,274DC65125920DD100F526A4 /* StoreKit.framework in Frameworks */,0856877A9F80F9E052DBFDB4 /* Pods_Runner.framework in Frameworks */,);@@ -403,6 +405,9 @@54D121F52F02675C00B8CC69 /* PBXTargetDependency */,);name = Runner;+ packageProductDependencies = (+ D3E09F452C70B1E300F01A03 /* FFmpegRuntime */,+ );productName = Runner;productReference = 97C146EE1CF9000F007C117D /* Runner.app */;productType = "com.apple.product-type.application";@@ -503,6 +508,9 @@Base,);mainGroup = 97C146E51CF9000F007C117D;+ packageReferences = (+ D3E09F452C70B1E300F01A02 /* XCLocalSwiftPackageReference "FFmpegRuntime" */,+ );productRefGroup = 97C146EF1CF9000F007C117D /* Products */;projectDirPath = "";projectRoot = "";@@ -688,14 +696,6 @@"${BUILT_PRODUCTS_DIR}/video_player_avfoundation/video_player_avfoundation.framework","${BUILT_PRODUCTS_DIR}/wakelock_plus/wakelock_plus.framework","${BUILT_PRODUCTS_DIR}/workmanager_apple/workmanager_apple.framework",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/ffmpegkit.framework/ffmpegkit",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libavcodec.framework/libavcodec",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libavdevice.framework/libavdevice",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libavfilter.framework/libavfilter",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libavformat.framework/libavformat",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libavutil.framework/libavutil",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libswresample.framework/libswresample",- "${PODS_XCFRAMEWORKS_BUILD_DIR}/ffmpeg_kit_custom/libswscale.framework/libswscale","${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Ass.framework/Ass","${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Avcodec.framework/Avcodec","${PODS_XCFRAMEWORKS_BUILD_DIR}/media_kit_libs_ios_video/Avfilter.framework/Avfilter",@@ -784,14 +784,6 @@"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/video_player_avfoundation.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/wakelock_plus.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/workmanager_apple.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ffmpegkit.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavcodec.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavdevice.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavfilter.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavformat.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavutil.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libswresample.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libswscale.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Ass.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Avcodec.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Avfilter.framework",@@ -1818,6 +1810,20 @@defaultConfigurationName = Release;};/* End XCConfigurationList section */++/* Begin XCLocalSwiftPackageReference section */+ D3E09F452C70B1E300F01A02 /* XCLocalSwiftPackageReference "FFmpegRuntime" */ = {+ isa = XCLocalSwiftPackageReference;+ relativePath = ../../../../apple/packages/FFmpegRuntime;+ };+/* End XCLocalSwiftPackageReference section */++/* Begin XCSwiftPackageProductDependency section */+ D3E09F452C70B1E300F01A03 /* FFmpegRuntime */ = {+ isa = XCSwiftPackageProductDependency;+ productName = FFmpegRuntime;+ };+/* End XCSwiftPackageProductDependency section */};rootObject = 97C146E61CF9000F007C117D /* Project object */;}diff --git a/mobile/apps/photos/lib/core/configuration.dart b/mobile/apps/photos/lib/core/configuration.dartindex 7f35117f89..406a13986f 100644--- a/mobile/apps/photos/lib/core/configuration.dart+++ b/mobile/apps/photos/lib/core/configuration.dart@@ -197,9 +197,7 @@ class Configuration implements LockScreenHost, AccountDeletionHost {_logger.info("Logging out, autoLogout: $autoLogout");MLService.instance.stopActiveRun(MlStopReason.logout);if (!autoLogout) {- if (flagService.stopStreamProcess) {- VideoPreviewService.instance.stop('logout');- }+ VideoPreviewService.instance.stop('logout');if (SyncService.instance.isSyncInProgress()) {SyncService.instance.stopSync();try {diff --git a/mobile/apps/photos/lib/main.dart b/mobile/apps/photos/lib/main.dartindex 6f5a2d0b5b..16e6e192a8 100644--- a/mobile/apps/photos/lib/main.dart+++ b/mobile/apps/photos/lib/main.dart@@ -13,7 +13,6 @@ import "package:ente_lock_screen/ui/lock_screen.dart";import "package:ente_pure_utils/ente_pure_utils.dart";import "package:ente_strings/ente_strings.dart";import "package:ente_ui/theme/theme_config.dart" as ente_ui;-import "package:ffmpeg_kit_flutter/ffmpeg_kit_config.dart";import 'package:flutter/foundation.dart';import "package:flutter/gestures.dart";import 'package:flutter/material.dart';@@ -124,7 +123,6 @@ void main() async {}});}- _initializeFFmpegKit().ignore();await rive.RiveNative.init();MediaKit.ensureInitialized();@@ -142,11 +140,6 @@ void main() async {unawaited(SystemChrome.setEnabledSystemUIMode(SystemUiMode.edgeToEdge));}-Future<void> _initializeFFmpegKit() async {- await FFmpegKitConfig.init();- await FFmpegKitConfig.disableLogs();-}-Future<void> _runInForeground(AdaptiveThemeMode? savedThemeMode,MediaExtentionAction initialMediaExtensionAction,diff --git a/mobile/apps/photos/lib/models/ffmpeg/ffprobe_props.dart b/mobile/apps/photos/lib/models/ffmpeg/ffprobe_props.dartindex 157cb8856b..b9cb5027e4 100644--- a/mobile/apps/photos/lib/models/ffmpeg/ffprobe_props.dart+++ b/mobile/apps/photos/lib/models/ffmpeg/ffprobe_props.dart@@ -181,33 +181,20 @@ class FFProbeProps {if (key == FFProbeKeys.rFrameRate) {result.fps = _formatFPS(stream[key]);parsedData[key] = result.fps;- }- //TODO: Use `height` and `width` instead of `codedHeight` and `codedWidth`- //for better accuracy. `height' and `width` will give the video's "visual"- //height and width.- else if (key == FFProbeKeys.codedWidth) {- final width = stream[key];- if (width != null && width != 0) {- result._width = width.toString();- parsedData[key] = result._width;- }- } else if (key == FFProbeKeys.codedHeight) {- final height = stream[key];- if (height != null && height != 0) {- result._height = height.toString();- parsedData[key] = result._height;- }- } else if (key == FFProbeKeys.width) {- final width = stream[key];+ } else if (key == FFProbeKeys.width || key == FFProbeKeys.codedWidth) {+ final width =+ stream[FFProbeKeys.width] ?? stream[FFProbeKeys.codedWidth];if (width != null && width != 0) {result._width = width.toString();- parsedData[key] = result._width;+ parsedData[FFProbeKeys.width] = result._width;}- } else if (key == FFProbeKeys.height) {- final height = stream[key];+ } else if (key == FFProbeKeys.height ||+ key == FFProbeKeys.codedHeight) {+ final height =+ stream[FFProbeKeys.height] ?? stream[FFProbeKeys.codedHeight];if (height != null && height != 0) {result._height = height.toString();- parsedData[key] = result._height;+ parsedData[FFProbeKeys.height] = result._height;}} else if (key == FFProbeKeys.sideDataList) {for (Map sideData in stream[key]) {diff --git a/mobile/apps/photos/lib/module/metadata/video.dart b/mobile/apps/photos/lib/module/metadata/video.dartindex 283190066b..594dea600b 100644--- a/mobile/apps/photos/lib/module/metadata/video.dart+++ b/mobile/apps/photos/lib/module/metadata/video.dart@@ -2,16 +2,14 @@ import "dart:io";import "package:logging/logging.dart";import "package:photos/models/ffmpeg/ffprobe_props.dart";-import "package:photos/services/isolated_ffmpeg_service.dart";+import "package:photos/services/ffmpeg_service.dart";final _logger = Logger("VideoMetadata");Future<FFProbeProps?> getVideoProps(File file) async {try {final stopwatch = Stopwatch()..start();- final mediaInfo = await IsolatedFfmpegService.instance.getVideoInfo(- file.path,- );+ final mediaInfo = await FfmpegService.instance.getVideoInfo(file.path);if (mediaInfo.isEmpty) {return null;}diff --git a/mobile/apps/photos/lib/services/ffmpeg_service.dart b/mobile/apps/photos/lib/services/ffmpeg_service.dartnew file mode 100644index 0000000000..3def5b5f21--- /dev/null+++ b/mobile/apps/photos/lib/services/ffmpeg_service.dart@@ -0,0 +1,96 @@+import 'dart:async';+import 'dart:collection';+import 'dart:ui';++import 'package:ffmpeg/ffmpeg.dart' as ffmpeg;+import 'package:photos/utils/ffprobe_util.dart';++class FfmpegJob {+ FfmpegJob._(this.arguments, this._onProgress);++ final List<String> arguments;+ final void Function(Duration)? _onProgress;+ final _completion = Completer<ffmpeg.Result>();+ void Function()? _cancelNative;+ bool _cancelled = false;+ Duration? _processedTime;++ Future<ffmpeg.Result> get completed => _completion.future;++ double? progress(Duration? duration) {+ if (duration == null || duration.inMicroseconds <= 0) return null;+ final time = _processedTime;+ if (time == null) return null;+ return (time.inMicroseconds / duration.inMicroseconds).clamp(0.0, 1.0);+ }++ Future<void> cancel() async {+ if (!_completion.isCompleted && !_cancelled) {+ _cancelled = true;+ if (_cancelNative case final cancel?) {+ cancel();+ } else {+ _completion.complete(const ffmpeg.Result(255, ''));+ }+ }+ await completed.then<void>((_) {}, onError: (Object _, StackTrace _) {});+ }++ void _updateProgress(Duration time) {+ if (_cancelled || _completion.isCompleted) return;+ _processedTime = time;+ _onProgress?.call(time);+ }+}++class FfmpegService {+ FfmpegService({+ ffmpeg.Session Function(void Function(Duration)) createSession =+ ffmpeg.Session.new,+ }) : _createSession = createSession;++ static final instance = FfmpegService();++ final ffmpeg.Session Function(void Function(Duration)) _createSession;+ final _queue = Queue<FfmpegJob>();+ bool _running = false;++ FfmpegJob start(+ List<String> arguments, {+ void Function(Duration)? onProgress,+ }) {+ if (RootIsolateToken.instance == null) {+ throw StateError('Submit FFmpeg jobs from the root isolate.');+ }+ final job = FfmpegJob._(List.unmodifiable(arguments), onProgress);+ _queue.add(job);+ unawaited(_drain());+ return job;+ }++ Future<void> _drain() async {+ if (_running) return;+ _running = true;+ try {+ while (_queue.isNotEmpty) {+ final job = _queue.removeFirst();+ if (job._cancelled) continue;+ try {+ final session = _createSession(job._updateProgress);+ job._cancelNative = session.cancel;+ final result = await session.execute(job.arguments);+ job._completion.complete(result);+ } catch (error, stack) {+ job._completion.completeError(error, stack);+ } finally {+ job._cancelNative = null;+ }+ }+ } finally {+ _running = false;+ }+ }++ Future<Map> getVideoInfo(String path) async =>+ FFProbeUtil.getMetadata(await ffmpeg.probeMedia(path));+}diff --git a/mobile/apps/photos/lib/services/isolated_ffmpeg_service.dart b/mobile/apps/photos/lib/services/isolated_ffmpeg_service.dartdeleted file mode 100644index 5f003f738a..0000000000--- a/mobile/apps/photos/lib/services/isolated_ffmpeg_service.dart+++ /dev/null@@ -1,126 +0,0 @@-import "dart:async";-import "dart:isolate";-import "package:ffmpeg_kit_flutter/ffmpeg_kit.dart";-import "package:ffmpeg_kit_flutter/ffmpeg_kit_config.dart";-import "package:ffmpeg_kit_flutter/ffmpeg_session.dart";-import "package:ffmpeg_kit_flutter/ffprobe_kit.dart";-import "package:flutter/services.dart";-import "package:photos/service_locator.dart";-import "package:photos/utils/ffprobe_util.dart";--class IsolatedFfmpegService {- IsolatedFfmpegService._privateConstructor();-- static final IsolatedFfmpegService instance =- IsolatedFfmpegService._privateConstructor();-- Future<Map> runFfmpeg(String command) async {- final rootIsolateToken = RootIsolateToken.instance!;- return await Isolate.run<Map>(() => _ffmpegRun(command, rootIsolateToken));- }-- Future<Map> runFfmpegCancellable(- String command,- void Function(int sessionId) onSessionStarted,- ) async {- if (!flagService.stopStreamProcess) {- return await runFfmpeg(command);- }-- final rootIsolateToken = RootIsolateToken.instance!;- final port = ReceivePort();- final completionPort = ReceivePort();- final completer = Completer<Map>();-- port.listen((msg) {- if (msg is int) {- FFmpegKit.registerSessionCompletionPort(msg, completionPort.sendPort);- onSessionStarted(msg);- }- });-- // ignore: unawaited_futures- completionPort.first.then((result) {- if (result is Map) {- completer.complete({- "returnCode": result["returnCode"],- "output": result["output"],- });- }- completionPort.close();- port.close();- });-- await Isolate.spawn(_ffmpegRunCancellable, (- command,- port.sendPort,- rootIsolateToken,- ));-- return completer.future;- }-- Future<Map> getVideoInfo(String file) async {- final rootIsolateToken = RootIsolateToken.instance!;- return await Isolate.run<Map>(() => _getVideoProps(file, rootIsolateToken));- }-- Future<double?> getSessionProgress({- required int? sessionId,- required Duration? duration,- }) async {- if (sessionId == null || duration == null) return null;- if (duration.inMilliseconds <= 0) return null;-- final session = await FFmpegKitConfig.getSession(sessionId);- if (session == null || session is! FFmpegSession) return null;-- final stats = await session.getStatistics();- if (stats.isEmpty) return null;-- final ms = stats.last.getTime().toDouble();- return (ms / duration.inMilliseconds).clamp(0.0, 1.0);- }-}--@pragma('vm:entry-point')-Future<Map> _getVideoProps(- String filePath,- RootIsolateToken rootIsolateToken,-) async {- BackgroundIsolateBinaryMessenger.ensureInitialized(rootIsolateToken);- // This path may run before or without the app-level initializer, so disable- // live log forwarding before probing to keep FFprobe output off the UI thread.- await FFmpegKitConfig.disableLogs();- final session = await FFprobeKit.getMediaInformation(filePath);- final mediaInfo = session.getMediaInformation();-- if (mediaInfo == null) {- return {};- }-- final metadata = await FFProbeUtil.getMetadata(mediaInfo);- return metadata;-}--@pragma('vm:entry-point')-Future<Map> _ffmpegRun(String value, RootIsolateToken rootIsolateToken) async {- BackgroundIsolateBinaryMessenger.ensureInitialized(rootIsolateToken);- final session = await FFmpegKit.execute(value, true);- final returnCode = await session.getReturnCode();- final output = await session.getOutput();-- return {"returnCode": returnCode?.getValue(), "output": output};-}--@pragma('vm:entry-point')-Future<void> _ffmpegRunCancellable(- (String, SendPort, RootIsolateToken) params,-) async {- final (command, sendPort, token) = params;- BackgroundIsolateBinaryMessenger.ensureInitialized(token);-- final session = await FFmpegKit.executeAsync(command);- final sessionId = session.getSessionId();- if (sessionId != null) sendPort.send(sessionId);-}diff --git a/mobile/apps/photos/lib/services/video_preview_service.dart b/mobile/apps/photos/lib/services/video_preview_service.dartindex bb13417e74..fc6e8bfdd0 100644--- a/mobile/apps/photos/lib/services/video_preview_service.dart+++ b/mobile/apps/photos/lib/services/video_preview_service.dart@@ -7,8 +7,7 @@ import "package:collection/collection.dart";import "package:dio/dio.dart";import "package:encrypt/encrypt.dart" as enc;import "package:ente_strings/ente_strings.dart";-import "package:ffmpeg_kit_flutter/ffmpeg_kit.dart";-import "package:ffmpeg_kit_flutter/return_code.dart";+import "package:ffmpeg/ffmpeg.dart" as ffmpeg;import "package:flutter/foundation.dart";import "package:flutter/widgets.dart";import "package:flutter_cache_manager/flutter_cache_manager.dart";@@ -37,10 +36,10 @@ import "package:photos/module/download/file.dart";import "package:photos/module/metadata/video.dart";import "package:photos/module/upload/service/file_uploader.dart";import "package:photos/service_locator.dart";+import "package:photos/services/ffmpeg_service.dart";import "package:photos/services/file_magic_service.dart";import "package:photos/services/filedata/model/file_data.dart";import "package:photos/services/filedata/preview_upload.dart";-import "package:photos/services/isolated_ffmpeg_service.dart";import "package:photos/services/machine_learning/compute_controller.dart";import "package:photos/ui/notification/toast.dart";import "package:photos/utils/file_key.dart";@@ -49,8 +48,6 @@ import "package:photos/utils/hls_playlist.dart";import "package:photos/utils/network_util.dart";const _maxRetryCount = 3;-const _maxFfmpegOutputLines = 24;-const _maxFfmpegOutputChars = 4000;const _missingVideoStreamError = "No video stream found in FFprobe metadata";class VideoPreviewService {@@ -66,14 +63,13 @@ class VideoPreviewService {: serviceLocator = ServiceLocator.instance,filesDB = FilesDB.instance,uploadLocksDB = UploadLocksDB.instance,- ffmpegService = IsolatedFfmpegService.instance,+ ffmpegService = FfmpegService.instance,fileMagicService = FileMagicService.instance,cacheManager = DefaultCacheManager(),videoCacheManager = VideoCacheManager.instance,+ _computeController = null,config = Configuration.instance {- if (flagService.stopStreamProcess) {- _registerStopSafelyListeners();- }+ _registerStopSafelyListeners();}void _registerStopSafelyListeners() {@@ -98,24 +94,28 @@ class VideoPreviewService {this.fileMagicService,this.ffmpegService,this.cacheManager,- this.videoCacheManager,- );+ this.videoCacheManager, {+ ComputeController? compute,+ }) : _computeController = compute;static final VideoPreviewService instance =VideoPreviewService._privateConstructor();int uploadingFileId = -1;CancelToken? _streamingCancelToken;- int? _currentFfmpegSessionId;+ FfmpegJob? _currentFfmpegJob;final Configuration config;final ServiceLocator serviceLocator;final FilesDB filesDB;final UploadLocksDB uploadLocksDB;final FileMagicService fileMagicService;- final IsolatedFfmpegService ffmpegService;+ final FfmpegService ffmpegService;final DefaultCacheManager cacheManager;final CacheManager videoCacheManager;+ final ComputeController? _computeController;++ ComputeController get _compute => _computeController ?? computeController;FileDataGateway get _fileDataGateway => fileDataGateway;@@ -140,37 +140,44 @@ class VideoPreviewService {}}- void clearQueue() {- final status = flagService.stopStreamProcess- ? PreviewItemStatus.paused- : PreviewItemStatus.uploaded;- for (final fileId in _items.keys) {- _fireVideoPreviewStateChange(fileId, status);+ void _clearQueue() {+ for (final entry in _items.entries) {+ _fireVideoPreviewStateChange(+ entry.key,+ entry.value.status == PreviewItemStatus.uploaded+ ? PreviewItemStatus.uploaded+ : PreviewItemStatus.paused,+ );}fileQueue.clear();_items.clear();}void stop(String reason) {- if (_items.isEmpty) return;-_logger.info("Stopping streaming: $reason");- if (flagService.stopStreamProcess) {- _streamingCancelToken?.cancel();- }+ _streamingCancelToken?.cancel();uploadingFileId = -1;- clearQueue();- computeController.releaseCompute(stream: true);+ _clearQueue();+ _compute.releaseCompute(stream: true);- if (flagService.stopStreamProcess && _currentFfmpegSessionId != null) {- unawaited(FFmpegKit.cancel(_currentFfmpegSessionId!));- _currentFfmpegSessionId = null;+ if (_currentFfmpegJob != null) {+ unawaited(_currentFfmpegJob!.cancel());}}- Future<void> stopSafely(String reason) async {- if (!flagService.stopStreamProcess) return;+ Future<void> pauseForExport() async {+ _compute.blockCompute(blocker: "video-export");+ final job = _currentFfmpegJob;+ if (job != null) stop("video export");+ await job?.cancel();+ }++ void resumeAfterExport() {+ _compute.unblockCompute(blocker: "video-export");+ queueFiles();+ }+ Future<void> stopSafely(String reason) async {final item = uploadingFileId >= 0 ? _items[uploadingFileId] : null;if (item == null) return;@@ -179,11 +186,8 @@ class VideoPreviewService {if (status == PreviewItemStatus.compressing) {final durationInSeconds = item.file.duration;- progress = await ffmpegService.getSessionProgress(- sessionId: _currentFfmpegSessionId,- duration: durationInSeconds == null- ? null- : Duration(seconds: durationInSeconds),+ progress = _currentFfmpegJob?.progress(+ durationInSeconds == null ? null : Duration(seconds: durationInSeconds),);}@@ -361,9 +365,10 @@ class VideoPreviewService {bool forceUpload = false,}) async {if (_items.isEmpty) return;- _streamingCancelToken ??= CancelToken();+ final cancelToken = _streamingCancelToken ??= CancelToken();+ if (cancelToken.isCancelled) return;- if (flagService.stopStreamProcess && FileUploader.instance.isUploading) {+ if (FileUploader.instance.isUploading) {stop("upload in progress");return;}@@ -375,6 +380,7 @@ class VideoPreviewService {Object? error;bool removeFile = false;+ String? previewDirectory;try {if (!enteFile.isUploaded) {removeFile = true;@@ -400,16 +406,19 @@ class VideoPreviewService {return;}}+ if (cancelToken.isCancelled) return;_logger.info("Starting video preview generation for ${enteFile.displayName}",);final isManual = await uploadLocksDB.isInStreamQueue(enteFile.uploadedFileID!,);+ if (cancelToken.isCancelled) return;var (props, result, file) = await _checkFileForPreviewCreation(enteFile,isManual,);+ if (cancelToken.isCancelled) return;if (result) {removeFile = true;return;@@ -449,13 +458,18 @@ class VideoPreviewService {);file ??= await getFile(enteFile, isOrigin: true);- if (_items.isEmpty) return;+ if (cancelToken.isCancelled || _items.isEmpty) return;if (file == null) {error = "Unable to fetch file";return;}props ??= await getVideoProps(file);+ if (cancelToken.isCancelled ||+ _items.isEmpty ||+ !_isPermissionGranted()) {+ return;+ }final fileSize = enteFile.fileSize ?? file.lengthSync();if (props == null) {@@ -494,6 +508,7 @@ class VideoPreviewService {final String tempDir = config.getTempDirectory();final String prefix ="${tempDir}_${enteFile.uploadedFileID}_${newID("pv")}";+ previewDirectory = prefix;Directory(prefix).createSync();_logger.info('Compressing video ${enteFile.displayName}');final key = enc.Key.fromLength(16);@@ -517,11 +532,9 @@ class VideoPreviewService {final needsTonemap = isHDR;final applyFPS = (double.tryParse(props.fps ?? "") ?? 100) > 30;- String filters = "";+ final videoFilters = <String>[];if (reencodeVideo) {- final videoFilters = <String>[];-if (rescaleVideo || needsTonemap) {videoFilters.add("scale='if(lt(iw,ih),min(720,iw),-2)':'if(lt(iw,ih),-2,min(720,ih))'",@@ -539,42 +552,52 @@ class VideoPreviewService {}videoFilters.add("format=yuv420p");-- filters = '-vf "${videoFilters.join(",")}" ';}- final command =- '$filters'- '${reencodeVideo ? '-c:v libx264 -maxrate 2000k -bufsize 4000k ' : '-c:v copy '}'- '-c:a aac -b:a 128k '- '-f hls -hls_flags single_file '- '-hls_list_size 0 -hls_key_info_file ${keyinfo.path} ';-- final playlistGenResult = await ffmpegService- .runFfmpegCancellable(- '-i "${file.path}" $command$prefix/output.m3u8',- (id) {- _currentFfmpegSessionId = id;- _logger.info("FFmpeg[$id]: $command");- },- )- .whenComplete(() => _currentFfmpegSessionId = null)- .onError((error, stackTrace) {- _logger.warning("FFmpeg command failed", error, stackTrace);- return {};- });+ final command = <String>[+ '-i',+ file.path,+ if (videoFilters.isNotEmpty) ...['-vf', videoFilters.join(',')],+ if (reencodeVideo) ...[+ '-c:v',+ 'libx264',+ '-maxrate',+ '2000k',+ '-bufsize',+ '4000k',+ ] else ...[+ '-c:v',+ 'copy',+ ],+ '-c:a',+ 'aac',+ '-b:a',+ '128k',+ '-f',+ 'hls',+ '-hls_flags',+ 'single_file',+ '-hls_list_size',+ '0',+ '-hls_key_info_file',+ keyinfo.path,+ '$prefix/output.m3u8',+ ];++ final playlistGenResult = await _runFfmpeg(command);+ if (cancelToken.isCancelled) return;if (_items.isEmpty) {Directory(prefix).delete(recursive: true).ignore();return;}- final playlistGenReturnCode = playlistGenResult["returnCode"] as int?;+ final playlistGenReturnCode = playlistGenResult.returnCode;String? objectId;int? objectSize;- if (ReturnCode.success == playlistGenReturnCode) {+ if (playlistGenResult.isSuccess) {try {_items[enteFile.uploadedFileID!] = PreviewItem(status: PreviewItemStatus.uploading,@@ -597,29 +620,38 @@ class VideoPreviewService {gateway: _fileDataGateway,dio: serviceLocator.enteDio,useUploadV2: flagService.previewUploadV2,- cancelToken: _streamingCancelToken,+ cancelToken: cancelToken,);+ if (cancelToken.isCancelled) return;objectId = result.$1;objectSize = result.$2;final playlistFrameResult = await ffmpegService- .runFfmpeg(- '-allowed_extensions ALL -i "$prefix/output.m3u8" -frames:v 1 -c copy "$prefix/frame.ts"',- )+ .start([+ '-allowed_extensions',+ 'ALL',+ '-i',+ '$prefix/output.m3u8',+ '-frames:v',+ '1',+ '-c',+ 'copy',+ '$prefix/frame.ts',+ ])+ .completed.onError((error, stackTrace) {_logger.warning(- "FFmpeg command failed for frame",+ "FFmpeg frame extraction failed",error,stackTrace,);- return {};+ return ffmpeg.Result(-1, error.toString());});- final playlistFrameReturnCode =- playlistFrameResult["returnCode"] as int?;+ if (cancelToken.isCancelled) return;int? width, height;try {- if (ReturnCode.success == playlistFrameReturnCode) {+ if (playlistFrameResult.isSuccess) {FFProbeProps? playlistFrameProps;final file2 = File("$prefix/frame.ts");@@ -630,6 +662,7 @@ class VideoPreviewService {} catch (err, sT) {_logger.warning("Failed to fetch resolution of stream", err, sT);}+ if (cancelToken.isCancelled) return;await _reportVideoPreview(enteFile,@@ -638,18 +671,21 @@ class VideoPreviewService {objectSize: objectSize,width: width,height: height,+ cancelToken: cancelToken,);+ if (cancelToken.isCancelled) return;_logger.info("Video preview uploaded for $enteFile");} catch (err, sT) {+ if (cancelToken.isCancelled) return;error = "Failed to upload video preview\nError: $err";_logger.shout("Something went wrong with preview upload", err, sT);}} else {- final output = playlistGenResult["output"] as String?;+ final output = playlistGenResult.output;_logger.warning("FFmpeg command failed with return code $playlistGenReturnCode\n"- "${_summarizeFfmpegOutput(output)}",+ "${output.isEmpty ? 'FFmpeg output unavailable' : output}",);error ="Failed to generate video preview (return code $playlistGenReturnCode)";@@ -676,37 +712,60 @@ class VideoPreviewService {Directory(prefix).delete(recursive: true).ignore();}} finally {- if (error != null) {- _retryFile(enteFile, error);- } else if (removeFile) {- _removeFile(enteFile);- _removeFromLocks(enteFile).ignore();- }- final bool shouldStopProcessing = _isNetworkError(error);-- if (fileQueue.isNotEmpty && !shouldStopProcessing) {+ if (cancelToken.isCancelled) {+ if (previewDirectory != null) {+ Directory(previewDirectory).delete(recursive: true).ignore();+ }+ } else {if (error != null) {- _logger.info(- "[chunk] Error occurred, waiting before processing next item. Queue size: ${fileQueue.length}",- );- await Future.delayed(const Duration(seconds: 2));+ _retryFile(enteFile, error);+ } else if (removeFile) {+ _removeFile(enteFile);+ _removeFromLocks(enteFile).ignore();}+ final bool shouldStopProcessing = _isNetworkError(error);- _logger.info("[chunk] Processing ${_items.length} items for streaming");- final entry = fileQueue.entries.first;- final file = entry.value;- fileQueue.remove(entry.key);- if (ctx != null && ctx.mounted) {- await chunkAndUploadVideo(ctx, file, continuation: true);+ if (fileQueue.isNotEmpty && !shouldStopProcessing) {+ if (error != null) {+ _logger.info(+ "[chunk] Error occurred, waiting before processing next item. Queue size: ${fileQueue.length}",+ );+ await Future.delayed(const Duration(seconds: 2));+ }++ if (!cancelToken.isCancelled && fileQueue.isNotEmpty) {+ _logger.info(+ "[chunk] Processing ${_items.length} items for streaming",+ );+ final entry = fileQueue.entries.first;+ final file = entry.value;+ fileQueue.remove(entry.key);+ if (ctx != null && ctx.mounted) {+ await chunkAndUploadVideo(ctx, file, continuation: true);+ } else {+ await chunkAndUploadVideo(null, file, continuation: true);+ }+ }} else {- await chunkAndUploadVideo(null, file, continuation: true);+ stop(shouldStopProcessing ? "network error" : "nothing to process");}- } else {- stop(shouldStopProcessing ? "network error" : "nothing to process");}}}+ Future<ffmpeg.Result> _runFfmpeg(List<String> arguments) async {+ final job = ffmpegService.start(arguments);+ _currentFfmpegJob = job;+ try {+ return await job.completed;+ } catch (error, stackTrace) {+ _logger.warning("FFmpeg command failed", error, stackTrace);+ return ffmpeg.Result(-1, error.toString());+ } finally {+ if (identical(_currentFfmpegJob, job)) _currentFfmpegJob = null;+ }+ }+Future<void> _removeFromLocks(EnteFile enteFile) async {final bool isFailurePresent =_failureFiles?.contains(enteFile.uploadedFileID!) ?? false;@@ -809,6 +868,7 @@ class VideoPreviewService {required int objectSize,required int? width,required int? height,+ required CancelToken cancelToken,}) async {_logger.fine("Pushing playlist for ${file.uploadedFileID}");try {@@ -827,7 +887,7 @@ class VideoPreviewService {objectSize: objectSize,playlist: result.encData,playlistHeader: result.header,- cancelToken: _streamingCancelToken,+ cancelToken: cancelToken,);} catch (e, s) {_logger.severe("Failed to report video preview", e, s);@@ -1195,7 +1255,7 @@ class VideoPreviewService {return (props, skipFile, file);}- Future<bool> _putFilesForPreviewCreation() async {+ Future<bool> _putFilesForPreviewCreation(CancelToken cancelToken) async {if (!isVideoStreamingEnabled || !await canUseHighBandwidth()) return false;Map<int, String> failureFiles = {};@@ -1227,6 +1287,7 @@ class VideoPreviewService {beginDate: DateTime.now().subtract(const Duration(days: 60)),onlyFilesWithLocalId: true,);+ if (cancelToken.isCancelled) return false;final previewIds = fileDataService.previewIds;_logger.info(@@ -1251,6 +1312,7 @@ class VideoPreviewService {queueFile ??= await filesDB.getAnyUploadedFile(queueFileId).catchError((e) => null);+ if (cancelToken.isCancelled) return false;if (queueFile == null) {await uploadLocksDB@@ -1271,6 +1333,7 @@ class VideoPreviewService {fileQueue[queueFile.uploadedFileID!] = queueFile;}+ if (cancelToken.isCancelled) return false;final allFiles = files.where((file) =>@@ -1338,13 +1401,12 @@ class VideoPreviewService {}bool _allowStream() {- return isVideoStreamingEnabled &&- computeController.requestCompute(stream: true);+ return isVideoStreamingEnabled && _compute.requestCompute(stream: true);}bool _allowManualStream() {return isVideoStreamingEnabled &&- computeController.requestCompute(+ _compute.requestCompute(stream: true,bypassInteractionCheck: true,bypassMLWaiting: true,@@ -1353,8 +1415,9 @@ class VideoPreviewService {bool _isPermissionGranted() {return isVideoStreamingEnabled &&- computeController.computeState == ComputeRunState.generatingStream &&- computeController.isDeviceHealthy;+ !_compute.computeBlocked &&+ _compute.computeState == ComputeRunState.generatingStream &&+ _compute.isDeviceHealthy;}void queueFiles({@@ -1363,10 +1426,9 @@ class VideoPreviewService {bool forceProcess = false,}) {Future.delayed(duration, () async {- _streamingCancelToken = null;if (_hasQueuedFile && !forceProcess) return;- if (flagService.stopStreamProcess && FileUploader.instance.isUploading) {+ if (FileUploader.instance.isUploading) {_logger.info("Skipping stream queue - file upload in progress");return;}@@ -1374,38 +1436,14 @@ class VideoPreviewService {final isStreamAllowed = isManual ? _allowManualStream() : _allowStream();if (!isStreamAllowed) return;+ final cancelToken = CancelToken();+ _streamingCancelToken = cancelToken;await _ensurePreviewIdsInitialized();- final result = await _putFilesForPreviewCreation();- if (!result) {- computeController.releaseCompute(stream: true);+ if (cancelToken.isCancelled) return;+ final result = await _putFilesForPreviewCreation(cancelToken);+ if (!result && !cancelToken.isCancelled) {+ _compute.releaseCompute(stream: true);}});}}--String _summarizeFfmpegOutput(String? output) {- final trimmedOutput = output?.trim();- if (trimmedOutput == null || trimmedOutput.isEmpty) {- return "FFmpeg output unavailable";- }-- final lines = const LineSplitter()- .convert(trimmedOutput)- .map((line) => line.trim())- .where((line) => line.isNotEmpty)- .toList();- final summarizedLines = lines.length > _maxFfmpegOutputLines- ? lines.sublist(lines.length - _maxFfmpegOutputLines)- : lines;- var summary = summarizedLines.join("\n");- if (summary.length > _maxFfmpegOutputChars) {- summary = summary.substring(summary.length - _maxFfmpegOutputChars);- }-- final wasTruncated =- lines.length > summarizedLines.length ||- trimmedOutput.length > _maxFfmpegOutputChars;- return wasTruncated- ? "FFmpeg output (truncated):\n$summary"- : "FFmpeg output:\n$summary";-}diff --git a/mobile/apps/photos/lib/ui/tools/editor/export_video_service.dart b/mobile/apps/photos/lib/ui/tools/editor/export_video_service.dartindex 67598d41fd..2dc83bd702 100644--- a/mobile/apps/photos/lib/ui/tools/editor/export_video_service.dart+++ b/mobile/apps/photos/lib/ui/tools/editor/export_video_service.dart@@ -1,10 +1,8 @@-import 'dart:async';import 'dart:io';import 'dart:ui';-import 'package:ffmpeg_kit_flutter/ffmpeg_kit.dart';-import 'package:ffmpeg_kit_flutter/return_code.dart';-import 'package:ffmpeg_kit_flutter/statistics.dart';+import 'package:photos/services/ffmpeg_service.dart';+import 'package:photos/services/video_preview_service.dart';import 'package:photos/ui/tools/editor/video_crop_util.dart';import 'package:photos/ui/tools/editor/video_editor/video_editor_controller.dart';@@ -80,49 +78,23 @@ class ExportService {FfmpegVideoExportPlan plan, {void Function(double)? onProgress,}) async {- final completer = Completer<File>();- try {- await FFmpegKit.executeWithArgumentsAsync(- plan.arguments,- (session) async {- try {- final returnCode = await session.getReturnCode();- if (returnCode != null && ReturnCode.isSuccess(returnCode)) {- final file = File(plan.outputPath);- if (!await file.exists() || await file.length() == 0) {- completer.completeError(- StateError('FFmpeg produced no output at ${plan.outputPath}'),- );- } else {- completer.complete(file);- }- return;- }- final output = await session.getOutput();- completer.completeError(- StateError(- 'FFmpeg exited with ${returnCode?.getValue() ?? -1}: $output',- ),- );- } catch (error, stackTrace) {- if (!completer.isCompleted) {- completer.completeError(error, stackTrace);- }- }- },- null,- (statistics) {- if (onProgress != null) {- onProgress(_progress(statistics, plan.duration));- }- },+ final job = FfmpegService.instance.start(+ plan.arguments,+ onProgress: onProgress == null+ ? null+ : (time) => onProgress(_progress(time, plan.duration)),+ );+ final result = await job.completed;+ if (!result.isSuccess) {+ throw StateError(+ 'FFmpeg exited with ${result.returnCode}: ${result.output}',);- } catch (error, stackTrace) {- if (!completer.isCompleted) {- completer.completeError(error, stackTrace);- }}- return completer.future;+ final file = File(plan.outputPath);+ if (!await file.exists() || await file.length() == 0) {+ throw StateError('FFmpeg produced no output at ${plan.outputPath}');+ }+ return file;}static Future<File> exportVideo({@@ -131,7 +103,9 @@ class ExportService {void Function(double)? onProgress,void Function(Object, StackTrace)? onError,}) async {+ final previews = VideoPreviewService.instance;try {+ await previews.pauseForExport();return await runFFmpegCommand(createPlan(controller: controller, outputPath: outputPath),onProgress: onProgress,@@ -139,12 +113,14 @@ class ExportService {} catch (error, stackTrace) {onError?.call(error, stackTrace);rethrow;+ } finally {+ previews.resumeAfterExport();}}- static double _progress(Statistics statistics, Duration duration) {+ static double _progress(Duration time, Duration duration) {if (duration.inMilliseconds <= 0) return 0;- return (statistics.getTime() / duration.inMilliseconds).clamp(0.0, 1.0);+ return (time.inMicroseconds / duration.inMicroseconds).clamp(0.0, 1.0);}static String _seconds(Duration duration) =>diff --git a/mobile/apps/photos/lib/utils/ffprobe_util.dart b/mobile/apps/photos/lib/utils/ffprobe_util.dartindex 5eaf02bd97..5d9a2780e6 100644--- a/mobile/apps/photos/lib/utils/ffprobe_util.dart+++ b/mobile/apps/photos/lib/utils/ffprobe_util.dart@@ -1,37 +1,13 @@// Adapted from: https://github.com/deckerst/aves-import "package:ffmpeg_kit_flutter/media_information.dart";-import "package:logging/logging.dart";import "package:photos/models/ffmpeg/ffprobe_keys.dart";-import "package:photos/models/ffmpeg/ffprobe_props.dart";class FFProbeUtil {- static final _logger = Logger('FFProbeUtil');static const chaptersKey = 'chapters';static const formatKey = 'format';static const streamsKey = 'streams';- static Future<FFProbeProps> getProperties(- MediaInformation mediaInformation,- ) async {- final properties = await getMetadata(mediaInformation);-- try {- return FFProbeProps.parseData(properties);- } catch (e, stackTrace) {- _logger.severe(- "Error parsing FFProbe properties: $properties",- e,- stackTrace,- );- rethrow;- }- }-- static Future<Map> getMetadata(MediaInformation information) async {- final props = information.getAllProperties();- if (props == null) return {};-+ static Map getMetadata(Map props) {final chapters = props[chaptersKey];if (chapters is List) {if (chapters.isEmpty) {@@ -115,7 +91,13 @@ class FFProbeUtil {}for (var key in <String>{+ FFProbeKeys.bitrate,FFProbeKeys.codecProfileId,+ FFProbeKeys.colorPrimaries,+ FFProbeKeys.colorRange,+ FFProbeKeys.colorSpace,+ FFProbeKeys.colorTransfer,+ FFProbeKeys.frameCount,FFProbeKeys.rFrameRate,'bits_per_sample','closed_captions',diff --git a/mobile/apps/photos/plugins/ffmpeg/android/build.gradle b/mobile/apps/photos/plugins/ffmpeg/android/build.gradlenew file mode 100644index 0000000000..4a24f727ae--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/android/build.gradle@@ -0,0 +1,75 @@+import java.security.MessageDigest++plugins {+ id 'com.android.library'+}++group 'io.ente.ffmpeg'+version '0.0.1'++def release = '9.0.2'+def checksum = '9cd0d3c53bc81ddfb13076ce9ecb3a55f8cf772e7324d6dd7a02f19c2df043d6'+def localAar = providers.environmentVariable('FFMPEG_AAR').orNull+def archive = configurations.create('ffmpeg') {+ canBeConsumed = false+ transitive = false+}++if (localAar == null) {+ repositories {+ exclusiveContent {+ forRepository {+ ivy {+ url = 'https://github.com/ente/ffmpeg-packaging/releases/download'+ patternLayout { artifact '[revision]/android-ffmpeg-runtime.aar' }+ metadataSources { artifact() }+ }+ }+ filter { includeModule('io.ente', 'ffmpeg-runtime') }+ }+ }+}++dependencies.add(archive.name, localAar == null ? "io.ente:ffmpeg-runtime:$release@aar" : files(localAar))++def nativeLibraries = layout.buildDirectory.dir('ffmpeg-jni')+def prepareFfmpeg = tasks.register('prepareFfmpeg') {+ inputs.files(archive)+ inputs.property('checksum', checksum)+ inputs.property('sourceBuilt', localAar != null)+ outputs.dir(nativeLibraries)++ doLast {+ def aar = archive.singleFile+ if (localAar == null) {+ def actual = MessageDigest.getInstance('SHA-256').digest(aar.bytes).encodeHex().toString()+ if (actual != checksum) {+ throw new GradleException('FFmpeg checksum mismatch')+ }+ }+ def libraries = zipTree(aar).matching { include 'jni/*/libffmpeg_runtime.so' }+ for (abi in ['arm64-v8a', 'armeabi-v7a']) {+ if (libraries.matching { include "jni/$abi/libffmpeg_runtime.so" }.isEmpty()) {+ throw new GradleException("FFmpeg AAR is missing $abi/libffmpeg_runtime.so")+ }+ }+ project.sync { spec ->+ spec.from(libraries)+ spec.eachFile { details -> details.path = details.path.substring('jni/'.length()) }+ spec.includeEmptyDirs = false+ spec.into(nativeLibraries)+ }+ }+}++android {+ namespace 'io.ente.ffmpeg'+ defaultConfig {+ minSdkVersion 26+ }+ sourceSets.main.jniLibs.srcDirs = [nativeLibraries]+}++tasks.named('preBuild').configure {+ dependsOn prepareFfmpeg+}diff --git a/mobile/apps/photos/plugins/ffmpeg/android/src/main/AndroidManifest.xml b/mobile/apps/photos/plugins/ffmpeg/android/src/main/AndroidManifest.xmlnew file mode 100644index 0000000000..94cbbcfc39--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/android/src/main/AndroidManifest.xml@@ -0,0 +1 @@+<manifest xmlns:android="http://schemas.android.com/apk/res/android" />diff --git a/mobile/apps/photos/plugins/ffmpeg/lib/ffmpeg.dart b/mobile/apps/photos/plugins/ffmpeg/lib/ffmpeg.dartnew file mode 100644index 0000000000..da6a9cedf3--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/lib/ffmpeg.dart@@ -0,0 +1 @@+export 'src/session.dart';diff --git a/mobile/apps/photos/plugins/ffmpeg/lib/src/bindings.dart b/mobile/apps/photos/plugins/ffmpeg/lib/src/bindings.dartnew file mode 100644index 0000000000..b547e974e3--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/lib/src/bindings.dart@@ -0,0 +1,80 @@+import 'dart:ffi';+import 'dart:io';++import 'package:ffi/ffi.dart';++final class SessionHandle extends Opaque {}++typedef ProgressCallback = Void Function(Pointer<Void>, Int64, Int32);++final class Bindings {+ Bindings(DynamicLibrary library)+ : sessionNew = library+ .lookupFunction<+ Pointer<SessionHandle> Function(+ Pointer<NativeFunction<ProgressCallback>>,+ Pointer<Void>,+ ),+ Pointer<SessionHandle> Function(+ Pointer<NativeFunction<ProgressCallback>>,+ Pointer<Void>,+ )+ >('ffmpeg_session_new'),+ sessionFree = library+ .lookupFunction<+ Void Function(Pointer<SessionHandle>),+ void Function(Pointer<SessionHandle>)+ >('ffmpeg_session_free'),+ sessionOutput = library+ .lookupFunction<+ Pointer<Utf8> Function(Pointer<SessionHandle>),+ Pointer<Utf8> Function(Pointer<SessionHandle>)+ >('ffmpeg_session_output'),+ execute = library+ .lookupFunction<+ Int32 Function(+ Pointer<SessionHandle>,+ Int32,+ Pointer<Pointer<Utf8>>,+ ),+ int Function(Pointer<SessionHandle>, int, Pointer<Pointer<Utf8>>)+ >('ffmpeg_execute'),+ cancel = library+ .lookupFunction<+ Void Function(Pointer<SessionHandle>),+ void Function(Pointer<SessionHandle>)+ >('ffmpeg_cancel'),+ probe = library+ .lookupFunction<+ Int32 Function(Pointer<Utf8>, Pointer<Pointer<Utf8>>),+ int Function(Pointer<Utf8>, Pointer<Pointer<Utf8>>)+ >('ffmpeg_probe_media_json'),+ freeString = library+ .lookupFunction<+ Void Function(Pointer<Utf8>),+ void Function(Pointer<Utf8>)+ >('ffmpeg_free_string');++ static final instance = Bindings(_openLibrary());++ final Pointer<SessionHandle> Function(+ Pointer<NativeFunction<ProgressCallback>>,+ Pointer<Void>,+ )+ sessionNew;+ final void Function(Pointer<SessionHandle>) sessionFree;+ final Pointer<Utf8> Function(Pointer<SessionHandle>) sessionOutput;+ final int Function(Pointer<SessionHandle>, int, Pointer<Pointer<Utf8>>)+ execute;+ final void Function(Pointer<SessionHandle>) cancel;+ final int Function(Pointer<Utf8>, Pointer<Pointer<Utf8>>) probe;+ final void Function(Pointer<Utf8>) freeString;++ static DynamicLibrary _openLibrary() {+ if (Platform.isAndroid) return DynamicLibrary.open('libffmpeg_runtime.so');+ if (Platform.isIOS) {+ return DynamicLibrary.open('FFmpegRuntime.framework/FFmpegRuntime');+ }+ throw UnsupportedError('FFmpeg supports Android and iOS.');+ }+}diff --git a/mobile/apps/photos/plugins/ffmpeg/lib/src/session.dart b/mobile/apps/photos/plugins/ffmpeg/lib/src/session.dartnew file mode 100644index 0000000000..964a99cf26--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/lib/src/session.dart@@ -0,0 +1,114 @@+import 'dart:convert';+import 'dart:ffi';+import 'dart:isolate';++import 'package:ffi/ffi.dart';+import 'package:ffmpeg/src/bindings.dart';++class Result {+ const Result(this.returnCode, this.output);++ final int returnCode;+ final String output;++ bool get isSuccess => returnCode == 0;+ bool get isCancelled => returnCode == 255;+}++abstract interface class Session {+ factory Session(void Function(Duration) onProgress) = _NativeSession;++ Future<Result> execute(List<String> arguments);+ void cancel();+}++final class _NativeSession implements Session {+ _NativeSession(void Function(Duration) onProgress) {+ final bindings = Bindings.instance;+ void progress(Pointer<Void> opaque, int time, int isFinal) {+ if (!_finished && time >= 0) {+ onProgress(Duration(microseconds: time));+ }+ }++ _callback = NativeCallable<ProgressCallback>.listener(progress);+ _session = bindings.sessionNew(_callback.nativeFunction, nullptr);+ if (_session == nullptr) {+ _callback.close();+ throw StateError('Could not allocate an FFmpeg session.');+ }+ }++ late final NativeCallable<ProgressCallback> _callback;+ late final Pointer<SessionHandle> _session;+ bool _started = false;+ bool _finished = false;++ @override+ Future<Result> execute(List<String> arguments) async {+ if (_started) throw StateError('An FFmpeg session runs once.');+ _started = true;+ try {+ _checkStrings(arguments);+ return await _executeInWorker(_session.address, arguments);+ } finally {+ _finished = true;+ _callback.close();+ Bindings.instance.sessionFree(_session);+ }+ }++ @override+ void cancel() {+ if (!_finished) Bindings.instance.cancel(_session);+ }+}++Future<Result> _executeInWorker(int address, List<String> arguments) =>+ Isolate.run(() => _execute(address, arguments));++Result _execute(int address, List<String> arguments) {+ return using((arena) {+ final values = ['ffmpeg', ...arguments];+ final argv = arena<Pointer<Utf8>>(values.length + 1);+ for (var i = 0; i < values.length; i++) {+ argv[i] = values[i].toNativeUtf8(allocator: arena);+ }+ final bindings = Bindings.instance;+ final session = Pointer<SessionHandle>.fromAddress(address);+ final code = bindings.execute(session, values.length, argv);+ final output = bindings.sessionOutput(session);+ return Result(+ code,+ utf8.decode(+ output.cast<Uint8>().asTypedList(output.length),+ allowMalformed: true,+ ),+ );+ });+}++Future<Map<String, dynamic>> probeMedia(String path) {+ _checkStrings([path]);+ return Isolate.run(() => _probe(path));+}++Map<String, dynamic> _probe(String path) {+ return using((arena) {+ final bindings = Bindings.instance;+ final output = arena<Pointer<Utf8>>();+ final code = bindings.probe(path.toNativeUtf8(allocator: arena), output);+ try {+ if (code < 0) return {};+ return jsonDecode(output.value.toDartString()) as Map<String, dynamic>;+ } finally {+ bindings.freeString(output.value);+ }+ });+}++void _checkStrings(List<String> values) {+ if (values.any((value) => value.contains('\u0000'))) {+ throw ArgumentError('FFmpeg arguments cannot contain NUL characters.');+ }+}diff --git a/mobile/apps/photos/plugins/ffmpeg/pubspec.yaml b/mobile/apps/photos/plugins/ffmpeg/pubspec.yamlnew file mode 100644index 0000000000..8a01855a64--- /dev/null+++ b/mobile/apps/photos/plugins/ffmpeg/pubspec.yaml@@ -0,0 +1,17 @@+name: ffmpeg+description: Native FFmpeg execution and media probing for Photos.+version: 0.0.1+publish_to: none+resolution: workspace++environment:+ sdk: ">=3.10.0 <4.0.0"++dependencies:+ ffi: 2.2.0++flutter:+ plugin:+ platforms:+ android:+ ffiPlugin: truediff --git a/mobile/apps/photos/pubspec.yaml b/mobile/apps/photos/pubspec.yamlindex 8d2648d382..bab9211e2b 100644--- a/mobile/apps/photos/pubspec.yaml+++ b/mobile/apps/photos/pubspec.yaml@@ -86,11 +86,8 @@ dependencies:extended_image: 9.1.0fade_indexed_stack: 0.2.2fast_base58: 0.2.1- ffmpeg_kit_flutter:- git:- url: https://github.com/ente/ffmpeg-kit- path: flutter/flutter- ref: 15afc08775a75fe9c12de3366fa99e39bf171ccc+ ffmpeg:+ path: plugins/ffmpegfigma_squircle: 0.6.3file_saver: 0.3.1firebase_core: 3.15.1diff --git a/mobile/apps/photos/test/services/video_preview_service_test.dart b/mobile/apps/photos/test/services/video_preview_service_test.dartindex c98202529d..a08758ba43 100644--- a/mobile/apps/photos/test/services/video_preview_service_test.dart+++ b/mobile/apps/photos/test/services/video_preview_service_test.dart@@ -1,3 +1,6 @@+import 'dart:async';++import 'package:dio/dio.dart';import "package:flutter_cache_manager/flutter_cache_manager.dart";import 'package:flutter_test/flutter_test.dart';import "package:photos/core/configuration.dart";@@ -6,15 +9,28 @@ import "package:photos/db/upload_locks_db.dart";import 'package:photos/models/file/file.dart';import 'package:photos/models/file/file_type.dart';import 'package:photos/models/metadata/file_magic.dart';+import 'package:photos/models/preview/playlist_data.dart';+import 'package:photos/models/preview/preview_item_status.dart';+import 'package:photos/module/upload/service/file_uploader.dart';import "package:photos/service_locator.dart";+import "package:photos/services/ffmpeg_service.dart";import "package:photos/services/file_magic_service.dart";import 'package:photos/services/filedata/model/file_data.dart';-import "package:photos/services/isolated_ffmpeg_service.dart";+import 'package:photos/services/machine_learning/compute_controller.dart';import 'package:photos/services/video_preview_service.dart';+import 'package:shared_preferences/shared_preferences.dart';void main() {+ TestWidgetsFlutterBinding.ensureInitialized();late VideoPreviewService videoPreviewService;+ setUpAll(() async {+ SharedPreferences.setMockInitialValues({});+ ServiceLocator.instance.prefs = await SharedPreferences.getInstance();+ ServiceLocator.instance.enteDio = Dio();+ FileUploader.instance = _FakeFileUploader();+ });+setUp(() {videoPreviewService = VideoPreviewService(_FakeConfiguration(),@@ -22,7 +38,7 @@ void main() {_FakeFilesDB(),_FakeUploadLocksDB(),_FakeFileMagicService(),- _FakeIsolatedFfmpegService(),+ _FakeFfmpegService(),_FakeDefaultCacheManager(),_FakeCacheManager(),);@@ -64,6 +80,49 @@ void main() {expect(status, equals(0.75));});});++ test('export pauses scheduling without clearing pending previews', () async {+ final compute = _FakeComputeController();+ final service = _ControlledPreviewService(compute);+ final file = EnteFile()..uploadedFileID = 42;+ service.uploadingFileId = 42;+ await service.addToManualQueue(file, 'create');++ await service.pauseForExport();+ expect(compute.computeBlocked, isTrue);+ expect(compute.releases, 0);+ expect(service.getProcessingStatus(42), PreviewItemStatus.inQueue);+ expect(service.uploadingFileId, 42);++ service.resumeAfterExport();+ expect(compute.computeBlocked, isFalse);+ expect(service.resumeRequests, 1);+ });++ test(+ 'stopped preparation cannot repopulate a resumed preview queue',+ () async {+ final compute = _FakeComputeController();+ final service = _ControlledPreviewService(compute);+ final file = EnteFile()+ ..uploadedFileID = 42+ ..fileType = FileType.video;+ service.uploadingFileId = 42;+ await service.addToManualQueue(file, 'create');+ service.uploadingFileId = -1;++ final preparation = service.chunkAndUploadVideo(null, file);+ await service.playlistRequested.future;+ service.stop('export');+ service.resumeAfterExport();+ service.playlist.complete(null);+ await preparation;++ expect(service.getProcessingStatus(42), isNull);+ expect(service.fileQueue, isEmpty);+ expect(compute.releases, 1);+ },+ );}class _FakeServiceLocator extends Fake implements ServiceLocator {}@@ -72,13 +131,82 @@ class _FakeConfiguration extends Fake implements Configuration {}class _FakeFilesDB extends Fake implements FilesDB {}-class _FakeUploadLocksDB extends Fake implements UploadLocksDB {}+class _FakeUploadLocksDB extends Fake implements UploadLocksDB {+ @override+ Future<bool> isInStreamQueue(int fileID) async => false;++ @override+ Future<void> addToStreamQueue(int fileID, String queueType) async {}++ @override+ Future<Map<int, String>> getStreamQueue() async => {};+}class _FakeFileMagicService extends Fake implements FileMagicService {}-class _FakeIsolatedFfmpegService extends Fake- implements IsolatedFfmpegService {}+class _FakeFfmpegService extends Fake implements FfmpegService {}class _FakeDefaultCacheManager extends Fake implements DefaultCacheManager {}class _FakeCacheManager extends Fake implements CacheManager {}++class _FakeFileUploader extends Fake implements FileUploader {+ @override+ bool get isUploading => false;+}++class _FakeComputeController extends Fake implements ComputeController {+ @override+ bool computeBlocked = false;+ int releases = 0;++ @override+ ComputeRunState get computeState => ComputeRunState.generatingStream;++ @override+ bool get isDeviceHealthy => true;++ @override+ void blockCompute({required String blocker}) => computeBlocked = true;++ @override+ void unblockCompute({required String blocker}) => computeBlocked = false;++ @override+ void releaseCompute({bool ml = false, bool stream = false}) => releases++;+}++class _ControlledPreviewService extends VideoPreviewService {+ _ControlledPreviewService(ComputeController compute)+ : super(+ _FakeConfiguration(),+ _FakeServiceLocator(),+ _FakeFilesDB(),+ _FakeUploadLocksDB(),+ _FakeFileMagicService(),+ _FakeFfmpegService(),+ _FakeDefaultCacheManager(),+ _FakeCacheManager(),+ compute: compute,+ );++ final playlistRequested = Completer<void>();+ final playlist = Completer<PlaylistData?>();+ int resumeRequests = 0;++ @override+ bool get isVideoStreamingEnabled => true;++ @override+ Future<PlaylistData?> getPlaylist(EnteFile file) {+ playlistRequested.complete();+ return playlist.future;+ }++ @override+ void queueFiles({+ Duration duration = const Duration(seconds: 5),+ bool isManual = false,+ bool forceProcess = false,+ }) => resumeRequests++;+}diff --git a/mobile/packages/feature_flag/lib/src/flag_service.dart b/mobile/packages/feature_flag/lib/src/flag_service.dartindex 93155bce5a..03ddba30cd 100644--- a/mobile/packages/feature_flag/lib/src/flag_service.dart+++ b/mobile/packages/feature_flag/lib/src/flag_service.dart@@ -106,8 +106,6 @@ class FlagService {bool get facesTimeline => true;bool get ritualsFlag => true;- bool get stopStreamProcess => true;-bool get streamEnabledByDefault => _isServerFlagEnabled(_videoStreamingFlag);bool get previewUploadV2 => _isServerFlagEnabled(_previewUploadV2Flag);diff --git a/mobile/pubspec.lock b/mobile/pubspec.lockindex 08d3bf0a35..4d283de86b 100644--- a/mobile/pubspec.lock+++ b/mobile/pubspec.lock@@ -673,24 +673,6 @@ packages:url: "https://pub.dev"source: hostedversion: "2.2.0"- ffmpeg_kit_flutter:- dependency: transitive- description:- path: "flutter/flutter"- ref: "15afc08775a75fe9c12de3366fa99e39bf171ccc"- resolved-ref: "15afc08775a75fe9c12de3366fa99e39bf171ccc"- url: "https://github.com/ente/ffmpeg-kit"- source: git- version: "6.0.3"- ffmpeg_kit_flutter_platform_interface:- dependency: transitive- description:- path: "flutter/flutter_platform_interface"- ref: "15afc08775a75fe9c12de3366fa99e39bf171ccc"- resolved-ref: "15afc08775a75fe9c12de3366fa99e39bf171ccc"- url: "https://github.com/ente/ffmpeg-kit"- source: git- version: "0.2.1"figma_squircle:dependency: transitivedescription:diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yamlindex d39d1cc5ce..8d96bd7257 100644--- a/mobile/pubspec.yaml+++ b/mobile/pubspec.yaml@@ -22,6 +22,7 @@ workspace:- apps/photos/plugins/ente_panorama_viewer- apps/photos/plugins/ente_photos_platform- apps/photos/plugins/ente_vision+ - apps/photos/plugins/ffmpeg- apps/photos/plugins/media_extension- packages/accounts- packages/account_deletion