Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][photos] Export video speed changes with native editors

r4khul60180c3f8077
diff --git a/mobile/apps/photos/lib/ui/tools/editor/native_video_export_service.dart b/mobile/apps/photos/lib/ui/tools/editor/native_video_export_service.dart
index 9d80c47dce..2b469dc781 100644
--- a/mobile/apps/photos/lib/ui/tools/editor/native_video_export_service.dart
+++ b/mobile/apps/photos/lib/ui/tools/editor/native_video_export_service.dart
@@ -37,8 +37,9 @@ class NativeVideoExportService {
controller.maxCrop != const Offset(1.0, 1.0);
final needsRotate = controller.rotation != 0;
final needsTrim = controller.isTrimmed;
+ final needsSpeed = controller.speed != 1.0;
- if (!(needsCrop || needsRotate || needsTrim)) {
+ if (!(needsCrop || needsRotate || needsTrim || needsSpeed)) {
await File(inputPath).copy(outputPath);
return VideoEditResult(outputPath: outputPath, isReEncoded: false);
}
@@ -57,6 +58,7 @@ class NativeVideoExportService {
trimEnd: needsTrim ? controller.endTrim : null,
rotateDegrees: needsRotate ? controller.rotation : null,
cropRect: cropRect,
+ speed: controller.speed,
onProgress: onProgress,
);
}
diff --git a/mobile/apps/photos/lib/ui/tools/editor/video_editor_page.dart b/mobile/apps/photos/lib/ui/tools/editor/video_editor_page.dart
index 2ba4358a56..47d6a5b387 100644
--- a/mobile/apps/photos/lib/ui/tools/editor/video_editor_page.dart
+++ b/mobile/apps/photos/lib/ui/tools/editor/video_editor_page.dart
@@ -246,11 +246,9 @@ class _VideoEditorPageState extends State<VideoEditorPage> {
if (!await ensurePhotoLibraryAddPermission(context)) return;
if (!mounted) return;
- final shouldUseNative =
- (flagService.internalUser
- ? _useNativeExport
- : flagService.useNativeVideoEditor) &&
- _controller!.speed == 1.0;
+ final shouldUseNative = flagService.internalUser
+ ? _useNativeExport
+ : flagService.useNativeVideoEditor;
_logEditState(shouldUseNative: shouldUseNative);
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
index 02d2855358..d74be172e8 100644
--- a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
@@ -307,7 +307,8 @@ class NativeVideoEditorPlugin : FlutterPlugin, MethodCallHandler, EventChannel.S
trimStartMs = call.argument<Number>("trimStartMs")?.toLong(),
trimEndMs = call.argument<Number>("trimEndMs")?.toLong(),
rotateDegrees = call.argument<Int>("rotateDegrees"),
- crop = crop
+ crop = crop,
+ speed = call.argument<Number>("speed")?.toFloat() ?: 1f
)
}.getOrNull()
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoExportEngine.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoExportEngine.kt
index 7be16233ec..d4ae5ec080 100644
--- a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoExportEngine.kt
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoExportEngine.kt
@@ -6,9 +6,12 @@ import android.media.MediaFormat
import android.net.Uri
import android.os.Handler
import android.os.Looper
+import androidx.media3.common.C
import androidx.media3.common.Effect
import androidx.media3.common.MediaItem
import androidx.media3.common.MimeTypes
+import androidx.media3.common.audio.SonicAudioProcessor
+import androidx.media3.common.audio.SpeedProvider
import androidx.media3.common.util.ExperimentalApi
import androidx.media3.common.util.UnstableApi
import androidx.media3.common.util.Size
@@ -21,6 +24,9 @@ import androidx.media3.transformer.EditedMediaItemSequence
import androidx.media3.transformer.Effects
import androidx.media3.transformer.ExportException
import androidx.media3.transformer.ExportResult
+import androidx.media3.transformer.InAppMp4Muxer
+import androidx.media3.muxer.BufferInfo
+import androidx.media3.muxer.Muxer
import androidx.media3.transformer.ProgressHolder
import androidx.media3.transformer.Transformer
import kotlinx.coroutines.CancellationException
@@ -33,6 +39,8 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.File
+import java.nio.ByteBuffer
+import kotlin.math.roundToLong
@UnstableApi
class VideoExportEngine(
@@ -58,9 +66,17 @@ class VideoExportEngine(
val mediaItem = buildMediaItem(inputFile, clipRange)
val videoInfo = request.crop?.let { metadataReader.read(inputFile.path) }
val effects = buildVideoEffects(videoInfo, request.crop, request.rotateDegrees)
+ val audioProcessors = if (request.speed != 1f) {
+ listOf(SonicAudioProcessor().apply { setPitch(1f / request.speed) })
+ } else {
+ emptyList()
+ }
val editedMediaItem = EditedMediaItem.Builder(mediaItem).apply {
- if (effects.isNotEmpty()) {
- setEffects(Effects(emptyList(), effects))
+ if (request.speed != 1f) {
+ setSpeed(ConstantSpeedProvider(request.speed))
+ }
+ if (effects.isNotEmpty() || audioProcessors.isNotEmpty()) {
+ setEffects(Effects(audioProcessors, effects))
}
}.build()
@Suppress("DEPRECATION")
@@ -71,8 +87,15 @@ class VideoExportEngine(
export(
composition = composition,
outputPath = outputFile.path,
- transcodeVideo = effects.isNotEmpty(),
- optimizeTrim = request.trimStartMs != null && effects.isEmpty(),
+ transcodeVideo = effects.isNotEmpty() || request.speed != 1f,
+ optimizeTrim = request.trimStartMs != null &&
+ effects.isEmpty() &&
+ request.speed == 1f,
+ maxDurationUs = if (request.speed != 1f) {
+ ((clipRange.endUs - clipRange.startUs) / request.speed.toDouble()).roundToLong()
+ } else {
+ null
+ },
onProgress = onProgress
)
}
@@ -162,12 +185,16 @@ class VideoExportEngine(
outputPath: String,
transcodeVideo: Boolean,
optimizeTrim: Boolean,
+ maxDurationUs: Long?,
onProgress: ((Float) -> Unit)?
): ExportResult = coroutineScope {
var progressJob: Job? = null
try {
suspendCancellableCoroutine { continuation ->
val transformerBuilder = Transformer.Builder(context)
+ if (maxDurationUs != null) {
+ transformerBuilder.setMuxerFactory(DurationLimitedMuxerFactory(maxDurationUs))
+ }
if (transcodeVideo) {
transformerBuilder.setVideoMimeType(MimeTypes.VIDEO_H264)
}
@@ -240,6 +267,28 @@ class VideoExportEngine(
}
}
+private class DurationLimitedMuxerFactory(
+ private val durationUs: Long,
+ private val factory: Muxer.Factory = InAppMp4Muxer.Factory().setVideoDurationUs(durationUs)
+) : Muxer.Factory by factory {
+ override fun create(path: String): Muxer {
+ val muxer = factory.create(path)
+ return object : Muxer by muxer {
+ override fun writeSampleData(trackId: Int, byteBuffer: ByteBuffer, bufferInfo: BufferInfo) {
+ if (bufferInfo.presentationTimeUs < durationUs) {
+ muxer.writeSampleData(trackId, byteBuffer, bufferInfo)
+ }
+ }
+ }
+ }
+}
+
+private class ConstantSpeedProvider(private val speed: Float) : SpeedProvider {
+ override fun getSpeed(timeUs: Long): Float = speed
+
+ override fun getNextSpeedChangeTimeUs(timeUs: Long): Long = C.TIME_UNSET
+}
+
internal data class VideoClipRangeUs(val startUs: Long, val endUs: Long)
internal fun resolveVideoClipRangeUs(
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt
index 7b0e3c5791..1285d8fc9f 100644
--- a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt
@@ -76,7 +76,8 @@ data class VideoEditRequest(
val trimStartMs: Long?,
val trimEndMs: Long?,
val rotateDegrees: Int?,
- val crop: VideoCrop?
+ val crop: VideoCrop?,
+ val speed: Float
) {
init {
require(inputPath.isNotBlank() && outputPath.isNotBlank()) { "Missing video path" }
@@ -89,6 +90,7 @@ data class VideoEditRequest(
require(rotateDegrees == null || rotateDegrees in setOf(0, 90, 180, 270)) {
"Invalid rotation"
}
+ require(speed.isFinite() && speed > 0f) { "Invalid speed" }
}
}
diff --git a/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift b/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
index d3c55b8ca6..0a79776fab 100644
--- a/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
+++ b/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
@@ -337,7 +337,8 @@ public final class NativeVideoEditorPlugin: NSObject, @preconcurrency FlutterPlu
trimStartMs: int64(args["trimStartMs"]),
trimEndMs: int64(args["trimEndMs"]),
rotateDegrees: int(args["rotateDegrees"]),
- crop: crop
+ crop: crop,
+ speed: double(args["speed"]) ?? 1
)
}
@@ -349,6 +350,10 @@ public final class NativeVideoEditorPlugin: NSObject, @preconcurrency FlutterPlu
(value as? NSNumber)?.int64Value
}
+ private func double(_ value: Any?) -> Double? {
+ (value as? NSNumber)?.doubleValue
+ }
+
private func flutterError(
_ error: Error,
defaultCode: String
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift b/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift
index 7fcf05415a..8e979a6747 100644
--- a/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift
@@ -83,22 +83,24 @@ public final class VideoExportEngine: Sendable {
)
}
+ var compositionAudioTrack: AVMutableCompositionTrack?
do {
try compositionVideoTrack.insertTimeRange(videoTimeRange, of: videoTrack, at: .zero)
let audioTracks = try await asset.loadTracks(withMediaType: .audio)
if let audioTrack = audioTracks.first {
let audioTrackRange = try await audioTrack.load(.timeRange)
if let audioTimeRange = validIntersection(videoTimeRange, audioTrackRange),
- let compositionAudioTrack = composition.addMutableTrack(
+ let mutableAudioTrack = composition.addMutableTrack(
withMediaType: .audio,
preferredTrackID: kCMPersistentTrackID_Invalid
)
{
- try compositionAudioTrack.insertTimeRange(
+ try mutableAudioTrack.insertTimeRange(
audioTimeRange,
of: audioTrack,
at: CMTimeSubtract(audioTimeRange.start, videoTimeRange.start)
)
+ compositionAudioTrack = mutableAudioTrack
}
}
} catch is CancellationError {
@@ -107,7 +109,18 @@ public final class VideoExportEngine: Sendable {
throw VideoEditorError.composition("Failed to insert media track", error)
}
- let isReEncoded = request.crop != nil || (request.rotateDegrees ?? 0) != 0
+ if request.speed != 1 {
+ let sourceTimeRange = CMTimeRange(start: .zero, duration: composition.duration)
+ let outputDuration = CMTimeMultiplyByFloat64(
+ sourceTimeRange.duration,
+ multiplier: 1 / request.speed
+ )
+ compositionVideoTrack.scaleTimeRange(sourceTimeRange, toDuration: outputDuration)
+ compositionAudioTrack?.scaleTimeRange(sourceTimeRange, toDuration: outputDuration)
+ }
+
+ let isReEncoded =
+ request.crop != nil || (request.rotateDegrees ?? 0) != 0 || request.speed != 1
var videoComposition: AVMutableVideoComposition?
if isReEncoded {
let transformPlan = try VideoTransformPlanner.makePlan(
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift b/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift
index 4b33b267da..c2ebfc1cdf 100644
--- a/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift
@@ -115,6 +115,7 @@ public struct VideoEditRequest: Sendable {
public let trimEndMs: Int64?
public let rotateDegrees: Int?
public let crop: VideoCrop?
+ public let speed: Double
public init(
inputURL: URL,
@@ -122,7 +123,8 @@ public struct VideoEditRequest: Sendable {
trimStartMs: Int64?,
trimEndMs: Int64?,
rotateDegrees: Int?,
- crop: VideoCrop?
+ crop: VideoCrop?,
+ speed: Double
) throws {
let inputURL = inputURL.standardizedFileURL.resolvingSymlinksInPath()
let outputURL = outputURL.standardizedFileURL.resolvingSymlinksInPath()
@@ -130,7 +132,9 @@ public struct VideoEditRequest: Sendable {
FileManager.default.fileExists(atPath: inputURL.path),
(trimStartMs == nil) == (trimEndMs == nil),
trimStartMs == nil || (trimStartMs! >= 0 && trimStartMs! < trimEndMs!),
- rotateDegrees == nil || [0, 90, 180, 270].contains(rotateDegrees!)
+ rotateDegrees == nil || [0, 90, 180, 270].contains(rotateDegrees!),
+ speed.isFinite,
+ speed > 0
else {
throw VideoEditorError.invalidRequest("Invalid video processing request")
}
@@ -140,6 +144,7 @@ public struct VideoEditRequest: Sendable {
self.trimEndMs = trimEndMs
self.rotateDegrees = rotateDegrees
self.crop = crop
+ self.speed = speed
}
}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift b/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift
index f97cd4da8a..f431efb8f5 100644
--- a/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift
@@ -20,14 +20,11 @@ public enum VideoTransformPlanner {
rotateDegrees: rotateDegrees ?? 0
)
}
- if let rotateDegrees, rotateDegrees != 0 {
- return rotationPlan(
- naturalSize: naturalSize,
- preferredTransform: preferredTransform,
- rotateDegrees: rotateDegrees
- )
- }
- return VideoTransformPlan(transform: preferredTransform, renderSize: naturalSize)
+ return rotationPlan(
+ naturalSize: naturalSize,
+ preferredTransform: preferredTransform,
+ rotateDegrees: rotateDegrees ?? 0
+ )
}
private static func cropPlan(
diff --git a/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift b/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift
index 916d5140cc..8f698f4001 100644
--- a/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift
+++ b/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift
@@ -16,6 +16,19 @@ final class VideoTransformPlannerTests: XCTestCase {
assertEqual(sourceBounds.applying(plan.transform), CGRect(origin: .zero, size: plan.renderSize))
}
+ func testSpeedOnlyExportNormalizesMetadataRotation() throws {
+ let sourceBounds = CGRect(x: 0, y: 0, width: 1080, height: 1920)
+ let plan = try VideoTransformPlanner.makePlan(
+ naturalSize: sourceBounds.size,
+ preferredTransform: CGAffineTransform(a: 0, b: -1, c: 1, d: 0, tx: 0, ty: 0),
+ crop: nil,
+ rotateDegrees: nil
+ )
+
+ assertEqual(plan.renderSize, CGSize(width: 1920, height: 1080))
+ assertEqual(sourceBounds.applying(plan.transform), CGRect(origin: .zero, size: plan.renderSize))
+ }
+
func testCropMapsDisplayRectangleToOutputBounds() throws {
let crop = try VideoCrop(x: 20, y: 30, width: 100, height: 60)
let plan = try VideoTransformPlanner.makePlan(
diff --git a/mobile/packages/native_video_editor/lib/native_video_editor.dart b/mobile/packages/native_video_editor/lib/native_video_editor.dart
index 8431eacdd2..298e41e2e3 100644
--- a/mobile/packages/native_video_editor/lib/native_video_editor.dart
+++ b/mobile/packages/native_video_editor/lib/native_video_editor.dart
@@ -266,6 +266,7 @@ class NativeVideoEditor {
Duration? trimEnd,
int? rotateDegrees,
Rect? cropRect,
+ double speed = 1.0,
void Function(double progress)? onProgress,
}) async {
_ensureInputPathExists(inputPath);
@@ -282,6 +283,9 @@ class NativeVideoEditor {
if (rotateDegrees != null && rotateDegrees != 0) {
_validateRotationDegrees(rotateDegrees);
}
+ if (!speed.isFinite || speed <= 0) {
+ throw ArgumentError.value(speed, 'speed');
+ }
if (cropRect != null &&
(!cropRect.left.isFinite ||
!cropRect.top.isFinite ||
@@ -306,6 +310,7 @@ class NativeVideoEditor {
'trimEndMs': trimEnd!.inMilliseconds,
},
'rotateDegrees': ?rotateDegrees,
+ 'speed': speed,
if (cropRect != null) ...{
'cropX': cropRect.left.toInt(),
'cropY': cropRect.top.toInt(),
diff --git a/mobile/packages/native_video_editor/test/native_video_editor_test.dart b/mobile/packages/native_video_editor/test/native_video_editor_test.dart
index a7d210b817..ace188a9da 100644
--- a/mobile/packages/native_video_editor/test/native_video_editor_test.dart
+++ b/mobile/packages/native_video_editor/test/native_video_editor_test.dart
@@ -204,6 +204,16 @@ void main() {
expect(lastCall, isNull);
});
+ test('processing sends the requested speed', () async {
+ await NativeVideoEditor.processVideo(
+ inputPath: input.path,
+ outputPath: '${directory.path}/output.mp4',
+ speed: 0.25,
+ );
+
+ expect(lastCall?.arguments, containsPair('speed', 0.25));
+ });
+
test('processing result must match the requested output path', () async {
processResultOverride = {
'outputPath': '${directory.path}/wrong.mp4',