Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Separate native video core from Flutter adapters

Neeraj Gupta9fb22a68a01b
diff --git a/mobile/apps/photos/ios/Podfile.lock b/mobile/apps/photos/ios/Podfile.lock
index a2c833719d..19cc3355e1 100644
--- a/mobile/apps/photos/ios/Podfile.lock
+++ b/mobile/apps/photos/ios/Podfile.lock
@@ -514,7 +514,7 @@ SPEC CHECKSUMS:
motionphoto: 23e2aeb5c6380112f69468d71f970fa7438e5ed1
move_to_background: 7e3467dd2a1d1013e98c9c1cb93fd53cd7ef9d84
nanopb: fad817b59e0457d11a5dfbde799381cd727c1275
- native_video_editor: 5af94a14017a00dcb0a0055abb869fce79e5493e
+ native_video_editor: a6a0fbb3b418961d09f4fecf5730c740f7210247
native_video_player: 6809dec117e8997161dbfb42a6f90d6df71a504d
open_mail_app: 7314a609e88eed22d53671279e189af7a0ab0f11
OrderedSet: e539b66b644ff081c73a262d24ad552a69be3a94
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
index 5e8865ef52..28bb784712 100644
--- a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/NativeVideoEditorPlugin.kt
@@ -1,641 +1,337 @@
package io.ente.native_video_editor
-import android.content.Context
-import android.graphics.Bitmap
-import android.media.*
-import android.net.Uri
-import android.os.Build
import android.util.Log
import androidx.annotation.NonNull
-import androidx.media3.common.MediaItem
import androidx.media3.common.util.UnstableApi
-import androidx.media3.effect.Presentation
-import androidx.media3.exoplayer.SeekParameters
-import androidx.media3.inspector.frame.FrameExtractor
import androidx.media3.transformer.ExportException
-import com.google.common.util.concurrent.ListenableFuture
-import com.google.common.util.concurrent.MoreExecutors
+import io.ente.native_video_editor.core.AndroidVideoExportEngine
+import io.ente.native_video_editor.core.AndroidVideoFrameExtractor
+import io.ente.native_video_editor.core.AndroidVideoMetadataReader
+import io.ente.native_video_editor.core.VideoCrop
+import io.ente.native_video_editor.core.VideoEditRequest
+import io.ente.native_video_editor.core.VideoEditResult
+import io.ente.native_video_editor.core.VideoFrameExtractionResult
+import io.ente.native_video_editor.core.VideoFramePolicy
+import io.ente.native_video_editor.core.VideoFrameRequest
+import io.ente.native_video_editor.core.VideoMetadata
import io.flutter.embedding.engine.plugins.FlutterPlugin
import io.flutter.plugin.common.EventChannel
import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel
import io.flutter.plugin.common.MethodChannel.MethodCallHandler
import io.flutter.plugin.common.MethodChannel.Result
-import kotlinx.coroutines.*
-import kotlinx.coroutines.sync.Semaphore
-import kotlinx.coroutines.sync.withPermit
-import java.io.File
-import java.io.FileOutputStream
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.CoroutineStart
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
import java.util.UUID
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.ExecutionException
-import java.util.concurrent.atomic.AtomicBoolean
-import kotlin.math.max
-import kotlin.math.roundToInt
@UnstableApi
class NativeVideoEditorPlugin : FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler {
private lateinit var channel: MethodChannel
private lateinit var progressChannel: EventChannel
private var progressEventSink: EventChannel.EventSink? = null
- private lateinit var context: Context
- private var scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
+ private var scope = newPluginScope()
private var currentExportJob: Job? = null
- private val frameJobs = ConcurrentHashMap<String, Job>()
- private val frameSemaphore = Semaphore(2)
+ private val frameJobs = mutableMapOf<String, Job>()
+ private lateinit var metadataReader: AndroidVideoMetadataReader
+ private lateinit var frameExtractor: AndroidVideoFrameExtractor
+ private lateinit var videoExporter: AndroidVideoExportEngine
- // Unified Media3 Transformer processor for all operations
- private lateinit var media3Processor: Media3TransformerProcessor
+ override fun onAttachedToEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) {
+ if (!scope.isActive) scope = newPluginScope()
- companion object {
- private const val TAG = "NativeVideoEditorPlugin"
- }
-
- override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
- if (!scope.isActive) {
- scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
- }
- context = flutterPluginBinding.applicationContext
- channel = MethodChannel(flutterPluginBinding.binaryMessenger, "native_video_editor")
+ channel = MethodChannel(binding.binaryMessenger, METHOD_CHANNEL)
channel.setMethodCallHandler(this)
-
- progressChannel = EventChannel(flutterPluginBinding.binaryMessenger, "native_video_editor/progress")
+ progressChannel = EventChannel(binding.binaryMessenger, PROGRESS_CHANNEL)
progressChannel.setStreamHandler(this)
- // Initialize unified Media3 processor
- media3Processor = Media3TransformerProcessor(context)
+ metadataReader = AndroidVideoMetadataReader()
+ frameExtractor = AndroidVideoFrameExtractor(binding.applicationContext, metadataReader)
+ videoExporter = AndroidVideoExportEngine(binding.applicationContext, metadataReader)
}
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
- val engineScope = scope
- if (!engineScope.isActive) return
- when (call.method) {
- "processVideo" -> {
- handleProcessVideo(call, result, engineScope)
- }
-
- "getVideoInfo" -> {
- val videoPath = call.argument<String>("videoPath")
- if (videoPath.isNullOrBlank()) {
- result.error("INVALID_ARGS", "Missing video path", null)
- return
- }
- engineScope.launch {
- try {
- val info = getVideoInfo(videoPath)
- withContext(Dispatchers.Main) {
- if (canReply(engineScope)) {
- result.success(info)
- }
- }
- } catch (e: LinkageError) {
- reportProcessingError(
- result,
- "INFO_LINKAGE_ERROR",
- "Get video info",
- e,
- engineScope
- )
- } catch (e: Exception) {
- reportProcessingError(
- result,
- "INFO_ERROR",
- "Get video info",
- e,
- engineScope
- )
- }
- }
- }
-
- "extractFrame" -> {
- val outputPath = call.argument<String>("outputPath")
- val positionMs = call.argument<Number>("positionMs")
- if (outputPath.isNullOrBlank() || positionMs == null) {
- result.error("INVALID_ARGS", "Invalid frame arguments", null)
- return
- }
- handleExtractFrames(
- call = call,
- result = result,
- requestId = UUID.randomUUID().toString(),
- outputPaths = listOf(outputPath),
- positionsMs = listOf(positionMs.toLong()),
- engineScope = engineScope
- )
- }
-
- "extractTimeline" -> {
- val requestId = call.argument<String>("requestId")
- val outputPaths = call.argument<List<String>>("outputPaths")
- val positions = call.argument<List<Number>>("positionsMs")
- if (requestId.isNullOrBlank() || outputPaths == null || positions == null) {
- result.error("INVALID_ARGS", "Invalid timeline arguments", null)
- return
- }
- val positionsMs = positions.map(Number::toLong)
- handleExtractFrames(
- call,
- result,
- requestId,
- outputPaths,
- positionsMs,
- engineScope
- )
- }
-
- "cancelFrameExtraction" -> {
- val requestId = call.argument<String>("requestId")
- if (requestId.isNullOrBlank()) {
- result.error("INVALID_ARGS", "Missing frame request ID", null)
- return
- }
- val frameJob = frameJobs[requestId]
- if (frameJob == null) {
- result.success(null)
- } else {
- engineScope.launch {
- frameJob.cancelAndJoin()
- withContext(Dispatchers.Main) {
- if (canReply(engineScope)) {
- result.success(null)
- }
- }
- }
- }
- }
-
- "cancelProcessing" -> {
- val exportJob = currentExportJob
- if (exportJob == null) {
- result.success(null)
- } else {
- engineScope.launch {
- exportJob.cancelAndJoin()
- withContext(Dispatchers.Main) {
- if (canReply(engineScope)) {
- result.success(null)
- }
- }
- }
- }
- }
+ val pluginScope = scope
+ if (!pluginScope.isActive) {
+ result.error("PLUGIN_DETACHED", "Native video editor is detached", null)
+ return
+ }
+ when (call.method) {
+ "getVideoInfo" -> inspectVideo(call, result, pluginScope)
+ "extractFrame" -> extractSingleFrame(call, result, pluginScope)
+ "extractTimeline" -> extractTimeline(call, result, pluginScope)
+ "cancelFrameExtraction" -> cancelFrameExtraction(call, result, pluginScope)
+ "processVideo" -> processVideo(call, result, pluginScope)
+ "cancelProcessing" -> cancelProcessing(result, pluginScope)
else -> result.notImplemented()
}
}
- private fun getVideoInfo(videoPath: String): Map<String, Any> {
- val retriever = MediaMetadataRetriever()
- return try {
- retriever.setDataSource(videoPath)
- getVideoInfo(retriever, videoPath)
- } finally {
- retriever.release()
+ private fun inspectVideo(call: MethodCall, result: Result, pluginScope: CoroutineScope) {
+ val videoPath = call.argument<String>("videoPath")
+ if (videoPath.isNullOrBlank()) {
+ result.error("INVALID_ARGS", "Missing video path", null)
+ return
}
- }
-
- private fun getVideoInfo(
- retriever: MediaMetadataRetriever,
- videoPath: String
- ): Map<String, Any> {
- val duration = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_DURATION
- )?.toLongOrNull() ?: run {
- Log.w(TAG, "Failed to extract duration metadata from video: $videoPath")
- 0L
- }
-
- val width = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH
- )?.toIntOrNull() ?: run {
- Log.w(TAG, "Failed to extract width metadata from video: $videoPath")
- 0
- }
- val height = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT
- )?.toIntOrNull() ?: run {
- Log.w(TAG, "Failed to extract height metadata from video: $videoPath")
- 0
- }
-
- val rotation = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION
- )?.toIntOrNull() ?: run {
- Log.w(TAG, "Failed to extract rotation metadata from video: $videoPath")
- 0
- }
-
- val bitrate = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_BITRATE
- )?.toLongOrNull()?.takeIf { it > 0 }
-
- val frameRate = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
- retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE
- )?.toFloatOrNull()?.takeIf { it > 0 }
- } else {
- null
+ pluginScope.launch {
+ try {
+ val info = withContext(Dispatchers.IO) { metadataReader.read(videoPath) }
+ if (canReply(pluginScope)) result.success(info.toChannelMap())
+ } catch (error: LinkageError) {
+ replyError(result, "INFO_LINKAGE_ERROR", "Get video info", error, pluginScope)
+ } catch (error: Exception) {
+ replyError(result, "INFO_ERROR", "Get video info", error, pluginScope)
}
+ }
+ }
- val normalizedRotation = ((rotation % 360) + 360) % 360
- val swapsDimensions = normalizedRotation == 90 || normalizedRotation == 270
+ private fun extractSingleFrame(
+ call: MethodCall,
+ result: Result,
+ pluginScope: CoroutineScope
+ ) {
+ val outputPath = call.argument<String>("outputPath")
+ val positionMs = call.argument<Number>("positionMs")?.toLong()
+ if (outputPath.isNullOrBlank() || positionMs == null) {
+ result.error("INVALID_ARGS", "Invalid frame arguments", null)
+ return
+ }
+ startFrameExtraction(
+ call,
+ result,
+ UUID.randomUUID().toString(),
+ listOf(outputPath),
+ listOf(positionMs),
+ pluginScope
+ )
+ }
- return buildMap {
- put("duration", duration)
- put("width", width)
- put("height", height)
- put("displayWidth", if (swapsDimensions) height else width)
- put("displayHeight", if (swapsDimensions) width else height)
- put("rotation", normalizedRotation)
- bitrate?.let { put("bitrate", it) }
- frameRate?.let { put("frameRate", it) }
- }
+ private fun extractTimeline(
+ call: MethodCall,
+ result: Result,
+ pluginScope: CoroutineScope
+ ) {
+ val requestId = call.argument<String>("requestId")
+ val outputPaths = call.argument<List<String>>("outputPaths")
+ val positionsMs = call.argument<List<Number>>("positionsMs")?.map(Number::toLong)
+ if (requestId.isNullOrBlank() || outputPaths == null || positionsMs == null) {
+ result.error("INVALID_ARGS", "Invalid timeline arguments", null)
+ return
+ }
+ startFrameExtraction(call, result, requestId, outputPaths, positionsMs, pluginScope)
}
- private fun handleExtractFrames(
+ private fun startFrameExtraction(
call: MethodCall,
result: Result,
requestId: String,
outputPaths: List<String>,
positionsMs: List<Long>,
- engineScope: CoroutineScope
+ pluginScope: CoroutineScope
) {
- val inputPath = call.argument<String>("inputPath")
- val maxWidth = call.argument<Int>("maxWidth")
- val maxHeight = call.argument<Int>("maxHeight")
- val quality = call.argument<Int>("quality")
- val policy = call.argument<String>("policy")
- val validPaths = try {
- val inputFile = inputPath?.let(::File)?.canonicalFile
- val outputs = outputPaths.map { File(it).canonicalFile }
- inputFile != null &&
- inputFile.isFile &&
- outputs.none { it == inputFile } &&
- outputs.distinct().size == outputs.size
- } catch (_: Exception) {
- false
+ if (frameJobs.containsKey(requestId)) {
+ result.error("DUPLICATE_REQUEST", "Frame request ID is already active", requestId)
+ return
}
- if (inputPath.isNullOrBlank() ||
- maxWidth == null || maxWidth <= 0 ||
- maxHeight == null || maxHeight <= 0 ||
- quality == null || quality !in 1..100 ||
- policy !in setOf("precise", "nearestSync") ||
- positionsMs.isEmpty() ||
- positionsMs.size != outputPaths.size ||
- positionsMs.any { it < 0 } ||
- outputPaths.any(String::isBlank) ||
- !validPaths
- ) {
+ val request = parseFrameRequest(call, outputPaths, positionsMs)
+ if (request == null) {
result.error("INVALID_ARGS", "Invalid frame extraction arguments", null)
return
}
- val precise = policy == "precise"
- val resultCompleted = AtomicBoolean(false)
- val job = engineScope.launch(start = CoroutineStart.LAZY) {
- val writtenFiles = mutableListOf<File>()
+ val job = pluginScope.launch(start = CoroutineStart.LAZY) {
try {
- val payload = frameSemaphore.withPermit {
- val retriever = MediaMetadataRetriever()
- var fallbackExtractor: FrameExtractor? = null
- try {
- retriever.setDataSource(inputPath)
- val videoInfo = getVideoInfo(retriever, inputPath)
- val durationMs = (videoInfo["duration"] as Long).coerceAtLeast(0L)
- val displayWidth = videoInfo["displayWidth"] as Int
- val displayHeight = videoInfo["displayHeight"] as Int
- val targetSize = boundedSize(
- displayWidth,
- displayHeight,
- maxWidth,
- maxHeight
- )
- val option = if (precise) {
- MediaMetadataRetriever.OPTION_CLOSEST
- } else {
- MediaMetadataRetriever.OPTION_CLOSEST_SYNC
- }
- val frames = ArrayList<Map<String, Any>>(positionsMs.size)
-
- positionsMs.forEachIndexed { index, requestedPositionMs ->
- ensureActive()
- val positionMs = clampFramePosition(requestedPositionMs, durationMs)
- val frame = try {
- extractScaledFrame(
- retriever,
- positionMs * 1_000,
- option,
- targetSize.first,
- targetSize.second
- )
- } catch (error: RuntimeException) {
- Log.w(TAG, "Platform frame decoder failed at ${positionMs}ms", error)
- null
- } ?: run {
- Log.w(
- TAG,
- "Platform decoder returned no frame at ${positionMs}ms; using Media3"
- )
- val extractor = fallbackExtractor ?: createFallbackFrameExtractor(
- inputPath,
- targetSize.second,
- precise
- ).also { fallbackExtractor = it }
- extractFallbackFrame(extractor, positionMs)
- }
- ensureActive()
-
- val outputFile = File(outputPaths[index])
- outputFile.parentFile?.mkdirs()
- try {
- try {
- FileOutputStream(outputFile, false).use { stream ->
- if (!frame.compress(Bitmap.CompressFormat.JPEG, quality, stream)) {
- throw IllegalStateException("JPEG encoder rejected the frame")
- }
- }
- } catch (error: Exception) {
- outputFile.delete()
- throw error
- }
- writtenFiles.add(outputFile)
- frames.add(
- mapOf(
- "outputPath" to outputFile.path,
- "width" to frame.width,
- "height" to frame.height
- )
- )
- } finally {
- frame.recycle()
- }
- }
-
- mapOf<String, Any>("videoInfo" to videoInfo, "frames" to frames)
- } finally {
- retriever.release()
- fallbackExtractor?.let { extractor ->
- withContext(NonCancellable + Dispatchers.Main) {
- extractor.close()
- }
- }
- }
- }
- withContext(Dispatchers.Main) {
- if (resultCompleted.compareAndSet(false, true) && canReply(engineScope)) {
- result.success(payload)
- }
- }
- } catch (e: CancellationException) {
- withContext(NonCancellable + Dispatchers.Main) {
- if (resultCompleted.compareAndSet(false, true)) {
- writtenFiles.forEach(File::delete)
- if (canReply(engineScope)) {
- result.error("FRAME_CANCELLED", "Frame extraction was cancelled", requestId)
- }
- }
- }
- } catch (e: Exception) {
- if (resultCompleted.compareAndSet(false, true)) {
- writtenFiles.forEach(File::delete)
- reportProcessingError(
- result,
- "FRAME_EXTRACTION_ERROR",
- "Frame extraction",
- e,
- engineScope
- )
- }
+ val extraction = frameExtractor.extract(request)
+ if (canReply(pluginScope)) result.success(extraction.toChannelMap())
+ } catch (error: CancellationException) {
+ replyError(
+ result,
+ "FRAME_CANCELLED",
+ "Frame extraction was cancelled",
+ error,
+ pluginScope,
+ requestId
+ )
+ } catch (error: LinkageError) {
+ replyError(
+ result,
+ "FRAME_LINKAGE_ERROR",
+ "Frame extraction",
+ error,
+ pluginScope
+ )
+ } catch (error: Exception) {
+ replyError(
+ result,
+ "FRAME_EXTRACTION_ERROR",
+ "Frame extraction",
+ error,
+ pluginScope
+ )
} finally {
- frameJobs.remove(requestId, coroutineContext.job)
+ frameJobs.remove(requestId)
}
}
- if (frameJobs.putIfAbsent(requestId, job) != null) {
- job.cancel()
- if (resultCompleted.compareAndSet(false, true)) {
- result.error("DUPLICATE_REQUEST", "Frame request ID is already active", requestId)
- }
- return
- }
+ frameJobs[requestId] = job
job.start()
}
- private fun clampFramePosition(positionMs: Long, durationMs: Long): Long {
- if (durationMs <= 0) return max(0L, positionMs)
- return positionMs.coerceIn(0L, max(0L, durationMs - 1L))
- }
-
- private fun boundedSize(
- width: Int,
- height: Int,
- maxWidth: Int,
- maxHeight: Int
- ): Pair<Int, Int> {
- require(width > 0 && height > 0) { "Video dimensions are unavailable" }
- val scale = minOf(
- 1.0,
- maxWidth.toDouble() / width,
- maxHeight.toDouble() / height
- )
- return Pair(
- max(1, (width * scale).roundToInt()),
- max(1, (height * scale).roundToInt())
- )
- }
-
- private fun extractScaledFrame(
- retriever: MediaMetadataRetriever,
- positionUs: Long,
- option: Int,
- targetWidth: Int,
- targetHeight: Int
- ): Bitmap? {
- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
- return retriever.getScaledFrameAtTime(
- positionUs,
- option,
- targetWidth,
- targetHeight
- )
+ private fun cancelFrameExtraction(
+ call: MethodCall,
+ result: Result,
+ pluginScope: CoroutineScope
+ ) {
+ val requestId = call.argument<String>("requestId")
+ if (requestId.isNullOrBlank()) {
+ result.error("INVALID_ARGS", "Missing frame request ID", null)
+ return
}
-
- val source = retriever.getFrameAtTime(positionUs, option) ?: return null
- if (source.width <= targetWidth && source.height <= targetHeight) {
- return source
+ val job = frameJobs[requestId]
+ if (job == null) {
+ result.success(null)
+ return
}
- val scale = minOf(
- 1.0,
- targetWidth.toDouble() / source.width,
- targetHeight.toDouble() / source.height
- )
- val scaled = Bitmap.createScaledBitmap(
- source,
- max(1, (source.width * scale).roundToInt()),
- max(1, (source.height * scale).roundToInt()),
- true
- )
- if (scaled !== source) source.recycle()
- return scaled
- }
-
- private suspend fun createFallbackFrameExtractor(
- inputPath: String,
- targetHeight: Int,
- precise: Boolean
- ): FrameExtractor = withContext(Dispatchers.Main) {
- FrameExtractor.Builder(
- context,
- MediaItem.fromUri(Uri.fromFile(File(inputPath)))
- )
- .setEffects(listOf(Presentation.createForHeight(targetHeight)))
- .setSeekParameters(
- if (precise) SeekParameters.EXACT else SeekParameters.CLOSEST_SYNC
- )
- .build()
- }
-
- private suspend fun extractFallbackFrame(
- extractor: FrameExtractor,
- positionMs: Long
- ): Bitmap {
- val future = withContext(Dispatchers.Main) {
- extractor.getFrame(positionMs)
+ pluginScope.launch {
+ job.cancelAndJoin()
+ if (canReply(pluginScope)) result.success(null)
}
- val frame = future.awaitResult()
- return frame.bitmap
}
- private suspend fun <T> ListenableFuture<T>.awaitResult(): T =
- suspendCancellableCoroutine { continuation ->
- continuation.invokeOnCancellation { cancel(true) }
- addListener(
- {
- try {
- continuation.resumeWith(kotlin.Result.success(get()))
- } catch (error: ExecutionException) {
- continuation.resumeWith(
- kotlin.Result.failure(error.cause ?: error)
- )
- } catch (error: Exception) {
- continuation.resumeWith(kotlin.Result.failure(error))
- }
- },
- MoreExecutors.directExecutor()
- )
- }
-
- private fun handleProcessVideo(
- call: MethodCall,
- result: Result,
- engineScope: CoroutineScope
- ) {
- val inputPath = call.argument<String>("inputPath")
- val outputPath = call.argument<String>("outputPath")
- if (inputPath.isNullOrBlank() ||
- outputPath.isNullOrBlank() ||
- currentExportJob?.isActive == true
- ) {
+ private fun processVideo(call: MethodCall, result: Result, pluginScope: CoroutineScope) {
+ if (currentExportJob?.isActive == true) {
result.error("INVALID_ARGS", "Invalid or concurrent export request", null)
return
}
+ val request = parseEditRequest(call)
+ if (request == null) {
+ result.error("INVALID_ARGS", "Invalid video processing arguments", null)
+ return
+ }
- val job = engineScope.launch(start = CoroutineStart.LAZY) {
+ val job = pluginScope.launch(start = CoroutineStart.LAZY) {
try {
- // Collect all transformation parameters
- val trimStartMs = call.argument<Number>("trimStartMs")?.toLong()
- val trimEndMs = call.argument<Number>("trimEndMs")?.toLong()
- val rotateDegrees = call.argument<Int>("rotateDegrees")
- val cropX = call.argument<Int>("cropX")
- val cropY = call.argument<Int>("cropY")
- val cropWidth = call.argument<Int>("cropWidth")
- val cropHeight = call.argument<Int>("cropHeight")
-
- // Process all transformations in a single pass with Media3 Transformer
- val isReEncoded = media3Processor.processVideo(
- inputPath = inputPath,
- outputPath = outputPath,
- trimStartMs = trimStartMs,
- trimEndMs = trimEndMs,
- rotateDegrees = rotateDegrees,
- cropX = cropX,
- cropY = cropY,
- cropWidth = cropWidth,
- cropHeight = cropHeight,
- onProgress = { progress ->
- engineScope.launch(Dispatchers.Main) {
- if (canReply(engineScope)) {
- progressEventSink?.success(progress)
- }
- }
- }
- )
-
- withContext(Dispatchers.Main) {
- if (canReply(engineScope)) {
- result.success(
- mapOf(
- "outputPath" to outputPath,
- "isReEncoded" to isReEncoded
- )
- )
- }
+ val export = videoExporter.processVideo(request) { progress ->
+ if (canReply(pluginScope)) progressEventSink?.success(progress)
}
- } catch (e: LinkageError) {
- reportProcessingError(
+ if (canReply(pluginScope)) result.success(export.toChannelMap())
+ } catch (error: LinkageError) {
+ replyError(
result,
"PROCESS_LINKAGE_ERROR",
"Process video",
- e,
- engineScope
+ error,
+ pluginScope
)
- } catch (e: CancellationException) {
- reportProcessingError(
+ } catch (error: CancellationException) {
+ replyError(
result,
"PROCESS_CANCELLED",
"Process video",
- e,
- engineScope
- )
- } catch (e: Exception) {
- reportProcessingError(
- result,
- "PROCESS_ERROR",
- "Process video",
- e,
- engineScope
+ error,
+ pluginScope
)
+ } catch (error: Exception) {
+ replyError(result, "PROCESS_ERROR", "Process video", error, pluginScope)
} finally {
- if (currentExportJob == coroutineContext.job) {
- currentExportJob = null
- }
+ if (currentExportJob == coroutineContext[Job]) currentExportJob = null
}
}
currentExportJob = job
job.start()
}
- private suspend fun reportProcessingError(
+ private fun cancelProcessing(result: Result, pluginScope: CoroutineScope) {
+ val job = currentExportJob
+ if (job == null) {
+ result.success(null)
+ return
+ }
+ pluginScope.launch {
+ job.cancelAndJoin()
+ if (canReply(pluginScope)) result.success(null)
+ }
+ }
+
+ private fun parseFrameRequest(
+ call: MethodCall,
+ outputPaths: List<String>,
+ positionsMs: List<Long>
+ ): VideoFrameRequest? = runCatching {
+ VideoFrameRequest(
+ inputPath = requireNotNull(call.argument<String>("inputPath")),
+ outputPaths = outputPaths,
+ positionsMs = positionsMs,
+ maxWidth = requireNotNull(call.argument<Int>("maxWidth")),
+ maxHeight = requireNotNull(call.argument<Int>("maxHeight")),
+ quality = requireNotNull(call.argument<Int>("quality")),
+ policy = when (call.argument<String>("policy")) {
+ "precise" -> VideoFramePolicy.PRECISE
+ "nearestSync" -> VideoFramePolicy.NEAREST_SYNC
+ else -> error("Invalid frame policy")
+ }
+ )
+ }.getOrNull()
+
+ private fun parseEditRequest(call: MethodCall): VideoEditRequest? = runCatching {
+ val cropValues = listOf(
+ call.argument<Int>("cropX"),
+ call.argument<Int>("cropY"),
+ call.argument<Int>("cropWidth"),
+ call.argument<Int>("cropHeight")
+ )
+ require(cropValues.all { it == null } || cropValues.all { it != null })
+ val crop = if (cropValues.first() == null) {
+ null
+ } else {
+ VideoCrop(
+ cropValues[0]!!,
+ cropValues[1]!!,
+ cropValues[2]!!,
+ cropValues[3]!!
+ )
+ }
+ VideoEditRequest(
+ inputPath = requireNotNull(call.argument<String>("inputPath")),
+ outputPath = requireNotNull(call.argument<String>("outputPath")),
+ trimStartMs = call.argument<Number>("trimStartMs")?.toLong(),
+ trimEndMs = call.argument<Number>("trimEndMs")?.toLong(),
+ rotateDegrees = call.argument<Int>("rotateDegrees"),
+ crop = crop
+ )
+ }.getOrNull()
+
+ private suspend fun replyError(
result: Result,
code: String,
operation: String,
- throwable: Throwable,
- engineScope: CoroutineScope
+ error: Throwable,
+ pluginScope: CoroutineScope,
+ details: Any? = null
) {
- Log.e(TAG, "$operation failed", throwable)
- withContext(NonCancellable + Dispatchers.Main) {
- if (canReply(engineScope)) {
+ Log.e(TAG, "$operation failed", error)
+ withContext(NonCancellable + Dispatchers.Main.immediate) {
+ if (canReply(pluginScope)) {
result.error(
code,
- throwable.message ?: throwable::class.java.simpleName,
- buildErrorDetails(throwable)
+ if (code.endsWith("CANCELLED")) operation else error.message ?: operation,
+ details ?: buildErrorDetails(error)
)
}
}
}
- private fun canReply(engineScope: CoroutineScope): Boolean = engineScope.isActive
+ private fun canReply(pluginScope: CoroutineScope): Boolean = pluginScope.isActive
override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) {
progressEventSink = null
@@ -647,37 +343,59 @@ class NativeVideoEditorPlugin : FlutterPlugin, MethodCallHandler, EventChannel.S
}
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
- if (scope.isActive) {
- progressEventSink = events
- }
+ if (scope.isActive) progressEventSink = events
}
override fun onCancel(arguments: Any?) {
progressEventSink = null
}
- private fun buildErrorDetails(throwable: Throwable): Map<String, Any?> {
+ private fun VideoMetadata.toChannelMap(): Map<String, Any> = buildMap {
+ put("duration", durationMs)
+ put("width", width)
+ put("height", height)
+ put("displayWidth", displayWidth)
+ put("displayHeight", displayHeight)
+ put("rotation", rotationDegrees)
+ bitrate?.let { put("bitrate", it) }
+ frameRate?.let { put("frameRate", it) }
+ }
+
+ private fun VideoFrameExtractionResult.toChannelMap(): Map<String, Any> = mapOf(
+ "videoInfo" to videoInfo.toChannelMap(),
+ "frames" to frames.map {
+ mapOf("outputPath" to it.outputPath, "width" to it.width, "height" to it.height)
+ }
+ )
+
+ private fun VideoEditResult.toChannelMap(): Map<String, Any> = mapOf(
+ "outputPath" to outputPath,
+ "isReEncoded" to isReEncoded
+ )
+
+ private fun buildErrorDetails(error: Throwable): Map<String, Any?> {
val details = mutableMapOf<String, Any?>(
- "type" to throwable::class.java.simpleName,
- "message" to (throwable.message ?: "")
+ "type" to error::class.java.simpleName,
+ "message" to (error.message ?: "")
)
-
val exportException = when {
- throwable is ExportException -> throwable
- throwable.cause is ExportException -> throwable.cause as ExportException
+ error is ExportException -> error
+ error.cause is ExportException -> error.cause as ExportException
else -> null
}
-
- exportException?.let {
- details["errorCode"] = it.errorCode
- }
-
- val rootCause = throwable.cause ?: exportException?.cause
- rootCause?.let {
+ exportException?.let { details["errorCode"] = it.errorCode }
+ (error.cause ?: exportException?.cause)?.let {
details["causeType"] = it::class.java.simpleName
details["causeMessage"] = it.message ?: ""
}
-
return details
}
+
+ private companion object {
+ const val TAG = "NativeVideoEditorPlugin"
+ const val METHOD_CHANNEL = "native_video_editor"
+ const val PROGRESS_CHANNEL = "native_video_editor/progress"
+
+ fun newPluginScope() = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
+ }
}
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/Media3TransformerProcessor.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoExportEngine.kt
similarity index 73%
rename from mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/Media3TransformerProcessor.kt
rename to mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoExportEngine.kt
index 156caa713f..06229d83bb 100644
--- a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/Media3TransformerProcessor.kt
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoExportEngine.kt
@@ -1,9 +1,8 @@
-package io.ente.native_video_editor
+package io.ente.native_video_editor.core
import android.content.Context
import android.media.MediaExtractor
import android.media.MediaFormat
-import android.media.MediaMetadataRetriever
import android.net.Uri
import android.os.Handler
import android.os.Looper
@@ -36,53 +35,29 @@ import kotlinx.coroutines.withContext
import java.io.File
@UnstableApi
-class Media3TransformerProcessor(private val context: Context) {
+class AndroidVideoExportEngine(
+ private val context: Context,
+ private val metadataReader: AndroidVideoMetadataReader
+) {
suspend fun processVideo(
- inputPath: String,
- outputPath: String,
- trimStartMs: Long? = null,
- trimEndMs: Long? = null,
- rotateDegrees: Int? = null,
- cropX: Int? = null,
- cropY: Int? = null,
- cropWidth: Int? = null,
- cropHeight: Int? = null,
+ request: VideoEditRequest,
onProgress: ((Float) -> Unit)? = null
- ): Boolean {
- val inputFile = File(inputPath).canonicalFile
- val outputFile = File(outputPath).canonicalFile
+ ): VideoEditResult = withContext(Dispatchers.IO) {
+ val inputFile = File(request.inputPath).canonicalFile
+ val outputFile = File(request.outputPath).canonicalFile
require(inputFile.isFile && inputFile != outputFile) {
"Input must be a video file and output must be a different path"
}
- require((trimStartMs == null) == (trimEndMs == null)) {
- "Trim bounds must be provided together"
- }
- require(trimStartMs == null || (trimStartMs >= 0 && trimStartMs < trimEndMs!!)) {
- "Invalid trim bounds"
- }
- require(rotateDegrees == null || rotateDegrees in setOf(0, 90, 180, 270)) {
- "Invalid rotation"
- }
- val cropValues = listOf(cropX, cropY, cropWidth, cropHeight)
- require(cropValues.all { it == null } || cropValues.all { it != null }) {
- "Crop values must be provided together"
- }
- val crop = if (cropX == null) {
- null
- } else {
- CropSpec(cropX, cropY!!, cropWidth!!, cropHeight!!)
- }
- require(crop == null || crop.isPositive) { "Invalid crop rectangle" }
- return try {
+ try {
val clipRange = resolveVideoClipRangeUs(
videoDurationUs = getVideoTrackDurationUs(inputFile.path),
- requestedStartUs = trimStartMs?.times(1_000L),
- requestedEndUs = trimEndMs?.times(1_000L)
+ requestedStartUs = request.trimStartMs?.times(1_000L),
+ requestedEndUs = request.trimEndMs?.times(1_000L)
)
val mediaItem = buildMediaItem(inputFile, clipRange)
- val videoInfo = crop?.let { getVideoInfo(inputFile.path) }
- val effects = buildVideoEffects(videoInfo, crop, rotateDegrees)
+ val videoInfo = request.crop?.let { metadataReader.read(inputFile.path) }
+ val effects = buildVideoEffects(videoInfo, request.crop, request.rotateDegrees)
val editedMediaItem = EditedMediaItem.Builder(mediaItem).apply {
if (effects.isNotEmpty()) {
setEffects(Effects(emptyList(), effects))
@@ -97,18 +72,19 @@ class Media3TransformerProcessor(private val context: Context) {
composition = composition,
outputPath = outputFile.path,
transcodeVideo = effects.isNotEmpty(),
- optimizeTrim = trimStartMs != null && effects.isEmpty(),
+ optimizeTrim = request.trimStartMs != null && effects.isEmpty(),
onProgress = onProgress
)
}
require(outputFile.isFile && outputFile.length() > 0) {
"Output file was not created"
}
- when (exportResult.videoConversionProcess) {
+ val isReEncoded = when (exportResult.videoConversionProcess) {
ExportResult.CONVERSION_PROCESS_TRANSCODED,
ExportResult.CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED -> true
else -> false
}
+ VideoEditResult(request.outputPath, isReEncoded)
} catch (error: CancellationException) {
outputFile.delete()
throw error
@@ -133,22 +109,14 @@ class Media3TransformerProcessor(private val context: Context) {
.build()
private fun buildVideoEffects(
- videoInfo: VideoInfo?,
- crop: CropSpec?,
+ videoInfo: VideoMetadata?,
+ crop: VideoCrop?,
rotateDegrees: Int?
): List<Effect> = buildList {
val cropSize = crop?.let {
val video = requireNotNull(videoInfo)
- val displayWidth = if (video.rotation == 90 || video.rotation == 270) {
- video.height
- } else {
- video.width
- }
- val displayHeight = if (video.rotation == 90 || video.rotation == 270) {
- video.width
- } else {
- video.height
- }
+ val displayWidth = video.displayWidth
+ val displayHeight = video.displayHeight
require(displayWidth > 0 && displayHeight > 0) {
"Video dimensions are unavailable"
}
@@ -249,26 +217,6 @@ class Media3TransformerProcessor(private val context: Context) {
}
}
- private fun getVideoInfo(videoPath: String): VideoInfo {
- val retriever = MediaMetadataRetriever()
- return try {
- retriever.setDataSource(videoPath)
- VideoInfo(
- width = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH
- )?.toIntOrNull() ?: 0,
- height = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT
- )?.toIntOrNull() ?: 0,
- rotation = retriever.extractMetadata(
- MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION
- )?.toIntOrNull()?.let { ((it % 360) + 360) % 360 } ?: 0
- )
- } finally {
- retriever.release()
- }
- }
-
private fun getVideoTrackDurationUs(videoPath: String): Long {
val extractor = MediaExtractor()
return try {
@@ -307,16 +255,4 @@ internal fun resolveVideoClipRangeUs(
return VideoClipRangeUs(startUs, endUs)
}
-private data class CropSpec(
- val x: Int,
- val y: Int,
- val width: Int,
- val height: Int
-) {
- val isPositive: Boolean
- get() = x >= 0 && y >= 0 && width > 0 && height > 0
-}
-
-private data class VideoInfo(val width: Int, val height: Int, val rotation: Int)
-
class VideoProcessingException(message: String, cause: Throwable? = null) : Exception(message, cause)
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoFrameExtractor.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoFrameExtractor.kt
new file mode 100644
index 0000000000..65064a3362
--- /dev/null
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoFrameExtractor.kt
@@ -0,0 +1,204 @@
+package io.ente.native_video_editor.core
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.media.MediaMetadataRetriever
+import android.net.Uri
+import android.os.Build
+import android.util.Log
+import androidx.media3.common.MediaItem
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.effect.Presentation
+import androidx.media3.exoplayer.SeekParameters
+import androidx.media3.inspector.frame.FrameExtractor
+import com.google.common.util.concurrent.ListenableFuture
+import com.google.common.util.concurrent.MoreExecutors
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
+import kotlinx.coroutines.withContext
+import java.io.File
+import java.io.FileOutputStream
+import java.util.concurrent.ExecutionException
+import kotlin.coroutines.coroutineContext
+
+@UnstableApi
+class AndroidVideoFrameExtractor(
+ private val context: Context,
+ private val metadataReader: AndroidVideoMetadataReader
+) {
+ private val permits = Semaphore(2)
+
+ suspend fun extract(request: VideoFrameRequest): VideoFrameExtractionResult {
+ val writtenFiles = mutableListOf<File>()
+ return try {
+ withContext(Dispatchers.IO) {
+ val inputFile = File(request.inputPath).canonicalFile
+ val outputFiles = request.outputPaths.map { File(it).canonicalFile }
+ require(inputFile.isFile && outputFiles.none { it == inputFile }) {
+ "Input must be a video file and outputs must use different paths"
+ }
+
+ permits.withPermit {
+ extractWithRetriever(request, outputFiles, writtenFiles)
+ }
+ }
+ } catch (error: Throwable) {
+ writtenFiles.forEach(File::delete)
+ throw error
+ }
+ }
+
+ private suspend fun extractWithRetriever(
+ request: VideoFrameRequest,
+ outputFiles: List<File>,
+ writtenFiles: MutableList<File>
+ ): VideoFrameExtractionResult {
+ val retriever = MediaMetadataRetriever()
+ var fallbackExtractor: FrameExtractor? = null
+ try {
+ retriever.setDataSource(request.inputPath)
+ val videoInfo = metadataReader.read(retriever, request.inputPath)
+ val targetSize = boundedVideoSize(
+ videoInfo.displayWidth,
+ videoInfo.displayHeight,
+ request.maxWidth,
+ request.maxHeight
+ )
+ val precise = request.policy == VideoFramePolicy.PRECISE
+ val option = if (precise) {
+ MediaMetadataRetriever.OPTION_CLOSEST
+ } else {
+ MediaMetadataRetriever.OPTION_CLOSEST_SYNC
+ }
+ val frames = ArrayList<VideoFrameResult>(request.positionsMs.size)
+
+ request.positionsMs.forEachIndexed { index, requestedPositionMs ->
+ coroutineContext.ensureActive()
+ val positionMs = clampFramePositionMs(requestedPositionMs, videoInfo.durationMs)
+ val frame = try {
+ extractScaledFrame(
+ retriever,
+ positionMs * 1_000,
+ option,
+ targetSize.width,
+ targetSize.height
+ )
+ } catch (error: RuntimeException) {
+ Log.w(TAG, "Platform frame decoder failed at ${positionMs}ms", error)
+ null
+ } ?: run {
+ Log.w(TAG, "Platform decoder returned no frame at ${positionMs}ms; using Media3")
+ val extractor = fallbackExtractor ?: createFallbackExtractor(
+ request.inputPath,
+ targetSize.height,
+ precise
+ ).also { fallbackExtractor = it }
+ extractFallbackFrame(extractor, positionMs)
+ }
+ coroutineContext.ensureActive()
+
+ val outputFile = outputFiles[index]
+ outputFile.parentFile?.mkdirs()
+ try {
+ try {
+ FileOutputStream(outputFile, false).use { stream ->
+ check(frame.compress(Bitmap.CompressFormat.JPEG, request.quality, stream)) {
+ "JPEG encoder rejected the frame"
+ }
+ }
+ } catch (error: Exception) {
+ outputFile.delete()
+ throw error
+ }
+ writtenFiles.add(outputFile)
+ frames.add(
+ VideoFrameResult(request.outputPaths[index], frame.width, frame.height)
+ )
+ } finally {
+ frame.recycle()
+ }
+ }
+
+ return VideoFrameExtractionResult(videoInfo, frames)
+ } finally {
+ retriever.release()
+ fallbackExtractor?.let { extractor ->
+ withContext(NonCancellable + Dispatchers.Main) {
+ extractor.close()
+ }
+ }
+ }
+ }
+
+ private fun extractScaledFrame(
+ retriever: MediaMetadataRetriever,
+ positionUs: Long,
+ option: Int,
+ targetWidth: Int,
+ targetHeight: Int
+ ): Bitmap? {
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
+ return retriever.getScaledFrameAtTime(
+ positionUs,
+ option,
+ targetWidth,
+ targetHeight
+ )
+ }
+
+ val source = retriever.getFrameAtTime(positionUs, option) ?: return null
+ if (source.width <= targetWidth && source.height <= targetHeight) return source
+
+ val size = boundedVideoSize(source.width, source.height, targetWidth, targetHeight)
+ val scaled = Bitmap.createScaledBitmap(source, size.width, size.height, true)
+ if (scaled !== source) source.recycle()
+ return scaled
+ }
+
+ private suspend fun createFallbackExtractor(
+ inputPath: String,
+ targetHeight: Int,
+ precise: Boolean
+ ): FrameExtractor = withContext(Dispatchers.Main) {
+ FrameExtractor.Builder(
+ context,
+ MediaItem.fromUri(Uri.fromFile(File(inputPath)))
+ )
+ .setEffects(listOf(Presentation.createForHeight(targetHeight)))
+ .setSeekParameters(if (precise) SeekParameters.EXACT else SeekParameters.CLOSEST_SYNC)
+ .build()
+ }
+
+ private suspend fun extractFallbackFrame(
+ extractor: FrameExtractor,
+ positionMs: Long
+ ): Bitmap {
+ val future = withContext(Dispatchers.Main) { extractor.getFrame(positionMs) }
+ return future.awaitResult().bitmap
+ }
+
+ private suspend fun <T> ListenableFuture<T>.awaitResult(): T =
+ suspendCancellableCoroutine { continuation ->
+ continuation.invokeOnCancellation { cancel(true) }
+ addListener(
+ {
+ try {
+ continuation.resumeWith(Result.success(get()))
+ } catch (error: ExecutionException) {
+ continuation.resumeWith(Result.failure(error.cause ?: error))
+ } catch (error: Exception) {
+ continuation.resumeWith(Result.failure(error))
+ }
+ },
+ MoreExecutors.directExecutor()
+ )
+ }
+
+ private companion object {
+ const val TAG = "VideoFrameExtractor"
+ }
+}
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoMetadataReader.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoMetadataReader.kt
new file mode 100644
index 0000000000..615fdd4aa2
--- /dev/null
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/AndroidVideoMetadataReader.kt
@@ -0,0 +1,90 @@
+package io.ente.native_video_editor.core
+
+import android.media.MediaMetadataRetriever
+import android.os.Build
+import android.util.Log
+
+class AndroidVideoMetadataReader {
+ fun read(videoPath: String): VideoMetadata {
+ val retriever = MediaMetadataRetriever()
+ return try {
+ retriever.setDataSource(videoPath)
+ read(retriever, videoPath)
+ } finally {
+ retriever.release()
+ }
+ }
+
+ fun read(retriever: MediaMetadataRetriever, videoPath: String): VideoMetadata {
+ val durationMs = metadataLong(
+ retriever,
+ MediaMetadataRetriever.METADATA_KEY_DURATION,
+ "duration",
+ videoPath
+ ) ?: 0L
+ val width = metadataInt(
+ retriever,
+ MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH,
+ "width",
+ videoPath
+ ) ?: 0
+ val height = metadataInt(
+ retriever,
+ MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT,
+ "height",
+ videoPath
+ ) ?: 0
+ val rotation = metadataInt(
+ retriever,
+ MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION,
+ "rotation",
+ videoPath
+ ) ?: 0
+ val bitrate = retriever.extractMetadata(
+ MediaMetadataRetriever.METADATA_KEY_BITRATE
+ )?.toLongOrNull()?.takeIf { it > 0 }
+ val frameRate = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+ retriever.extractMetadata(
+ MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE
+ )?.toFloatOrNull()?.takeIf { it > 0 }
+ } else {
+ null
+ }
+
+ val normalizedRotation = ((rotation % 360) + 360) % 360
+ require(width > 0 && height > 0 && normalizedRotation in setOf(0, 90, 180, 270)) {
+ "Invalid video dimensions or rotation"
+ }
+
+ return VideoMetadata(
+ durationMs = durationMs,
+ width = width,
+ height = height,
+ rotationDegrees = normalizedRotation,
+ bitrate = bitrate,
+ frameRate = frameRate
+ )
+ }
+
+ private fun metadataInt(
+ retriever: MediaMetadataRetriever,
+ key: Int,
+ name: String,
+ path: String
+ ): Int? = retriever.extractMetadata(key)?.toIntOrNull().also {
+ if (it == null) Log.w(TAG, "Failed to extract $name metadata from video: $path")
+ }
+
+ private fun metadataLong(
+ retriever: MediaMetadataRetriever,
+ key: Int,
+ name: String,
+ path: String
+ ): Long? = retriever.extractMetadata(key)?.toLongOrNull().also {
+ if (it == null) Log.w(TAG, "Failed to extract $name metadata from video: $path")
+ }
+
+ private companion object {
+ const val TAG = "VideoMetadataReader"
+ }
+}
diff --git a/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt
new file mode 100644
index 0000000000..7b0e3c5791
--- /dev/null
+++ b/mobile/packages/native_video_editor/android/src/main/kotlin/io/ente/native_video_editor/core/VideoModels.kt
@@ -0,0 +1,120 @@
+package io.ente.native_video_editor.core
+
+import kotlin.math.max
+import kotlin.math.roundToInt
+
+data class VideoMetadata(
+ val durationMs: Long,
+ val width: Int,
+ val height: Int,
+ val rotationDegrees: Int,
+ val bitrate: Long?,
+ val frameRate: Float?
+) {
+ val displayWidth: Int
+ get() = if (rotationDegrees == 90 || rotationDegrees == 270) height else width
+
+ val displayHeight: Int
+ get() = if (rotationDegrees == 90 || rotationDegrees == 270) width else height
+}
+
+enum class VideoFramePolicy {
+ PRECISE,
+ NEAREST_SYNC;
+}
+
+data class VideoFrameRequest(
+ val inputPath: String,
+ val outputPaths: List<String>,
+ val positionsMs: List<Long>,
+ val maxWidth: Int,
+ val maxHeight: Int,
+ val quality: Int,
+ val policy: VideoFramePolicy
+) {
+ init {
+ require(inputPath.isNotBlank()) { "Missing input path" }
+ require(outputPaths.isNotEmpty() && outputPaths.size == positionsMs.size) {
+ "Every timestamp requires an output path"
+ }
+ require(outputPaths.none(String::isBlank) && outputPaths.distinct().size == outputPaths.size) {
+ "Output paths must be non-empty and unique"
+ }
+ require(positionsMs.all { it >= 0 }) { "Frame timestamps must be non-negative" }
+ require(maxWidth > 0 && maxHeight > 0) { "Frame bounds must be positive" }
+ require(quality in 1..100) { "JPEG quality must be between 1 and 100" }
+ }
+}
+
+data class VideoFrameResult(
+ val outputPath: String,
+ val width: Int,
+ val height: Int
+)
+
+data class VideoFrameExtractionResult(
+ val videoInfo: VideoMetadata,
+ val frames: List<VideoFrameResult>
+)
+
+data class VideoCrop(
+ val x: Int,
+ val y: Int,
+ val width: Int,
+ val height: Int
+) {
+ init {
+ require(x >= 0 && y >= 0 && width > 0 && height > 0) {
+ "Invalid crop rectangle"
+ }
+ }
+}
+
+data class VideoEditRequest(
+ val inputPath: String,
+ val outputPath: String,
+ val trimStartMs: Long?,
+ val trimEndMs: Long?,
+ val rotateDegrees: Int?,
+ val crop: VideoCrop?
+) {
+ init {
+ require(inputPath.isNotBlank() && outputPath.isNotBlank()) { "Missing video path" }
+ require((trimStartMs == null) == (trimEndMs == null)) {
+ "Trim bounds must be provided together"
+ }
+ require(trimStartMs == null || (trimStartMs >= 0 && trimStartMs < trimEndMs!!)) {
+ "Invalid trim bounds"
+ }
+ require(rotateDegrees == null || rotateDegrees in setOf(0, 90, 180, 270)) {
+ "Invalid rotation"
+ }
+ }
+}
+
+data class VideoEditResult(val outputPath: String, val isReEncoded: Boolean)
+
+internal data class PixelSize(val width: Int, val height: Int)
+
+internal fun boundedVideoSize(
+ width: Int,
+ height: Int,
+ maxWidth: Int,
+ maxHeight: Int
+): PixelSize {
+ require(width > 0 && height > 0) { "Video dimensions are unavailable" }
+ val scale = minOf(
+ 1.0,
+ maxWidth.toDouble() / width,
+ maxHeight.toDouble() / height
+ )
+ return PixelSize(
+ max(1, (width * scale).roundToInt()),
+ max(1, (height * scale).roundToInt())
+ )
+}
+
+internal fun clampFramePositionMs(positionMs: Long, durationMs: Long): Long {
+ if (durationMs <= 0) return max(0L, positionMs)
+ return positionMs.coerceIn(0L, max(0L, durationMs - 1L))
+}
diff --git a/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoClipRangeUsTest.kt b/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoClipRangeUsTest.kt
index f6d58f6ff7..2dc9fba73b 100644
--- a/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoClipRangeUsTest.kt
+++ b/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoClipRangeUsTest.kt
@@ -1,4 +1,4 @@
-package io.ente.native_video_editor
+package io.ente.native_video_editor.core
import org.junit.Assert.assertEquals
import org.junit.Test
diff --git a/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoModelsTest.kt b/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoModelsTest.kt
new file mode 100644
index 0000000000..66b6c66aec
--- /dev/null
+++ b/mobile/packages/native_video_editor/android/src/test/kotlin/io/ente/native_video_editor/VideoModelsTest.kt
@@ -0,0 +1,40 @@
+package io.ente.native_video_editor.core
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class VideoModelsTest {
+ @Test
+ fun metadataRotationDefinesDisplayDimensions() {
+ val metadata = VideoMetadata(
+ durationMs = 1_000,
+ width = 1_920,
+ height = 1_080,
+ rotationDegrees = 270,
+ bitrate = null,
+ frameRate = null
+ )
+
+ assertEquals(1_080, metadata.displayWidth)
+ assertEquals(1_920, metadata.displayHeight)
+ }
+
+ @Test
+ fun boundedSizePreservesAspectRatioWithoutUpscaling() {
+ assertEquals(
+ PixelSize(width = 144, height = 81),
+ boundedVideoSize(1_920, 1_080, 144, 120)
+ )
+ assertEquals(
+ PixelSize(width = 80, height = 60),
+ boundedVideoSize(80, 60, 144, 120)
+ )
+ }
+
+ @Test
+ fun framePositionStaysInsideMediaInterval() {
+ assertEquals(0, clampFramePositionMs(-5, 1_000))
+ assertEquals(999, clampFramePositionMs(1_000, 1_000))
+ assertEquals(1_500, clampFramePositionMs(1_500, 0))
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/.swift-format b/mobile/packages/native_video_editor/ios/.swift-format
new file mode 100644
index 0000000000..ae267deafa
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/.swift-format
@@ -0,0 +1,12 @@
+{
+ "indentation": {
+ "spaces": 4
+ },
+ "lineLength": 120,
+ "rules": {
+ "AllPublicDeclarationsHaveDocumentation": false,
+ "NeverForceUnwrap": false,
+ "NoAccessLevelOnExtensionDeclaration": false
+ },
+ "version": 1
+}
diff --git a/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift b/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
index 69fe0b8116..d3c55b8ca6 100644
--- a/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
+++ b/mobile/packages/native_video_editor/ios/Classes/NativeVideoEditorPlugin.swift
@@ -1,889 +1,445 @@
-import Flutter
-import AVFoundation
-import ImageIO
+@preconcurrency import Flutter
+import Foundation
-public class NativeVideoEditorPlugin: NSObject, FlutterPlugin {
+@MainActor
+public final class NativeVideoEditorPlugin: NSObject, @preconcurrency FlutterPlugin {
private var isDetached = false
- private var currentExportSession: AVAssetExportSession?
- private var exportCancellationResult: FlutterResult?
private var progressEventSink: FlutterEventSink?
- private var progressTimer: Timer?
- private var frameGenerators: [String: AVAssetImageGenerator] = [:]
- private var frameOutputPaths: [String: [String]] = [:]
- private var cancelledFrameRequests = Set<String>()
+ private var frameTasks: [String: Task<Void, Never>] = [:]
private var frameCancellationResults: [String: FlutterResult] = [:]
- private let frameStartQueue = DispatchQueue(
- label: "io.ente.native-video-editor.frames"
- )
- private let frameSemaphore = DispatchSemaphore(value: 2)
+ private var exportTask: Task<Void, Never>?
+ private var exportCancellationResult: FlutterResult?
+ private let metadataReader = VideoMetadataReader()
+ private lazy var frameExtractor = VideoFrameExtractor(metadataReader: metadataReader)
+ private let videoExporter = VideoExportEngine()
public static func register(with registrar: FlutterPluginRegistrar) {
- let channel = FlutterMethodChannel(name: "native_video_editor", binaryMessenger: registrar.messenger())
+ let channel = FlutterMethodChannel(
+ name: "native_video_editor",
+ binaryMessenger: registrar.messenger()
+ )
let instance = NativeVideoEditorPlugin()
registrar.addMethodCallDelegate(instance, channel: channel)
registrar.publish(instance)
- let progressChannel = FlutterEventChannel(name: "native_video_editor/progress", binaryMessenger: registrar.messenger())
+ let progressChannel = FlutterEventChannel(
+ name: "native_video_editor/progress",
+ binaryMessenger: registrar.messenger()
+ )
progressChannel.setStreamHandler(instance)
}
public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
- switch call.method {
- case "processVideo":
- handleProcessVideo(call: call, result: result)
-
- case "getVideoInfo":
- guard let args = call.arguments as? [String: Any],
- let videoPath = args["videoPath"] as? String else {
- result(flutterError("INVALID_ARGS", message: "Invalid arguments"))
+ guard !isDetached else {
+ result(flutterError("PLUGIN_DETACHED", message: "Native video editor is detached"))
return
}
-
- getVideoInfo(videoPath: videoPath, result: result)
-
+ switch call.method {
+ case "getVideoInfo":
+ inspectVideo(call, result: result)
case "extractFrame":
- guard let args = call.arguments as? [String: Any],
- let outputPath = args["outputPath"] as? String,
- let position = args["positionMs"] as? NSNumber else {
- result(flutterError("INVALID_ARGS", message: "Invalid frame arguments"))
- return
- }
- extractFrames(
- args: args,
- requestID: UUID().uuidString,
- outputPaths: [outputPath],
- positionsMs: [position.int64Value],
- result: result
- )
-
+ extractSingleFrame(call, result: result)
case "extractTimeline":
- guard let args = call.arguments as? [String: Any],
- let requestID = args["requestId"] as? String,
- let outputPaths = args["outputPaths"] as? [String],
- let positions = args["positionsMs"] as? [NSNumber] else {
- result(flutterError("INVALID_ARGS", message: "Invalid timeline arguments"))
- return
- }
- extractFrames(
- args: args,
- requestID: requestID,
- outputPaths: outputPaths,
- positionsMs: positions.map(\.int64Value),
- result: result
- )
-
+ extractTimeline(call, result: result)
case "cancelFrameExtraction":
- guard let args = call.arguments as? [String: Any],
- let requestID = args["requestId"] as? String else {
- result(flutterError("INVALID_ARGS", message: "Missing frame request ID"))
- return
- }
- if let generator = frameGenerators[requestID] {
- cancelledFrameRequests.insert(requestID)
- if frameCancellationResults[requestID] != nil {
- result(nil)
- return
- }
- frameCancellationResults[requestID] = result
- generator.cancelAllCGImageGeneration()
- } else {
- result(nil)
- }
-
+ cancelFrameExtraction(call, result: result)
+ case "processVideo":
+ processVideo(call, result: result)
case "cancelProcessing":
- guard let exportSession = currentExportSession else {
- result(nil)
- return
- }
- guard exportCancellationResult == nil else {
- result(nil)
- return
- }
- exportCancellationResult = result
- exportSession.cancelExport()
-
+ cancelProcessing(result: result)
default:
result(FlutterMethodNotImplemented)
}
}
- private func extractFrames(
- args: [String: Any],
- requestID: String,
- outputPaths: [String],
- positionsMs: [Int64],
- result: @escaping FlutterResult
- ) {
- guard frameGenerators[requestID] == nil else {
- result(flutterError("DUPLICATE_REQUEST", message: "Frame request ID is already active"))
- return
- }
- guard let inputPath = args["inputPath"] as? String,
- let maxWidth = args["maxWidth"] as? NSNumber,
- let maxHeight = args["maxHeight"] as? NSNumber,
- let quality = args["quality"] as? NSNumber,
- let policy = args["policy"] as? String,
- !positionsMs.isEmpty,
- positionsMs.count == outputPaths.count,
- maxWidth.intValue > 0,
- maxHeight.intValue > 0,
- (1...100).contains(quality.intValue),
- policy == "precise" || policy == "nearestSync" else {
- result(flutterError("INVALID_ARGS", message: "Invalid frame extraction arguments"))
+ private func inspectVideo(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
+ guard let args = call.arguments as? [String: Any],
+ let videoPath = args["videoPath"] as? String,
+ !videoPath.isEmpty
+ else {
+ result(flutterError("INVALID_ARGS", message: "Invalid arguments"))
return
}
- let inputURL = URL(fileURLWithPath: inputPath)
- .standardizedFileURL.resolvingSymlinksInPath()
- let outputURLs = outputPaths.map {
- URL(fileURLWithPath: $0).standardizedFileURL.resolvingSymlinksInPath()
+
+ Task { [weak self] in
+ guard let self else { return }
+ do {
+ let info = try await metadataReader.read(
+ from: URL(fileURLWithPath: videoPath)
+ )
+ guard !isDetached else { return }
+ result(info.channelValue)
+ } catch {
+ guard !isDetached else { return }
+ result(flutterError(error, defaultCode: "INFO_ERROR"))
+ }
}
- guard FileManager.default.fileExists(atPath: inputURL.path),
- positionsMs.allSatisfy({ $0 >= 0 }),
- outputPaths.allSatisfy({ !$0.isEmpty }),
- !outputURLs.contains(inputURL),
- Set(outputURLs.map(\.path)).count == outputURLs.count else {
- result(flutterError("INVALID_ARGS", message: "Invalid frame paths or timestamps"))
+ }
+
+ private func extractSingleFrame(
+ _ call: FlutterMethodCall,
+ result: @escaping FlutterResult
+ ) {
+ guard let args = call.arguments as? [String: Any],
+ let outputPath = args["outputPath"] as? String,
+ !outputPath.isEmpty,
+ let positionMs = int64(args["positionMs"])
+ else {
+ result(flutterError("INVALID_ARGS", message: "Invalid frame arguments"))
return
}
-
- let asset = AVURLAsset(url: inputURL)
- let generator = AVAssetImageGenerator(asset: asset)
- generator.appliesPreferredTrackTransform = true
- generator.maximumSize = CGSize(
- width: maxWidth.doubleValue,
- height: maxHeight.doubleValue
+ startFrameExtraction(
+ args: args,
+ requestID: UUID().uuidString,
+ outputPaths: [outputPath],
+ positionsMs: [positionMs],
+ result: result
)
- if policy == "precise" {
- generator.requestedTimeToleranceBefore = .zero
- generator.requestedTimeToleranceAfter = .zero
- } else {
- let tolerance = CMTime(value: 300, timescale: 600)
- generator.requestedTimeToleranceBefore = tolerance
- generator.requestedTimeToleranceAfter = tolerance
- }
+ }
- cancelledFrameRequests.remove(requestID)
- frameGenerators[requestID] = generator
- frameOutputPaths[requestID] = outputPaths
- frameStartQueue.async { [weak self] in
- guard let self = self else { return }
- self.frameSemaphore.wait()
- DispatchQueue.main.async {
- if self.cancelledFrameRequests.contains(requestID) {
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: self.flutterError("FRAME_CANCELLED", message: "Frame extraction was cancelled")
- )
- return
- }
- self.loadVideoInfo(asset: asset) { [weak self] loadedInfo in
- guard let self = self else { return }
- DispatchQueue.main.async {
- switch loadedInfo {
- case .success(let info):
- let durationMs = info["duration"] as? Int64 ?? 0
- self.generateFrame(
- generator: generator,
- requestID: requestID,
- outputPaths: outputPaths,
- positionsMs: positionsMs,
- durationMs: durationMs,
- quality: quality.doubleValue / 100.0,
- index: 0,
- frames: [],
- videoInfo: info,
- result: result
- )
- case .failure(let error):
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: self.flutterError("FRAME_EXTRACTION_ERROR", message: "Failed to inspect video", error: error)
- )
- }
- }
- }
- }
+ private func extractTimeline(
+ _ call: FlutterMethodCall,
+ result: @escaping FlutterResult
+ ) {
+ guard let args = call.arguments as? [String: Any],
+ let requestID = args["requestId"] as? String,
+ !requestID.isEmpty,
+ let outputPaths = args["outputPaths"] as? [String],
+ outputPaths.allSatisfy({ !$0.isEmpty }),
+ let positions = args["positionsMs"] as? [NSNumber]
+ else {
+ result(flutterError("INVALID_ARGS", message: "Invalid timeline arguments"))
+ return
}
+ startFrameExtraction(
+ args: args,
+ requestID: requestID,
+ outputPaths: outputPaths,
+ positionsMs: positions.map(\.int64Value),
+ result: result
+ )
}
- private func generateFrame(
- generator: AVAssetImageGenerator,
+ private func startFrameExtraction(
+ args: [String: Any],
requestID: String,
outputPaths: [String],
positionsMs: [Int64],
- durationMs: Int64,
- quality: Double,
- index: Int,
- frames: [[String: Any]],
- videoInfo: [String: Any],
result: @escaping FlutterResult
) {
- if cancelledFrameRequests.contains(requestID) {
- removeFiles(at: outputPaths.prefix(index))
- finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: flutterError("FRAME_CANCELLED", message: "Frame extraction was cancelled")
+ guard frameTasks[requestID] == nil else {
+ result(
+ flutterError(
+ "DUPLICATE_REQUEST",
+ message: "Frame request ID is already active",
+ details: requestID
+ ))
+ return
+ }
+ let request: VideoFrameRequest
+ do {
+ request = try makeFrameRequest(
+ args: args,
+ outputPaths: outputPaths,
+ positionsMs: positionsMs
)
+ } catch {
+ result(flutterError(error, defaultCode: "INVALID_ARGS"))
return
}
- let upperBound = max(Int64(0), durationMs - 1)
- let positionMs = durationMs > 0
- ? min(max(Int64(0), positionsMs[index]), upperBound)
- : max(Int64(0), positionsMs[index])
- let requestedTime = CMTime(value: positionMs, timescale: 1000)
-
- generator.generateCGImagesAsynchronously(forTimes: [NSValue(time: requestedTime)]) {
- [weak self] _, image, _, generationResult, error in
- guard let self = self else { return }
- guard generationResult == .succeeded, let image = image else {
- DispatchQueue.main.async {
- self.removeFiles(at: outputPaths.prefix(index))
- let code = generationResult == .cancelled
- ? "FRAME_CANCELLED"
- : "FRAME_EXTRACTION_ERROR"
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: self.flutterError(code, message: "Failed to decode video frame", error: error)
- )
- }
- return
- }
-
+ let task = Task { [weak self] in
+ guard let self else { return }
+ let value: Any
do {
- try self.writeJPEG(image, to: outputPaths[index], quality: quality)
- DispatchQueue.main.async {
- if self.cancelledFrameRequests.contains(requestID) {
- self.removeFiles(at: outputPaths.prefix(index + 1))
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: self.flutterError(
- "FRAME_CANCELLED",
- message: "Frame extraction was cancelled"
- )
- )
- return
- }
- var nextFrames = frames
- nextFrames.append([
- "outputPath": outputPaths[index],
- "width": image.width,
- "height": image.height,
- ])
- let nextIndex = index + 1
- if nextIndex == positionsMs.count {
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: ["videoInfo": videoInfo, "frames": nextFrames]
- )
- } else {
- self.generateFrame(
- generator: generator,
- requestID: requestID,
- outputPaths: outputPaths,
- positionsMs: positionsMs,
- durationMs: durationMs,
- quality: quality,
- index: nextIndex,
- frames: nextFrames,
- videoInfo: videoInfo,
- result: result
- )
- }
- }
+ let extraction = try await frameExtractor.extract(request)
+ value = extraction.channelValue(outputPaths: outputPaths)
+ } catch is CancellationError {
+ value = flutterError(
+ "FRAME_CANCELLED",
+ message: "Frame extraction was cancelled",
+ details: requestID
+ )
} catch {
- DispatchQueue.main.async {
- self.removeFiles(at: outputPaths.prefix(index + 1))
- self.finishFrameRequest(
- requestID: requestID,
- generator: generator,
- result: result,
- value: self.flutterError("FRAME_WRITE_ERROR", message: "Failed to write video frame", error: error)
- )
- }
+ value = flutterError(error, defaultCode: "FRAME_EXTRACTION_ERROR")
}
+ finishFrameRequest(requestID: requestID, result: result, value: value)
}
+ frameTasks[requestID] = task
}
- private func writeJPEG(_ image: CGImage, to path: String, quality: Double) throws {
- let url = URL(fileURLWithPath: path)
- try FileManager.default.createDirectory(
- at: url.deletingLastPathComponent(),
- withIntermediateDirectories: true
- )
- try? FileManager.default.removeItem(at: url)
- guard let destination = CGImageDestinationCreateWithURL(
- url as CFURL,
- "public.jpeg" as CFString,
- 1,
- nil
- ) else {
- throw NSError(
- domain: "NativeVideoEditor",
- code: 3,
- userInfo: [NSLocalizedDescriptionKey: "Could not create JPEG destination"]
- )
+ private func cancelFrameExtraction(
+ _ call: FlutterMethodCall,
+ result: @escaping FlutterResult
+ ) {
+ guard let args = call.arguments as? [String: Any],
+ let requestID = args["requestId"] as? String
+ else {
+ result(flutterError("INVALID_ARGS", message: "Missing frame request ID"))
+ return
}
- let properties = [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary
- CGImageDestinationAddImage(destination, image, properties)
- if !CGImageDestinationFinalize(destination) {
- throw NSError(
- domain: "NativeVideoEditor",
- code: 4,
- userInfo: [NSLocalizedDescriptionKey: "Could not finalize JPEG output"]
- )
+ guard let task = frameTasks[requestID] else {
+ result(nil)
+ return
}
+ guard frameCancellationResults[requestID] == nil else {
+ result(nil)
+ return
+ }
+ frameCancellationResults[requestID] = result
+ task.cancel()
}
private func finishFrameRequest(
requestID: String,
- generator: AVAssetImageGenerator,
result: @escaping FlutterResult,
value: Any
) {
- guard let activeGenerator = frameGenerators[requestID],
- activeGenerator === generator else { return }
- frameGenerators.removeValue(forKey: requestID)
- frameOutputPaths.removeValue(forKey: requestID)
- cancelledFrameRequests.remove(requestID)
- frameSemaphore.signal()
+ guard frameTasks.removeValue(forKey: requestID) != nil else { return }
let cancellationResult = frameCancellationResults.removeValue(forKey: requestID)
- if !isDetached {
- result(value)
- cancellationResult?(nil)
- }
- }
-
- private func removeFiles<S: Sequence>(at paths: S) where S.Element == String {
- for path in paths {
- try? FileManager.default.removeItem(atPath: path)
- }
+ guard !isDetached else { return }
+ result(value)
+ cancellationResult?(nil)
}
- private func handleProcessVideo(call: FlutterMethodCall, result: @escaping FlutterResult) {
- guard currentExportSession == nil else {
+ private func processVideo(
+ _ call: FlutterMethodCall,
+ result: @escaping FlutterResult
+ ) {
+ guard exportTask == nil else {
result(flutterError("CONCURRENT_EXPORT", message: "A video export is already active"))
return
}
guard let args = call.arguments as? [String: Any],
- let inputPath = args["inputPath"] as? String,
- let outputPath = args["outputPath"] as? String else {
+ let outputPath = args["outputPath"] as? String,
+ !outputPath.isEmpty
+ else {
result(flutterError("INVALID_ARGS", message: "Invalid arguments"))
return
}
- let trimStartMs = args["trimStartMs"] as? Int
- let trimEndMs = args["trimEndMs"] as? Int
- let rotateDegrees = args["rotateDegrees"] as? Int
- let cropX = args["cropX"] as? Int
- let cropY = args["cropY"] as? Int
- let cropWidth = args["cropWidth"] as? Int
- let cropHeight = args["cropHeight"] as? Int
- let cropValues = [cropX, cropY, cropWidth, cropHeight]
- let inputURL = URL(fileURLWithPath: inputPath)
- .standardizedFileURL.resolvingSymlinksInPath()
- let outputURL = URL(fileURLWithPath: outputPath)
- .standardizedFileURL.resolvingSymlinksInPath()
- guard inputURL != outputURL,
- FileManager.default.fileExists(atPath: inputURL.path),
- (trimStartMs == nil) == (trimEndMs == nil),
- trimStartMs == nil || (trimStartMs! >= 0 && trimStartMs! < trimEndMs!),
- rotateDegrees == nil || [0, 90, 180, 270].contains(rotateDegrees!),
- cropValues.allSatisfy({ $0 == nil }) || cropValues.allSatisfy({ $0 != nil }),
- cropWidth == nil || (cropX! >= 0 && cropY! >= 0 && cropWidth! > 0 && cropHeight! > 0) else {
- result(flutterError("INVALID_ARGS", message: "Invalid video processing arguments"))
- return
- }
-
- let asset = AVAsset(url: inputURL)
- let composition = AVMutableComposition()
- var videoComposition: AVMutableVideoComposition?
-
- var requestedTimeRange = CMTimeRange(start: .zero, duration: asset.duration)
- if let trimStartMs, let trimEndMs {
- let startTime = CMTime(value: CMTimeValue(trimStartMs), timescale: 1000)
- let endTime = CMTime(value: CMTimeValue(trimEndMs), timescale: 1000)
- requestedTimeRange = CMTimeRange(start: startTime, end: endTime)
- }
-
- guard let videoTrack = asset.tracks(withMediaType: .video).first,
- let compositionVideoTrack = composition.addMutableTrack(
- withMediaType: .video,
- preferredTrackID: kCMPersistentTrackID_Invalid) else {
- result(flutterError("COMPOSITION_ERROR", message: "Failed to create composition"))
- return
- }
- guard let videoTimeRange = intersection(
- requestedTimeRange,
- with: videoTrack.timeRange
- ) else {
- result(flutterError("INVALID_TIME_RANGE", message: "The edit range does not contain video"))
- return
- }
-
+ let request: VideoEditRequest
do {
- try compositionVideoTrack.insertTimeRange(videoTimeRange, of: videoTrack, at: .zero)
-
- if let audioTrack = asset.tracks(withMediaType: .audio).first,
- let audioTimeRange = intersection(videoTimeRange, with: audioTrack.timeRange),
- let compositionAudioTrack = composition.addMutableTrack(
- withMediaType: .audio,
- preferredTrackID: kCMPersistentTrackID_Invalid) {
- let insertionTime = CMTimeSubtract(audioTimeRange.start, videoTimeRange.start)
- try compositionAudioTrack.insertTimeRange(
- audioTimeRange,
- of: audioTrack,
- at: insertionTime
- )
- }
+ request = try makeEditRequest(args: args)
} catch {
- result(flutterError("INSERT_ERROR", message: "Failed to insert track", error: error))
+ result(flutterError(error, defaultCode: "INVALID_ARGS"))
return
}
- var isReEncoded = false
-
- let naturalSize = videoTrack.naturalSize
- let preferredTransform = videoTrack.preferredTransform
- var finalTransform = preferredTransform
- var renderSize = naturalSize
-
- if let cropX, let cropY, let cropWidth, let cropHeight {
- isReEncoded = true
-
- let requestedRotation = rotateDegrees ?? 0
- let normalizedRotation = ((requestedRotation % 360) + 360) % 360
-
- let naturalBounds = CGRect(origin: .zero, size: naturalSize)
- let preferredBounds = naturalBounds.applying(preferredTransform)
- let orientationAdjustment = CGAffineTransform(
- translationX: -preferredBounds.minX,
- y: -preferredBounds.minY
- )
-
- let orientationTransform = preferredTransform.concatenating(orientationAdjustment)
-
- guard orientationTransform.isNearlyInvertible else {
- result(flutterError("PROCESS_ERROR", message: "Invalid orientation transform during crop"))
- return
- }
- let orientationInverse = orientationTransform.inverted()
-
- let cropRectDisplay = CGRect(
- x: CGFloat(cropX),
- y: CGFloat(cropY),
- width: CGFloat(cropWidth),
- height: CGFloat(cropHeight)
- )
- let displayBounds = CGRect(
- origin: .zero,
- size: CGSize(
- width: abs(preferredBounds.width),
- height: abs(preferredBounds.height)
- )
- )
- let cropTolerance: CGFloat = 0.5
- guard cropRectDisplay.minX >= displayBounds.minX - cropTolerance,
- cropRectDisplay.minY >= displayBounds.minY - cropTolerance,
- cropRectDisplay.maxX <= displayBounds.maxX + cropTolerance,
- cropRectDisplay.maxY <= displayBounds.maxY + cropTolerance else {
- result(flutterError("PROCESS_ERROR", message: "Crop rectangle exceeds the displayed video"))
- return
- }
-
- let displayCorners = [
- cropRectDisplay.origin,
- CGPoint(x: cropRectDisplay.maxX, y: cropRectDisplay.minY),
- CGPoint(x: cropRectDisplay.minX, y: cropRectDisplay.maxY),
- CGPoint(x: cropRectDisplay.maxX, y: cropRectDisplay.maxY)
- ]
-
- let fileCorners = displayCorners.map { $0.applying(orientationInverse) }
-
- let fileMinX = fileCorners.map { $0.x }.min() ?? 0
- let fileMinY = fileCorners.map { $0.y }.min() ?? 0
- let fileMaxX = fileCorners.map { $0.x }.max() ?? 0
- let fileMaxY = fileCorners.map { $0.y }.max() ?? 0
-
- let fileCropRect = CGRect(
- x: fileMinX,
- y: fileMinY,
- width: fileMaxX - fileMinX,
- height: fileMaxY - fileMinY
- )
-
- let cropTranslation = CGAffineTransform(
- translationX: -fileCropRect.origin.x,
- y: -fileCropRect.origin.y
- )
-
- var transform = cropTranslation.concatenating(orientationTransform)
-
- if normalizedRotation != 0 {
- let clockwiseRadians = CGFloat(normalizedRotation) * .pi / 180
- let rotationTransform = CGAffineTransform(rotationAngle: clockwiseRadians)
- transform = transform.concatenating(rotationTransform)
- }
-
- let transformedCorners = fileCorners.map { $0.applying(transform) }
-
- let minX = transformedCorners.map { $0.x }.min() ?? 0
- let minY = transformedCorners.map { $0.y }.min() ?? 0
- let maxX = transformedCorners.map { $0.x }.max() ?? 0
- let maxY = transformedCorners.map { $0.y }.max() ?? 0
-
- let correctionTransform = CGAffineTransform(
- translationX: -minX,
- y: -minY
- )
- transform = transform.concatenating(correctionTransform)
-
- let outputWidth = max((maxX - minX).rounded(), CGFloat(1))
- let outputHeight = max((maxY - minY).rounded(), CGFloat(1))
- renderSize = CGSize(width: outputWidth, height: outputHeight)
-
- finalTransform = transform
-
- } else if let rotateDegrees, rotateDegrees != 0 {
- isReEncoded = true
-
- // Apply orientation adjustment to handle existing metadata rotation properly
- let naturalBounds = CGRect(origin: .zero, size: naturalSize)
- let preferredBounds = naturalBounds.applying(preferredTransform)
- let orientationAdjustment = CGAffineTransform(
- translationX: -preferredBounds.minX,
- y: -preferredBounds.minY
- )
- let orientationTransform = preferredTransform.concatenating(orientationAdjustment)
-
- // Calculate oriented size for render calculations
- let orientedSize = CGSize(width: abs(preferredBounds.width), height: abs(preferredBounds.height))
-
- var transform = orientationTransform
-
- let clockwiseRadians = CGFloat(rotateDegrees) * .pi / 180
-
- // Rotate around the center of the input video
- let centerX = orientedSize.width / 2
- let centerY = orientedSize.height / 2
-
- transform = transform.translatedBy(x: centerX, y: centerY)
- transform = transform.rotated(by: clockwiseRadians)
- transform = transform.translatedBy(x: -centerX, y: -centerY)
-
- // Calculate renderSize from actual transformed bounds (more accurate for metadata-rotated videos)
- // Use naturalSize here because transform expects input in natural/file coordinate space
- let testRect = CGRect(origin: .zero, size: naturalSize)
- let transformedBounds = testRect.applying(transform)
-
- // Get the actual dimensions after the full transform chain
- let finalWidth = abs(transformedBounds.width)
- let finalHeight = abs(transformedBounds.height)
- renderSize = CGSize(width: finalWidth, height: finalHeight)
-
- // Center the video in the renderSize
- let targetMinX: CGFloat = (renderSize.width - transformedBounds.width) / 2
- let additionalTranslateX = targetMinX - transformedBounds.minX
-
- let targetMinY: CGFloat = (renderSize.height - transformedBounds.height) / 2
- let additionalTranslateY = targetMinY - transformedBounds.minY
-
- transform = transform.concatenating(CGAffineTransform(translationX: additionalTranslateX, y: additionalTranslateY))
-
- finalTransform = transform
- }
-
- if isReEncoded {
- videoComposition = AVMutableVideoComposition()
- videoComposition!.frameDuration = frameDuration(for: videoTrack)
- videoComposition!.renderSize = renderSize
-
- let instruction = AVMutableVideoCompositionInstruction()
- instruction.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
-
- let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack)
- layerInstruction.setTransform(finalTransform, at: .zero)
-
- instruction.layerInstructions = [layerInstruction]
- videoComposition!.instructions = [instruction]
- } else {
- compositionVideoTrack.preferredTransform = preferredTransform
- }
-
- // Export
- let presetName = isReEncoded ? AVAssetExportPresetHighestQuality : AVAssetExportPresetPassthrough
- guard let exportSession = AVAssetExportSession(asset: composition, presetName: presetName) else {
- result(flutterError("EXPORT_ERROR", message: "Failed to create export session"))
- return
- }
-
- currentExportSession = exportSession
-
- if let videoComp = videoComposition {
- exportSession.videoComposition = videoComp
- }
-
- exportSession.outputURL = outputURL
- exportSession.outputFileType = .mp4
- exportSession.shouldOptimizeForNetworkUse = true
- try? FileManager.default.removeItem(at: outputURL)
-
- if !isReEncoded {
- exportSession.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
- }
-
- startProgressReporting()
- exportSession.exportAsynchronously {
- DispatchQueue.main.async {
- self.stopProgressReporting()
- if self.isDetached {
- try? FileManager.default.removeItem(at: outputURL)
- } else {
- switch exportSession.status {
- case .completed:
- let outputSize = (try? outputURL.resourceValues(
- forKeys: [.fileSizeKey]
- ).fileSize) ?? 0
- if outputSize > 0 {
- result([
- "outputPath": outputPath,
- "isReEncoded": isReEncoded
- ])
- } else {
- try? FileManager.default.removeItem(at: outputURL)
- result(self.flutterError("PROCESS_ERROR", message: "Export produced no output"))
- }
- case .failed:
- try? FileManager.default.removeItem(at: outputURL)
- result(self.flutterError("PROCESS_ERROR", message: "Failed to process video", error: exportSession.error))
- case .cancelled:
- try? FileManager.default.removeItem(at: outputURL)
- result(self.flutterError("PROCESS_CANCELLED", message: "Export cancelled"))
- default:
- try? FileManager.default.removeItem(at: outputURL)
- result(self.flutterError("UNKNOWN", message: "Unknown export status", error: exportSession.error))
+ let task = Task { [weak self] in
+ guard let self else { return }
+ let value: Any
+ do {
+ let export = try await videoExporter.export(request) { [weak self] progress in
+ Task { @MainActor [weak self] in
+ guard let self, !isDetached else { return }
+ progressEventSink?(progress)
}
}
- self.currentExportSession = nil
- if !self.isDetached {
- self.exportCancellationResult?(nil)
- }
- self.exportCancellationResult = nil
+ value = [
+ "outputPath": outputPath,
+ "isReEncoded": export.isReEncoded,
+ ]
+ } catch is CancellationError {
+ value = flutterError("PROCESS_CANCELLED", message: "Export cancelled")
+ } catch {
+ value = flutterError(error, defaultCode: "PROCESS_ERROR")
}
+ finishExport(result: result, value: value)
}
+ exportTask = task
}
- private func intersection(
- _ requestedRange: CMTimeRange,
- with availableRange: CMTimeRange
- ) -> CMTimeRange? {
- let intersection = CMTimeRangeGetIntersection(
- requestedRange,
- otherRange: availableRange
- )
- guard intersection.isValid,
- !intersection.isEmpty,
- intersection.duration.isNumeric,
- CMTimeCompare(intersection.duration, .zero) > 0 else {
- return nil
- }
- return intersection
- }
-
- private func frameDuration(for track: AVAssetTrack) -> CMTime {
- let nominal = track.nominalFrameRate
- if nominal.isFinite && nominal > 0 {
- return CMTime(value: 1, timescale: Int32(round(nominal)))
- }
- if track.minFrameDuration.isValid && track.minFrameDuration.value != 0 {
- return track.minFrameDuration
+ private func cancelProcessing(result: @escaping FlutterResult) {
+ guard let exportTask else {
+ result(nil)
+ return
}
- return CMTime(value: 1, timescale: 30)
- }
-
- private func flutterError(_ code: String, message: String, error: Error? = nil, details: Any? = nil) -> FlutterError {
- let resolvedMessage = error?.localizedDescription ?? message
- let resolvedDetails: Any?
- if let details = details {
- resolvedDetails = details
- } else if let error = error {
- resolvedDetails = String(describing: error)
- } else {
- resolvedDetails = nil
+ guard exportCancellationResult == nil else {
+ result(nil)
+ return
}
- return FlutterError(code: code, message: resolvedMessage, details: resolvedDetails)
+ exportCancellationResult = result
+ exportTask.cancel()
}
- private func getVideoInfo(videoPath: String, result: @escaping FlutterResult) {
- let asset = AVURLAsset(url: URL(fileURLWithPath: videoPath))
- loadVideoInfo(asset: asset) { [weak self] loadedInfo in
- guard let self = self else { return }
- DispatchQueue.main.async {
- guard !self.isDetached else { return }
- switch loadedInfo {
- case .success(let info):
- result(info)
- case .failure(let error):
- result(self.flutterError(
- "INFO_ERROR",
- message: "Failed to load video metadata",
- error: error
- ))
- }
- }
- }
+ private func finishExport(result: @escaping FlutterResult, value: Any) {
+ guard exportTask != nil else { return }
+ exportTask = nil
+ let cancellationResult = exportCancellationResult
+ exportCancellationResult = nil
+ guard !isDetached else { return }
+ result(value)
+ cancellationResult?(nil)
}
- private func loadVideoInfo(
- asset: AVAsset,
- completion: @escaping (Result<[String: Any], Error>) -> Void
- ) {
- asset.loadValuesAsynchronously(forKeys: ["duration", "tracks"]) { [weak self] in
- guard let self = self else { return }
- var error: NSError?
- guard asset.statusOfValue(forKey: "duration", error: &error) == .loaded,
- asset.statusOfValue(forKey: "tracks", error: &error) == .loaded else {
- completion(.failure(error ?? NSError(
- domain: "NativeVideoEditor",
- code: 1,
- userInfo: [NSLocalizedDescriptionKey: "Video metadata is unavailable"]
- )))
- return
- }
- guard let videoTrack = asset.tracks.first(where: { $0.mediaType == .video }) else {
- completion(.failure(NSError(
- domain: "NativeVideoEditor",
- code: 2,
- userInfo: [NSLocalizedDescriptionKey: "Video track is unavailable"]
- )))
- return
- }
-
- let trackKeys = [
- "naturalSize",
- "preferredTransform",
- "nominalFrameRate",
- "estimatedDataRate",
- ]
- videoTrack.loadValuesAsynchronously(forKeys: trackKeys) {
- for key in ["naturalSize", "preferredTransform"] {
- var trackError: NSError?
- guard videoTrack.statusOfValue(forKey: key, error: &trackError) == .loaded else {
- completion(.failure(trackError ?? NSError(
- domain: "NativeVideoEditor",
- code: 3,
- userInfo: [NSLocalizedDescriptionKey: "Video track metadata is unavailable: \(key)"]
- )))
- return
- }
- }
- completion(.success(self.loadedVideoInfo(asset: asset)))
- }
- }
+ private func makeFrameRequest(
+ args: [String: Any],
+ outputPaths: [String],
+ positionsMs: [Int64]
+ ) throws -> VideoFrameRequest {
+ guard let inputPath = args["inputPath"] as? String,
+ !inputPath.isEmpty,
+ outputPaths.allSatisfy({ !$0.isEmpty }),
+ let maxWidth = int(args["maxWidth"]),
+ let maxHeight = int(args["maxHeight"]),
+ let quality = int(args["quality"]),
+ let policyValue = args["policy"] as? String
+ else {
+ throw VideoEditorError.invalidRequest("Invalid frame extraction arguments")
+ }
+ let policy: VideoFramePolicy
+ switch policyValue {
+ case "precise": policy = .precise
+ case "nearestSync": policy = .nearestSync
+ default:
+ throw VideoEditorError.invalidRequest("Invalid frame policy")
+ }
+ return try VideoFrameRequest(
+ inputURL: URL(fileURLWithPath: inputPath),
+ outputURLs: outputPaths.map { URL(fileURLWithPath: $0) },
+ positionsMs: positionsMs,
+ maxWidth: maxWidth,
+ maxHeight: maxHeight,
+ quality: quality,
+ policy: policy
+ )
}
- private func loadedVideoInfo(asset: AVAsset) -> [String: Any] {
- let durationSeconds = CMTimeGetSeconds(asset.duration)
- var info: [String: Any] = [
- "duration": durationSeconds.isFinite
- ? Int64(max(0, durationSeconds * 1000))
- : Int64(0),
- "width": 0,
- "height": 0,
- "displayWidth": 0,
- "displayHeight": 0,
- "rotation": 0,
+ private func makeEditRequest(args: [String: Any]) throws -> VideoEditRequest {
+ guard let inputPath = args["inputPath"] as? String,
+ !inputPath.isEmpty,
+ let outputPath = args["outputPath"] as? String,
+ !outputPath.isEmpty
+ else {
+ throw VideoEditorError.invalidRequest("Invalid video paths")
+ }
+ let cropValues = [
+ int(args["cropX"]),
+ int(args["cropY"]),
+ int(args["cropWidth"]),
+ int(args["cropHeight"]),
]
+ guard cropValues.allSatisfy({ $0 == nil }) || cropValues.allSatisfy({ $0 != nil }) else {
+ throw VideoEditorError.invalidRequest("Crop values must be provided together")
+ }
+ let crop =
+ cropValues[0] == nil
+ ? nil
+ : try VideoCrop(
+ x: cropValues[0]!,
+ y: cropValues[1]!,
+ width: cropValues[2]!,
+ height: cropValues[3]!
+ )
+ return try VideoEditRequest(
+ inputURL: URL(fileURLWithPath: inputPath),
+ outputURL: URL(fileURLWithPath: outputPath),
+ trimStartMs: int64(args["trimStartMs"]),
+ trimEndMs: int64(args["trimEndMs"]),
+ rotateDegrees: int(args["rotateDegrees"]),
+ crop: crop
+ )
+ }
- if let videoTrack = asset.tracks.first(where: { $0.mediaType == .video }) {
- let naturalSize = videoTrack.naturalSize
- let transform = videoTrack.preferredTransform
- let transformedBounds = CGRect(origin: .zero, size: naturalSize).applying(transform)
- let angle = atan2(transform.b, transform.a) * 180 / .pi
- let roundedAngle = Int(angle.rounded())
- let rotation = ((roundedAngle % 360) + 360) % 360
-
- info["width"] = Int(abs(naturalSize.width).rounded())
- info["height"] = Int(abs(naturalSize.height).rounded())
- info["displayWidth"] = Int(abs(transformedBounds.width).rounded())
- info["displayHeight"] = Int(abs(transformedBounds.height).rounded())
- info["rotation"] = rotation
- if videoTrack.nominalFrameRate > 0 {
- info["frameRate"] = Double(videoTrack.nominalFrameRate)
- }
- if videoTrack.estimatedDataRate > 0 {
- info["bitrate"] = Int64(videoTrack.estimatedDataRate)
- }
- }
- return info
+ private func int(_ value: Any?) -> Int? {
+ (value as? NSNumber)?.intValue
}
- private func startProgressReporting() {
- progressTimer?.invalidate()
- progressTimer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in
- guard let self = self,
- let session = self.currentExportSession,
- let sink = self.progressEventSink else { return }
+ private func int64(_ value: Any?) -> Int64? {
+ (value as? NSNumber)?.int64Value
+ }
- let progress = Double(session.progress)
- sink(progress)
+ private func flutterError(
+ _ error: Error,
+ defaultCode: String
+ ) -> FlutterError {
+ guard let videoError = error as? VideoEditorError else {
+ return flutterError(
+ defaultCode,
+ message: error.localizedDescription,
+ details: String(describing: error)
+ )
}
+ let code: String
+ switch videoError {
+ case .invalidRequest: code = "INVALID_ARGS"
+ case .metadataUnavailable: code = defaultCode
+ case .frameExtraction: code = "FRAME_EXTRACTION_ERROR"
+ case .frameWrite: code = "FRAME_WRITE_ERROR"
+ case .invalidTimeRange: code = "INVALID_TIME_RANGE"
+ case .composition: code = "COMPOSITION_ERROR"
+ case .export: code = defaultCode
+ }
+ return flutterError(
+ code,
+ message: videoError.message,
+ details: videoError.underlyingError.map(String.init(describing:))
+ )
}
- private func stopProgressReporting() {
- progressTimer?.invalidate()
- progressTimer = nil
+ private func flutterError(
+ _ code: String,
+ message: String,
+ details: Any? = nil
+ ) -> FlutterError {
+ FlutterError(code: code, message: message, details: details)
}
public func detachFromEngine(for registrar: FlutterPluginRegistrar) {
guard !isDetached else { return }
isDetached = true
progressEventSink = nil
- stopProgressReporting()
-
- currentExportSession?.cancelExport()
+ frameTasks.values.forEach { $0.cancel() }
+ frameTasks.removeAll()
+ frameCancellationResults.removeAll()
+ exportTask?.cancel()
+ exportTask = nil
exportCancellationResult = nil
+ }
+}
- cancelledFrameRequests.formUnion(frameGenerators.keys)
- for outputPaths in frameOutputPaths.values {
- removeFiles(at: outputPaths)
- }
- frameCancellationResults.removeAll()
- for generator in frameGenerators.values {
- generator.cancelAllCGImageGeneration()
- }
+private extension VideoMetadata {
+ var channelValue: [String: Any] {
+ var value: [String: Any] = [
+ "duration": durationMs,
+ "width": width,
+ "height": height,
+ "displayWidth": displayWidth,
+ "displayHeight": displayHeight,
+ "rotation": rotationDegrees,
+ ]
+ if let bitrate { value["bitrate"] = bitrate }
+ if let frameRate { value["frameRate"] = frameRate }
+ return value
}
}
-private extension CGAffineTransform {
- var isNearlyInvertible: Bool {
- let determinant = (a * d) - (b * c)
- return abs(determinant) > 1e-8
+private extension VideoFrameExtractionResult {
+ func channelValue(outputPaths: [String]) -> [String: Any] {
+ [
+ "videoInfo": videoInfo.channelValue,
+ "frames": zip(frames, outputPaths).map { frame, outputPath in
+ [
+ "outputPath": outputPath,
+ "width": frame.width,
+ "height": frame.height,
+ ] as [String: Any]
+ },
+ ]
}
}
-extension NativeVideoEditorPlugin: FlutterStreamHandler {
- public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
- if !isDetached {
- progressEventSink = events
- }
+extension NativeVideoEditorPlugin: @preconcurrency FlutterStreamHandler {
+ public func onListen(
+ withArguments arguments: Any?,
+ eventSink events: @escaping FlutterEventSink
+ ) -> FlutterError? {
+ if !isDetached { progressEventSink = events }
return nil
}
public func onCancel(withArguments arguments: Any?) -> FlutterError? {
progressEventSink = nil
- stopProgressReporting()
return nil
}
}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift b/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift
new file mode 100644
index 0000000000..7fcf05415a
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoExportEngine.swift
@@ -0,0 +1,230 @@
+import AVFoundation
+
+public final class VideoExportEngine: Sendable {
+ public init() {}
+
+ public func export(
+ _ request: VideoEditRequest,
+ onProgress: @escaping @Sendable (Double) -> Void
+ ) async throws -> VideoEditResult {
+ try? FileManager.default.removeItem(at: request.outputURL)
+ do {
+ let prepared = try await prepareExport(request)
+ let session = prepared.session
+ let sessionBox = ExportSessionBox(session)
+ let progressTask = Task {
+ while !Task.isCancelled {
+ onProgress(Double(sessionBox.value.progress))
+ try? await Task.sleep(nanoseconds: 100_000_000)
+ }
+ }
+ defer { progressTask.cancel() }
+
+ try await run(session)
+ onProgress(1)
+ let outputSize =
+ (try? request.outputURL.resourceValues(
+ forKeys: [.fileSizeKey]
+ ).fileSize) ?? 0
+ guard outputSize > 0 else {
+ throw VideoEditorError.export("Export produced no output")
+ }
+ return VideoEditResult(
+ outputURL: request.outputURL,
+ isReEncoded: prepared.isReEncoded
+ )
+ } catch is CancellationError {
+ try? FileManager.default.removeItem(at: request.outputURL)
+ throw CancellationError()
+ } catch let error as VideoEditorError {
+ try? FileManager.default.removeItem(at: request.outputURL)
+ throw error
+ } catch {
+ try? FileManager.default.removeItem(at: request.outputURL)
+ throw VideoEditorError.export("Failed to process video", error)
+ }
+ }
+
+ private func prepareExport(
+ _ request: VideoEditRequest
+ ) async throws -> (session: AVAssetExportSession, isReEncoded: Bool) {
+ let asset = AVURLAsset(url: request.inputURL)
+ let duration = try await asset.load(.duration)
+ let videoTracks = try await asset.loadTracks(withMediaType: .video)
+ guard let videoTrack = videoTracks.first else {
+ throw VideoEditorError.composition("Video track is unavailable")
+ }
+
+ let videoTrackRange = try await videoTrack.load(.timeRange)
+ let naturalSize = try await videoTrack.load(.naturalSize)
+ let preferredTransform = try await videoTrack.load(.preferredTransform)
+ let composition = AVMutableComposition()
+ guard
+ let compositionVideoTrack = composition.addMutableTrack(
+ withMediaType: .video,
+ preferredTrackID: kCMPersistentTrackID_Invalid
+ )
+ else {
+ throw VideoEditorError.composition("Failed to create video track")
+ }
+
+ let requestedRange: CMTimeRange
+ if let trimStartMs = request.trimStartMs, let trimEndMs = request.trimEndMs {
+ requestedRange = CMTimeRange(
+ start: CMTime(value: trimStartMs, timescale: 1000),
+ end: CMTime(value: trimEndMs, timescale: 1000)
+ )
+ } else {
+ requestedRange = CMTimeRange(start: .zero, duration: duration)
+ }
+ guard let videoTimeRange = validIntersection(requestedRange, videoTrackRange) else {
+ throw VideoEditorError.invalidTimeRange(
+ "The edit range does not contain video"
+ )
+ }
+
+ do {
+ try compositionVideoTrack.insertTimeRange(videoTimeRange, of: videoTrack, at: .zero)
+ let audioTracks = try await asset.loadTracks(withMediaType: .audio)
+ if let audioTrack = audioTracks.first {
+ let audioTrackRange = try await audioTrack.load(.timeRange)
+ if let audioTimeRange = validIntersection(videoTimeRange, audioTrackRange),
+ let compositionAudioTrack = composition.addMutableTrack(
+ withMediaType: .audio,
+ preferredTrackID: kCMPersistentTrackID_Invalid
+ )
+ {
+ try compositionAudioTrack.insertTimeRange(
+ audioTimeRange,
+ of: audioTrack,
+ at: CMTimeSubtract(audioTimeRange.start, videoTimeRange.start)
+ )
+ }
+ }
+ } catch is CancellationError {
+ throw CancellationError()
+ } catch {
+ throw VideoEditorError.composition("Failed to insert media track", error)
+ }
+
+ let isReEncoded = request.crop != nil || (request.rotateDegrees ?? 0) != 0
+ var videoComposition: AVMutableVideoComposition?
+ if isReEncoded {
+ let transformPlan = try VideoTransformPlanner.makePlan(
+ naturalSize: naturalSize,
+ preferredTransform: preferredTransform,
+ crop: request.crop,
+ rotateDegrees: request.rotateDegrees
+ )
+ let nominalFrameRate = try await videoTrack.load(.nominalFrameRate)
+ let minimumFrameDuration = try await videoTrack.load(.minFrameDuration)
+ let mutableVideoComposition = AVMutableVideoComposition()
+ mutableVideoComposition.frameDuration = frameDuration(
+ nominalFrameRate: nominalFrameRate,
+ minimumFrameDuration: minimumFrameDuration
+ )
+ mutableVideoComposition.renderSize = transformPlan.renderSize
+
+ let instruction = AVMutableVideoCompositionInstruction()
+ instruction.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
+ let layerInstruction = AVMutableVideoCompositionLayerInstruction(
+ assetTrack: compositionVideoTrack
+ )
+ layerInstruction.setTransform(transformPlan.transform, at: .zero)
+ instruction.layerInstructions = [layerInstruction]
+ mutableVideoComposition.instructions = [instruction]
+ videoComposition = mutableVideoComposition
+ } else {
+ compositionVideoTrack.preferredTransform = preferredTransform
+ }
+
+ let preset =
+ isReEncoded
+ ? AVAssetExportPresetHighestQuality
+ : AVAssetExportPresetPassthrough
+ guard let session = AVAssetExportSession(asset: composition, presetName: preset) else {
+ throw VideoEditorError.export("Failed to create export session")
+ }
+ session.videoComposition = videoComposition
+ session.outputURL = request.outputURL
+ session.outputFileType = .mp4
+ session.shouldOptimizeForNetworkUse = true
+ if !isReEncoded {
+ session.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
+ }
+ return (session, isReEncoded)
+ }
+
+ private func run(_ session: AVAssetExportSession) async throws {
+ let sessionBox = ExportSessionBox(session)
+ try await withTaskCancellationHandler(
+ operation: {
+ try await withCheckedThrowingContinuation { continuation in
+ sessionBox.value.exportAsynchronously {
+ switch sessionBox.value.status {
+ case .completed:
+ continuation.resume()
+ case .cancelled:
+ continuation.resume(throwing: CancellationError())
+ case .failed:
+ continuation.resume(
+ throwing: VideoEditorError.export(
+ "Failed to process video",
+ sessionBox.value.error
+ )
+ )
+ default:
+ continuation.resume(
+ throwing: VideoEditorError.export(
+ "Unknown export status",
+ sessionBox.value.error
+ )
+ )
+ }
+ }
+ }
+ },
+ onCancel: {
+ sessionBox.value.cancelExport()
+ })
+ }
+
+ private func validIntersection(
+ _ requestedRange: CMTimeRange,
+ _ availableRange: CMTimeRange
+ ) -> CMTimeRange? {
+ let intersection = CMTimeRangeGetIntersection(
+ requestedRange,
+ otherRange: availableRange
+ )
+ guard intersection.isValid,
+ !intersection.isEmpty,
+ intersection.duration.isNumeric,
+ CMTimeCompare(intersection.duration, .zero) > 0
+ else {
+ return nil
+ }
+ return intersection
+ }
+
+ private func frameDuration(
+ nominalFrameRate: Float,
+ minimumFrameDuration: CMTime
+ ) -> CMTime {
+ if nominalFrameRate.isFinite && nominalFrameRate > 0 {
+ return CMTime(value: 1, timescale: Int32(round(nominalFrameRate)))
+ }
+ if minimumFrameDuration.isValid && minimumFrameDuration.value != 0 {
+ return minimumFrameDuration
+ }
+ return CMTime(value: 1, timescale: 30)
+ }
+}
+
+private final class ExportSessionBox: @unchecked Sendable {
+ let value: AVAssetExportSession
+
+ init(_ value: AVAssetExportSession) {
+ self.value = value
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoFrameExtractor.swift b/mobile/packages/native_video_editor/ios/Classes/VideoFrameExtractor.swift
new file mode 100644
index 0000000000..c6bd0b6ab6
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoFrameExtractor.swift
@@ -0,0 +1,204 @@
+import AVFoundation
+import ImageIO
+
+public final class VideoFrameExtractor: @unchecked Sendable {
+ private let metadataReader: VideoMetadataReader
+ private let permits = AsyncPermitPool(limit: 2)
+
+ public init(metadataReader: VideoMetadataReader = VideoMetadataReader()) {
+ self.metadataReader = metadataReader
+ }
+
+ public func extract(_ request: VideoFrameRequest) async throws -> VideoFrameExtractionResult {
+ try await permits.acquire()
+ do {
+ let result = try await extractWithPermit(request)
+ await permits.release()
+ return result
+ } catch {
+ request.outputURLs.forEach { try? FileManager.default.removeItem(at: $0) }
+ await permits.release()
+ throw error
+ }
+ }
+
+ private func extractWithPermit(
+ _ request: VideoFrameRequest
+ ) async throws -> VideoFrameExtractionResult {
+ try Task.checkCancellation()
+ let asset = AVURLAsset(url: request.inputURL)
+ let videoInfo = try await metadataReader.read(from: asset)
+
+ let generator = AVAssetImageGenerator(asset: asset)
+ generator.appliesPreferredTrackTransform = true
+ generator.maximumSize = CGSize(width: request.maxWidth, height: request.maxHeight)
+ if request.policy == .precise {
+ generator.requestedTimeToleranceBefore = .zero
+ generator.requestedTimeToleranceAfter = .zero
+ } else {
+ let tolerance = CMTime(value: 300, timescale: 600)
+ generator.requestedTimeToleranceBefore = tolerance
+ generator.requestedTimeToleranceAfter = tolerance
+ }
+
+ var frames: [VideoFrameResult] = []
+ frames.reserveCapacity(request.positionsMs.count)
+ for (index, requestedPositionMs) in request.positionsMs.enumerated() {
+ try Task.checkCancellation()
+ let positionMs = clampedPosition(
+ requestedPositionMs,
+ durationMs: videoInfo.durationMs
+ )
+ let image = try await generateImage(
+ generator: generator,
+ at: CMTime(value: positionMs, timescale: 1000)
+ ).value
+ let outputURL = request.outputURLs[index]
+ try writeJPEG(
+ image,
+ to: outputURL,
+ quality: Double(request.quality) / 100
+ )
+ try Task.checkCancellation()
+ frames.append(
+ VideoFrameResult(
+ outputURL: outputURL,
+ width: image.width,
+ height: image.height
+ )
+ )
+ }
+ return VideoFrameExtractionResult(videoInfo: videoInfo, frames: frames)
+ }
+
+ private func clampedPosition(_ positionMs: Int64, durationMs: Int64) -> Int64 {
+ guard durationMs > 0 else { return max(0, positionMs) }
+ return min(max(0, positionMs), max(0, durationMs - 1))
+ }
+
+ private func generateImage(
+ generator: AVAssetImageGenerator,
+ at time: CMTime
+ ) async throws -> ImageBox {
+ let generatorBox = ImageGeneratorBox(generator)
+ return try await withTaskCancellationHandler {
+ try await withCheckedThrowingContinuation {
+ (continuation: CheckedContinuation<ImageBox, Error>) in
+ generatorBox.value.generateCGImagesAsynchronously(forTimes: [NSValue(time: time)]) {
+ _, image, _, result, error in
+ if result == .cancelled {
+ continuation.resume(throwing: CancellationError())
+ } else if result == .succeeded, let image {
+ continuation.resume(returning: ImageBox(image))
+ } else {
+ continuation.resume(
+ throwing: VideoEditorError.frameExtraction(
+ "Failed to decode video frame",
+ error
+ )
+ )
+ }
+ }
+ }
+ } onCancel: {
+ generatorBox.value.cancelAllCGImageGeneration()
+ }
+ }
+
+ private func writeJPEG(_ image: CGImage, to url: URL, quality: Double) throws {
+ do {
+ try FileManager.default.createDirectory(
+ at: url.deletingLastPathComponent(),
+ withIntermediateDirectories: true
+ )
+ try? FileManager.default.removeItem(at: url)
+ guard
+ let destination = CGImageDestinationCreateWithURL(
+ url as CFURL,
+ "public.jpeg" as CFString,
+ 1,
+ nil
+ )
+ else {
+ throw VideoEditorError.frameWrite("Could not create JPEG destination")
+ }
+ let properties =
+ [
+ kCGImageDestinationLossyCompressionQuality: quality
+ ] as CFDictionary
+ CGImageDestinationAddImage(destination, image, properties)
+ guard CGImageDestinationFinalize(destination) else {
+ throw VideoEditorError.frameWrite("Could not finalize JPEG output")
+ }
+ } catch let error as VideoEditorError {
+ throw error
+ } catch {
+ throw VideoEditorError.frameWrite("Failed to write video frame", error)
+ }
+ }
+}
+
+private final class ImageGeneratorBox: @unchecked Sendable {
+ let value: AVAssetImageGenerator
+
+ init(_ value: AVAssetImageGenerator) {
+ self.value = value
+ }
+}
+
+private final class ImageBox: @unchecked Sendable {
+ let value: CGImage
+
+ init(_ value: CGImage) {
+ self.value = value
+ }
+}
+
+private actor AsyncPermitPool {
+ private struct Waiter {
+ let id: Int
+ let continuation: CheckedContinuation<Void, Error>
+ }
+
+ private var available: Int
+ private var nextWaiterID = 0
+ private var waiters: [Waiter] = []
+
+ init(limit: Int) {
+ available = limit
+ }
+
+ func acquire() async throws {
+ try Task.checkCancellation()
+ let waiterID = nextWaiterID
+ nextWaiterID += 1
+ try await withTaskCancellationHandler {
+ try await withCheckedThrowingContinuation {
+ (continuation: CheckedContinuation<Void, Error>) in
+ if Task.isCancelled {
+ continuation.resume(throwing: CancellationError())
+ } else if available > 0 {
+ available -= 1
+ continuation.resume()
+ } else {
+ waiters.append(Waiter(id: waiterID, continuation: continuation))
+ }
+ }
+ } onCancel: {
+ Task { await self.cancel(waiterID) }
+ }
+ }
+
+ func release() {
+ if waiters.isEmpty {
+ available += 1
+ } else {
+ waiters.removeFirst().continuation.resume()
+ }
+ }
+
+ private func cancel(_ waiterID: Int) {
+ guard let index = waiters.firstIndex(where: { $0.id == waiterID }) else { return }
+ waiters.remove(at: index).continuation.resume(throwing: CancellationError())
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoMetadataReader.swift b/mobile/packages/native_video_editor/ios/Classes/VideoMetadataReader.swift
new file mode 100644
index 0000000000..26843a79d3
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoMetadataReader.swift
@@ -0,0 +1,79 @@
+import AVFoundation
+
+public struct VideoMetadataReader: Sendable {
+ public init() {}
+
+ public func read(from url: URL) async throws -> VideoMetadata {
+ try await read(from: AVURLAsset(url: url))
+ }
+
+ func read(from asset: AVAsset) async throws -> VideoMetadata {
+ do {
+ let duration = try await asset.load(.duration)
+ let videoTracks = try await asset.loadTracks(withMediaType: .video)
+ guard let videoTrack = videoTracks.first else {
+ throw VideoEditorError.metadataUnavailable("Video track is unavailable")
+ }
+
+ let naturalSize = try await videoTrack.load(.naturalSize)
+ let transform = try await videoTrack.load(.preferredTransform)
+ let frameRate = try await videoTrack.load(.nominalFrameRate)
+ let dataRate = try await videoTrack.load(.estimatedDataRate)
+ return try makeMetadata(
+ duration: duration,
+ naturalSize: naturalSize,
+ transform: transform,
+ frameRate: frameRate,
+ dataRate: dataRate
+ )
+ } catch is CancellationError {
+ throw CancellationError()
+ } catch let error as VideoEditorError {
+ throw error
+ } catch {
+ throw VideoEditorError.metadataUnavailable(
+ "Failed to load video metadata",
+ error
+ )
+ }
+ }
+
+ private func makeMetadata(
+ duration: CMTime,
+ naturalSize: CGSize,
+ transform: CGAffineTransform,
+ frameRate: Float,
+ dataRate: Float
+ ) throws -> VideoMetadata {
+ let durationSeconds = CMTimeGetSeconds(duration)
+ let transformedBounds = CGRect(origin: .zero, size: naturalSize).applying(transform)
+ let angle = atan2(transform.b, transform.a) * 180 / .pi
+ let roundedAngle = Int(angle.rounded())
+ let rotation = ((roundedAngle % 360) + 360) % 360
+ let width = Int(abs(naturalSize.width).rounded())
+ let height = Int(abs(naturalSize.height).rounded())
+ let displayWidth = Int(abs(transformedBounds.width).rounded())
+ let displayHeight = Int(abs(transformedBounds.height).rounded())
+ guard width > 0,
+ height > 0,
+ displayWidth > 0,
+ displayHeight > 0,
+ [0, 90, 180, 270].contains(rotation)
+ else {
+ throw VideoEditorError.metadataUnavailable("Invalid video dimensions or rotation")
+ }
+
+ return VideoMetadata(
+ durationMs: durationSeconds.isFinite
+ ? Int64(max(0, durationSeconds * 1000))
+ : 0,
+ width: width,
+ height: height,
+ displayWidth: displayWidth,
+ displayHeight: displayHeight,
+ rotationDegrees: rotation,
+ bitrate: dataRate > 0 ? Int64(dataRate) : nil,
+ frameRate: frameRate > 0 ? Double(frameRate) : nil
+ )
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift b/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift
new file mode 100644
index 0000000000..4b33b267da
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoModels.swift
@@ -0,0 +1,187 @@
+import AVFoundation
+
+public struct VideoMetadata: Equatable, Sendable {
+ public let durationMs: Int64
+ public let width: Int
+ public let height: Int
+ public let displayWidth: Int
+ public let displayHeight: Int
+ public let rotationDegrees: Int
+ public let bitrate: Int64?
+ public let frameRate: Double?
+
+ public init(
+ durationMs: Int64,
+ width: Int,
+ height: Int,
+ displayWidth: Int,
+ displayHeight: Int,
+ rotationDegrees: Int,
+ bitrate: Int64?,
+ frameRate: Double?
+ ) {
+ self.durationMs = durationMs
+ self.width = width
+ self.height = height
+ self.displayWidth = displayWidth
+ self.displayHeight = displayHeight
+ self.rotationDegrees = rotationDegrees
+ self.bitrate = bitrate
+ self.frameRate = frameRate
+ }
+}
+
+public enum VideoFramePolicy: Sendable {
+ case precise
+ case nearestSync
+}
+
+public struct VideoFrameRequest: Sendable {
+ public let inputURL: URL
+ public let outputURLs: [URL]
+ public let positionsMs: [Int64]
+ public let maxWidth: Int
+ public let maxHeight: Int
+ public let quality: Int
+ public let policy: VideoFramePolicy
+
+ public init(
+ inputURL: URL,
+ outputURLs: [URL],
+ positionsMs: [Int64],
+ maxWidth: Int,
+ maxHeight: Int,
+ quality: Int,
+ policy: VideoFramePolicy
+ ) throws {
+ let inputURL = inputURL.standardizedFileURL.resolvingSymlinksInPath()
+ let outputURLs = outputURLs.map {
+ $0.standardizedFileURL.resolvingSymlinksInPath()
+ }
+ guard FileManager.default.fileExists(atPath: inputURL.path),
+ !positionsMs.isEmpty,
+ positionsMs.count == outputURLs.count,
+ positionsMs.allSatisfy({ $0 >= 0 }),
+ !outputURLs.contains(inputURL),
+ Set(outputURLs.map(\.path)).count == outputURLs.count,
+ maxWidth > 0,
+ maxHeight > 0,
+ (1...100).contains(quality)
+ else {
+ throw VideoEditorError.invalidRequest("Invalid frame extraction request")
+ }
+ self.inputURL = inputURL
+ self.outputURLs = outputURLs
+ self.positionsMs = positionsMs
+ self.maxWidth = maxWidth
+ self.maxHeight = maxHeight
+ self.quality = quality
+ self.policy = policy
+ }
+}
+
+public struct VideoFrameResult: Equatable, Sendable {
+ public let outputURL: URL
+ public let width: Int
+ public let height: Int
+}
+
+public struct VideoFrameExtractionResult: Equatable, Sendable {
+ public let videoInfo: VideoMetadata
+ public let frames: [VideoFrameResult]
+}
+
+public struct VideoCrop: Equatable, Sendable {
+ public let x: Int
+ public let y: Int
+ public let width: Int
+ public let height: Int
+
+ public init(x: Int, y: Int, width: Int, height: Int) throws {
+ guard x >= 0, y >= 0, width > 0, height > 0 else {
+ throw VideoEditorError.invalidRequest("Invalid crop rectangle")
+ }
+ self.x = x
+ self.y = y
+ self.width = width
+ self.height = height
+ }
+}
+
+public struct VideoEditRequest: Sendable {
+ public let inputURL: URL
+ public let outputURL: URL
+ public let trimStartMs: Int64?
+ public let trimEndMs: Int64?
+ public let rotateDegrees: Int?
+ public let crop: VideoCrop?
+
+ public init(
+ inputURL: URL,
+ outputURL: URL,
+ trimStartMs: Int64?,
+ trimEndMs: Int64?,
+ rotateDegrees: Int?,
+ crop: VideoCrop?
+ ) throws {
+ let inputURL = inputURL.standardizedFileURL.resolvingSymlinksInPath()
+ let outputURL = outputURL.standardizedFileURL.resolvingSymlinksInPath()
+ guard inputURL != outputURL,
+ FileManager.default.fileExists(atPath: inputURL.path),
+ (trimStartMs == nil) == (trimEndMs == nil),
+ trimStartMs == nil || (trimStartMs! >= 0 && trimStartMs! < trimEndMs!),
+ rotateDegrees == nil || [0, 90, 180, 270].contains(rotateDegrees!)
+ else {
+ throw VideoEditorError.invalidRequest("Invalid video processing request")
+ }
+ self.inputURL = inputURL
+ self.outputURL = outputURL
+ self.trimStartMs = trimStartMs
+ self.trimEndMs = trimEndMs
+ self.rotateDegrees = rotateDegrees
+ self.crop = crop
+ }
+}
+
+public struct VideoEditResult: Equatable, Sendable {
+ public let outputURL: URL
+ public let isReEncoded: Bool
+}
+
+public enum VideoEditorError: LocalizedError {
+ case invalidRequest(String)
+ case metadataUnavailable(String, Error? = nil)
+ case frameExtraction(String, Error? = nil)
+ case frameWrite(String, Error? = nil)
+ case invalidTimeRange(String)
+ case composition(String, Error? = nil)
+ case export(String, Error? = nil)
+
+ public var message: String {
+ switch self {
+ case .invalidRequest(let message),
+ .metadataUnavailable(let message, _),
+ .frameExtraction(let message, _),
+ .frameWrite(let message, _),
+ .invalidTimeRange(let message),
+ .composition(let message, _),
+ .export(let message, _):
+ return message
+ }
+ }
+
+ public var errorDescription: String? { message }
+
+ public var underlyingError: Error? {
+ switch self {
+ case .metadataUnavailable(_, let error),
+ .frameExtraction(_, let error),
+ .frameWrite(_, let error),
+ .composition(_, let error),
+ .export(_, let error):
+ return error
+ case .invalidRequest, .invalidTimeRange:
+ return nil
+ }
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift b/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift
new file mode 100644
index 0000000000..f97cd4da8a
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Classes/VideoTransformPlanner.swift
@@ -0,0 +1,167 @@
+import CoreGraphics
+
+public struct VideoTransformPlan: Sendable {
+ public let transform: CGAffineTransform
+ public let renderSize: CGSize
+}
+
+public enum VideoTransformPlanner {
+ public static func makePlan(
+ naturalSize: CGSize,
+ preferredTransform: CGAffineTransform,
+ crop: VideoCrop?,
+ rotateDegrees: Int?
+ ) throws -> VideoTransformPlan {
+ if let crop {
+ return try cropPlan(
+ naturalSize: naturalSize,
+ preferredTransform: preferredTransform,
+ crop: crop,
+ rotateDegrees: rotateDegrees ?? 0
+ )
+ }
+ if let rotateDegrees, rotateDegrees != 0 {
+ return rotationPlan(
+ naturalSize: naturalSize,
+ preferredTransform: preferredTransform,
+ rotateDegrees: rotateDegrees
+ )
+ }
+ return VideoTransformPlan(transform: preferredTransform, renderSize: naturalSize)
+ }
+
+ private static func cropPlan(
+ naturalSize: CGSize,
+ preferredTransform: CGAffineTransform,
+ crop: VideoCrop,
+ rotateDegrees: Int
+ ) throws -> VideoTransformPlan {
+ let normalizedRotation = ((rotateDegrees % 360) + 360) % 360
+ let naturalBounds = CGRect(origin: .zero, size: naturalSize)
+ let preferredBounds = naturalBounds.applying(preferredTransform)
+ let orientationAdjustment = CGAffineTransform(
+ translationX: -preferredBounds.minX,
+ y: -preferredBounds.minY
+ )
+ let orientationTransform = preferredTransform.concatenating(orientationAdjustment)
+ guard orientationTransform.isNearlyInvertible else {
+ throw VideoEditorError.export("Invalid orientation transform during crop")
+ }
+
+ let cropRectDisplay = CGRect(
+ x: CGFloat(crop.x),
+ y: CGFloat(crop.y),
+ width: CGFloat(crop.width),
+ height: CGFloat(crop.height)
+ )
+ let displayBounds = CGRect(
+ origin: .zero,
+ size: CGSize(
+ width: abs(preferredBounds.width),
+ height: abs(preferredBounds.height)
+ )
+ )
+ let tolerance: CGFloat = 0.5
+ guard cropRectDisplay.minX >= displayBounds.minX - tolerance,
+ cropRectDisplay.minY >= displayBounds.minY - tolerance,
+ cropRectDisplay.maxX <= displayBounds.maxX + tolerance,
+ cropRectDisplay.maxY <= displayBounds.maxY + tolerance
+ else {
+ throw VideoEditorError.export("Crop rectangle exceeds the displayed video")
+ }
+
+ let displayCorners = [
+ cropRectDisplay.origin,
+ CGPoint(x: cropRectDisplay.maxX, y: cropRectDisplay.minY),
+ CGPoint(x: cropRectDisplay.minX, y: cropRectDisplay.maxY),
+ CGPoint(x: cropRectDisplay.maxX, y: cropRectDisplay.maxY),
+ ]
+ let orientationInverse = orientationTransform.inverted()
+ let fileCorners = displayCorners.map { $0.applying(orientationInverse) }
+ let fileMinX = fileCorners.map(\.x).min() ?? 0
+ let fileMinY = fileCorners.map(\.y).min() ?? 0
+ let fileMaxX = fileCorners.map(\.x).max() ?? 0
+ let fileMaxY = fileCorners.map(\.y).max() ?? 0
+ let fileCropRect = CGRect(
+ x: fileMinX,
+ y: fileMinY,
+ width: fileMaxX - fileMinX,
+ height: fileMaxY - fileMinY
+ )
+
+ var transform = CGAffineTransform(
+ translationX: -fileCropRect.origin.x,
+ y: -fileCropRect.origin.y
+ ).concatenating(orientationTransform)
+ if normalizedRotation != 0 {
+ let clockwiseRadians = CGFloat(normalizedRotation) * .pi / 180
+ transform = transform.concatenating(
+ CGAffineTransform(rotationAngle: clockwiseRadians)
+ )
+ }
+
+ let transformedCorners = fileCorners.map { $0.applying(transform) }
+ let minX = transformedCorners.map(\.x).min() ?? 0
+ let minY = transformedCorners.map(\.y).min() ?? 0
+ let maxX = transformedCorners.map(\.x).max() ?? 0
+ let maxY = transformedCorners.map(\.y).max() ?? 0
+ transform = transform.concatenating(
+ CGAffineTransform(translationX: -minX, y: -minY)
+ )
+
+ return VideoTransformPlan(
+ transform: transform,
+ renderSize: CGSize(
+ width: max((maxX - minX).rounded(), 1),
+ height: max((maxY - minY).rounded(), 1)
+ )
+ )
+ }
+
+ private static func rotationPlan(
+ naturalSize: CGSize,
+ preferredTransform: CGAffineTransform,
+ rotateDegrees: Int
+ ) -> VideoTransformPlan {
+ let naturalBounds = CGRect(origin: .zero, size: naturalSize)
+ let preferredBounds = naturalBounds.applying(preferredTransform)
+ let orientationAdjustment = CGAffineTransform(
+ translationX: -preferredBounds.minX,
+ y: -preferredBounds.minY
+ )
+ let orientationTransform = preferredTransform.concatenating(orientationAdjustment)
+ let orientedSize = CGSize(
+ width: abs(preferredBounds.width),
+ height: abs(preferredBounds.height)
+ )
+ let clockwiseRadians = CGFloat(rotateDegrees) * .pi / 180
+ let centerX = orientedSize.width / 2
+ let centerY = orientedSize.height / 2
+
+ var transform = orientationTransform
+ transform = transform.translatedBy(x: centerX, y: centerY)
+ transform = transform.rotated(by: clockwiseRadians)
+ transform = transform.translatedBy(x: -centerX, y: -centerY)
+
+ let transformedBounds = naturalBounds.applying(transform)
+ let renderSize = CGSize(
+ width: abs(transformedBounds.width),
+ height: abs(transformedBounds.height)
+ )
+ let targetMinX = (renderSize.width - transformedBounds.width) / 2
+ let targetMinY = (renderSize.height - transformedBounds.height) / 2
+ transform = transform.concatenating(
+ CGAffineTransform(
+ translationX: targetMinX - transformedBounds.minX,
+ y: targetMinY - transformedBounds.minY
+ )
+ )
+ return VideoTransformPlan(transform: transform, renderSize: renderSize)
+ }
+}
+
+private extension CGAffineTransform {
+ var isNearlyInvertible: Bool {
+ abs((a * d) - (b * c)) > 1e-8
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/Package.swift b/mobile/packages/native_video_editor/ios/Package.swift
new file mode 100644
index 0000000000..053b2a267b
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Package.swift
@@ -0,0 +1,27 @@
+// swift-tools-version: 5.9
+
+import PackageDescription
+
+let package = Package(
+ name: "EnteVideoCore",
+ platforms: [
+ .iOS(.v15),
+ .macOS(.v12),
+ ],
+ products: [
+ .library(name: "EnteVideoCore", targets: ["EnteVideoCore"])
+ ],
+ targets: [
+ .target(
+ name: "EnteVideoCore",
+ path: "Classes",
+ exclude: ["NativeVideoEditorPlugin.swift"]
+ ),
+ .testTarget(
+ name: "EnteVideoCoreTests",
+ dependencies: ["EnteVideoCore"],
+ path: "Tests"
+ ),
+ ],
+ swiftLanguageVersions: [.v5]
+)
diff --git a/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift b/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift
new file mode 100644
index 0000000000..cbe5a77b86
--- /dev/null
+++ b/mobile/packages/native_video_editor/ios/Tests/VideoTransformPlannerTests.swift
@@ -0,0 +1,76 @@
+import CoreGraphics
+import XCTest
+@testable import EnteVideoCore
+
+final class VideoTransformPlannerTests: XCTestCase {
+ func testClockwiseQuarterTurnProducesNormalizedPortraitBounds() throws {
+ let sourceBounds = CGRect(x: 0, y: 0, width: 320, height: 180)
+ let plan = try VideoTransformPlanner.makePlan(
+ naturalSize: sourceBounds.size,
+ preferredTransform: .identity,
+ crop: nil,
+ rotateDegrees: 90
+ )
+
+ assertEqual(plan.renderSize, CGSize(width: 180, height: 320))
+ assertEqual(sourceBounds.applying(plan.transform), CGRect(origin: .zero, size: plan.renderSize))
+ }
+
+ func testCropMapsDisplayRectangleToOutputBounds() throws {
+ let crop = try VideoCrop(x: 20, y: 30, width: 100, height: 60)
+ let plan = try VideoTransformPlanner.makePlan(
+ naturalSize: CGSize(width: 320, height: 180),
+ preferredTransform: .identity,
+ crop: crop,
+ rotateDegrees: 0
+ )
+
+ assertEqual(plan.renderSize, CGSize(width: 100, height: 60))
+ assertEqual(
+ CGRect(x: 20, y: 30, width: 100, height: 60).applying(plan.transform),
+ CGRect(origin: .zero, size: plan.renderSize)
+ )
+ }
+
+ func testDisplaySpaceCropRespectsMetadataRotation() throws {
+ let portraitTransform = CGAffineTransform(
+ a: 0,
+ b: 1,
+ c: -1,
+ d: 0,
+ tx: 180,
+ ty: 0
+ )
+ let crop = try VideoCrop(x: 20, y: 30, width: 80, height: 120)
+ let plan = try VideoTransformPlanner.makePlan(
+ naturalSize: CGSize(width: 320, height: 180),
+ preferredTransform: portraitTransform,
+ crop: crop,
+ rotateDegrees: 0
+ )
+
+ assertEqual(plan.renderSize, CGSize(width: 80, height: 120))
+ }
+
+ private func assertEqual(
+ _ actual: CGSize,
+ _ expected: CGSize,
+ file: StaticString = #filePath,
+ line: UInt = #line
+ ) {
+ XCTAssertEqual(actual.width, expected.width, accuracy: 0.001, file: file, line: line)
+ XCTAssertEqual(actual.height, expected.height, accuracy: 0.001, file: file, line: line)
+ }
+
+ private func assertEqual(
+ _ actual: CGRect,
+ _ expected: CGRect,
+ file: StaticString = #filePath,
+ line: UInt = #line
+ ) {
+ XCTAssertEqual(actual.minX, expected.minX, accuracy: 0.001, file: file, line: line)
+ XCTAssertEqual(actual.minY, expected.minY, accuracy: 0.001, file: file, line: line)
+ XCTAssertEqual(actual.width, expected.width, accuracy: 0.001, file: file, line: line)
+ XCTAssertEqual(actual.height, expected.height, accuracy: 0.001, file: file, line: line)
+ }
+}
diff --git a/mobile/packages/native_video_editor/ios/native_video_editor.podspec b/mobile/packages/native_video_editor/ios/native_video_editor.podspec
index bb4209790e..5888b65378 100644
--- a/mobile/packages/native_video_editor/ios/native_video_editor.podspec
+++ b/mobile/packages/native_video_editor/ios/native_video_editor.podspec
@@ -3,7 +3,7 @@ Pod::Spec.new do |s|
s.version = '0.0.1'
s.summary = 'Native video editing operations for iOS'
s.description = <<-DESC
-Native video editing operations for trim, crop, and rotate without re-encoding.
+Native video inspection, frame extraction, and editing with passthrough when eligible.
DESC
s.homepage = 'https://github.com/ente/ente'
s.license = { :file => '../LICENSE' }
@@ -11,9 +11,9 @@ Native video editing operations for trim, crop, and rotate without re-encoding.
s.source = { :path => '.' }
s.source_files = 'Classes/**/*'
s.dependency 'Flutter'
- s.platform = :ios, '11.0'
+ s.platform = :ios, '15.1'
s.swift_version = '5.0'
# Flutter.framework does not contain a i386 slice.
s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' }
-end
\ No newline at end of file
+end