VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(mobile): restore dub video surfaces and playback audio sessions
debpalash538a9f58debb
diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.ymlindex d270440bb..3cdb369b1 100644--- a/.github/workflows/mobile.yml+++ b/.github/workflows/mobile.yml@@ -142,11 +142,12 @@ jobs:-resultBundlePath "$RUNNER_TEMP/ui.xcresult" \CODE_SIGNING_ALLOWED=NO test-without-building- - name: Native video export regression+ - name: Native video export and playback regressionsrun: |xcodebuild -project mobile/ios/VoiceStudio/VoiceStudio.xcodeproj \-scheme VoiceStudioNative -destination "id=$SIM_UDID" \'-only-testing:VoiceStudioKitTests/phoneVideoMuxPreservesOrientationAndMarksPlayableAudio()' \+ '-only-testing:VoiceStudioKitTests/phoneDubPlaybackRestoresPlaybackAfterDictation()' \-parallel-testing-enabled NO CODE_SIGNING_ALLOWED=NO test# ScreenshotTour keeps a screenshot of every screen; export them for review.- name: Export UI screenshotsdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5d24fa76a..0f6ac1246 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -29,6 +29,7 @@ metadata and the backend fallback mirror it.### Fixed+- Phone dub playback restores sound after dictation on iOS and keeps the video visible when switching results on Android (#2969)- Android dubbing errors remain visible when worker updates arrive while their receipt is being saved (#3031)- iOS reserves storage across simultaneous model downloads and keeps saved speech paired with the submitted text (#2969)diff --git a/docs/mobile.md b/docs/mobile.mdindex 6e4845c2a..c98890e0f 100644--- a/docs/mobile.md+++ b/docs/mobile.md@@ -21,6 +21,8 @@ choose, per task, to run on **This phone** or **My computer**.clone on the phone stay on the phone.- **Dub**: pick a video or audio file and target languages; the phonetranscribes, translates and re-voices it in the original speaker's voice.+ Play completed dubs directly in the app, including after using Dictate;+ switching between video results keeps both picture and sound.- **Dictate**: tap the mic and watch the transcript appear live.- **Models**: download, update or delete the on-device models.diff --git a/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.kt b/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.ktnew file mode 100644index 000000000..9b142a7c9--- /dev/null+++ b/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.kt@@ -0,0 +1,51 @@+package com.voicestudio.mobile++import android.app.Application+import android.view.SurfaceView+import androidx.compose.runtime.*+import androidx.compose.ui.test.junit4.createComposeRule+import androidx.media3.common.Player+import androidx.test.core.app.ApplicationProvider+import com.voicestudio.mobile.designsystem.VoiceStudioTheme+import com.voicestudio.mobile.features.LocalAudioPlayer+import com.voicestudio.mobile.features.LocalAudioPlayerFactory+import com.voicestudio.mobile.features.LocalL10n+import com.voicestudio.mobile.l10n.L10n+import org.junit.Assert.assertSame+import org.junit.Rule+import org.junit.Test+import org.junit.runner.RunWith+import org.mockito.ArgumentCaptor+import org.mockito.Mockito.*+import org.robolectric.RobolectricTestRunner+import org.robolectric.annotation.Config+import java.io.File++@RunWith(RobolectricTestRunner::class)+@Config(sdk=[34])+class LocalVideoPlaybackTest {+ @get:Rule val compose=createComposeRule()++ @Test fun switchingVideosAttachesTheRetainedSurfaceToTheNewPlayer() {+ val first=mock(Player::class.java)+ val second=mock(Player::class.java)+ val players=ArrayDeque(listOf(first,second))+ var file by mutableStateOf(File("first.mp4"))+ val app=ApplicationProvider.getApplicationContext<Application>()+ compose.setContent {+ CompositionLocalProvider(LocalAudioPlayerFactory provides {players.removeFirst()},LocalL10n provides L10n.load(app,"en")) {+ VoiceStudioTheme {LocalAudioPlayer(file) {throw AssertionError(it)}}+ }+ }+ compose.waitForIdle()+ val initial=ArgumentCaptor.forClass(SurfaceView::class.java)+ verify(first,atLeastOnce()).setVideoSurfaceView(initial.capture())+ compose.runOnIdle {file=File("second.mp4")}+ compose.waitForIdle()+ val replacement=ArgumentCaptor.forClass(SurfaceView::class.java)+ verify(second,atLeastOnce()).setVideoSurfaceView(replacement.capture())+ assertSame(initial.value,replacement.value)+ verify(first).release()+ verify(second,never()).release()+ }+}diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.ktindex 6909e8cc3..2aba4b63a 100644--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt@@ -142,9 +142,13 @@ val LocalHostPlayerFactory=staticCompositionLocalOf<(Context,VoiceStudioApi)->Pl}}+val LocalAudioPlayerFactory=staticCompositionLocalOf<(Context)->Player> {+ { context -> ExoPlayer.Builder(context).build() }+}@Composable fun LocalAudioPlayer(file: File, failure: (Throwable)->Unit) {val context=LocalContext.current- val player=remember(file) {ExoPlayer.Builder(context).build()}+ val factory=LocalAudioPlayerFactory.current+ val player=remember(file) {factory(context)}var playing by remember(file) {mutableStateOf(false)}var position by remember(file) {mutableLongStateOf(0)}var duration by remember(file) {mutableLongStateOf(0)}@@ -162,7 +166,7 @@ val LocalHostPlayerFactory=staticCompositionLocalOf<(Context,VoiceStudioApi)->PlGroup {Column(Modifier.fillMaxWidth().padding(VSTheme.inset)) {if(file.extension=="mp4") Box(Modifier.fillMaxWidth().heightIn(max=(com.voicestudio.mobile.designsystem.VSTokens.SPACING_GAP*25).dp),contentAlignment=Alignment.Center) {- androidx.compose.ui.viewinterop.AndroidView(factory={android.view.SurfaceView(it).also(player::setVideoSurfaceView)},modifier=Modifier.aspectRatio(videoRatio,matchHeightConstraintsFirst=true))+ androidx.compose.ui.viewinterop.AndroidView(factory={android.view.SurfaceView(it)},update={player.setVideoSurfaceView(it)},modifier=Modifier.aspectRatio(videoRatio,matchHeightConstraintsFirst=true))}IconAction(if(playing) "player.pause" else "player.play",if(playing) VSIcons.Pause else Icons.Default.PlayArrow) {if(playing) player.pause() else {if(player.playbackState==Player.STATE_ENDED) player.seekTo(0);player.play()}diff --git a/mobile/ios/VoiceStudio/Features/PhoneDubView.swift b/mobile/ios/VoiceStudio/Features/PhoneDubView.swiftindex 4ac455f54..df2af1bcf 100644--- a/mobile/ios/VoiceStudio/Features/PhoneDubView.swift+++ b/mobile/ios/VoiceStudio/Features/PhoneDubView.swift@@ -133,10 +133,20 @@ struct PhoneDubView: View {Task { await model.performPhone { try await model.phone.dubStore?.delete(record.id); await model.phone.refresh() } }}}-private struct PhoneDubPlayback: View {+struct PhoneDubPlayback: View {+ @Environment(AppModel.self) private var modellet url: URL@State private var player: AVPlayer?+ @MainActor static func startPlayback(url: URL) throws -> AVPlayer {+ try activatePlaybackSession()+ let player = AVPlayer(url: url)+ player.play()+ return player+ }var body: some View {- VideoPlayer(player: player).onAppear { player = AVPlayer(url: url); player?.play() }.onDisappear { player?.pause(); player = nil }+ VideoPlayer(player: player).onAppear {+ do { player = try Self.startPlayback(url: url) }+ catch { model.error = ClientError("mobile.errors.audio") }+ }.onDisappear { player?.pause(); player = nil }}}diff --git a/mobile/ios/VoiceStudio/Platform/Audio.swift b/mobile/ios/VoiceStudio/Platform/Audio.swiftindex 78ca53fb3..37bb456d3 100644--- a/mobile/ios/VoiceStudio/Platform/Audio.swift+++ b/mobile/ios/VoiceStudio/Platform/Audio.swift@@ -5,6 +5,11 @@ import Observationimport VoiceStudioKitimport UniformTypeIdentifiers+@MainActor func activatePlaybackSession() throws {+ try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)+ try AVAudioSession.sharedInstance().setActive(true)+}+@MainActor @Observable final class AudioPlayer {private var player: AVPlayer?private var ticker: Task<Void, Never>?@@ -64,8 +69,7 @@ import UniformTypeIdentifiersfunc play() {guard let player else { return }do {- try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)- try AVAudioSession.sharedInstance().setActive(true)+ try activatePlaybackSession()} catch { self.error = ClientError("mobile.errors.audio"); return }if duration > 0 && position >= duration { seek(0) }player.play(); playing = truediff --git a/mobile/ios/VoiceStudio/project.yml b/mobile/ios/VoiceStudio/project.ymlindex 7231ee93b..af8e0ca58 100644--- a/mobile/ios/VoiceStudio/project.yml+++ b/mobile/ios/VoiceStudio/project.yml@@ -32,7 +32,6 @@ targets:ASSETCATALOG_COMPILER_APPICON_NAME: AppIconGENERATE_INFOPLIST_FILE: 'NO'INFOPLIST_FILE: $(DERIVED_FILE_DIR)/VoiceStudio-Info.plist- ASSETCATALOG_COMPILER_APPICON_NAME: AppIcondependencies:- package: VoiceStudioKit- framework: ../../native/ios/build/xcframeworks/sherpa-onnx.xcframeworkdiff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swiftnew file mode 100644index 000000000..d4f0cfc8d--- /dev/null+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swift@@ -0,0 +1,17 @@+#if os(iOS) && canImport(VoiceStudio)+import AVFoundation+import Foundation+import Testing+@testable import VoiceStudio++@Test @MainActor func phoneDubPlaybackRestoresPlaybackAfterDictation() throws {+ let session = AVAudioSession.sharedInstance()+ try session.setCategory(.record, mode: .measurement)+ defer { try? session.setActive(false) }+ let player = try PhoneDubPlayback.startPlayback(url: URL(fileURLWithPath: "/tmp/dub.mp4"))+ defer { player.pause() }+ #expect(session.category == .playback)+ #expect(session.mode == .default)+ #expect(player.currentItem != nil)+}+#endif