Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(mobile): restore dub video surfaces and playback audio sessions

debpalash538a9f58debb
diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml
index d270440bb..3cdb369b1 100644
--- a/.github/workflows/mobile.yml
+++ b/.github/workflows/mobile.yml
@@ -142,11 +142,12 @@ jobs:
-resultBundlePath "$RUNNER_TEMP/ui.xcresult" \
CODE_SIGNING_ALLOWED=NO test-without-building
- - name: Native video export regression
+ - name: Native video export and playback regressions
run: |
xcodebuild -project mobile/ios/VoiceStudio/VoiceStudio.xcodeproj \
-scheme VoiceStudioNative -destination "id=$SIM_UDID" \
'-only-testing:VoiceStudioKitTests/phoneVideoMuxPreservesOrientationAndMarksPlayableAudio()' \
+ '-only-testing:VoiceStudioKitTests/phoneDubPlaybackRestoresPlaybackAfterDictation()' \
-parallel-testing-enabled NO CODE_SIGNING_ALLOWED=NO test
# ScreenshotTour keeps a screenshot of every screen; export them for review.
- name: Export UI screenshots
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5d24fa76a..0f6ac1246 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ metadata and the backend fallback mirror it.
### Fixed
+- Phone dub playback restores sound after dictation on iOS and keeps the video visible when switching results on Android (#2969)
- Android dubbing errors remain visible when worker updates arrive while their receipt is being saved (#3031)
- iOS reserves storage across simultaneous model downloads and keeps saved speech paired with the submitted text (#2969)
diff --git a/docs/mobile.md b/docs/mobile.md
index 6e4845c2a..c98890e0f 100644
--- a/docs/mobile.md
+++ b/docs/mobile.md
@@ -21,6 +21,8 @@ choose, per task, to run on **This phone** or **My computer**.
clone on the phone stay on the phone.
- **Dub**: pick a video or audio file and target languages; the phone
transcribes, translates and re-voices it in the original speaker's voice.
+ Play completed dubs directly in the app, including after using Dictate;
+ switching between video results keeps both picture and sound.
- **Dictate**: tap the mic and watch the transcript appear live.
- **Models**: download, update or delete the on-device models.
diff --git a/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.kt b/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.kt
new file mode 100644
index 000000000..9b142a7c9
--- /dev/null
+++ b/mobile/android/app/src/test/kotlin/com/voicestudio/mobile/LocalVideoPlaybackTest.kt
@@ -0,0 +1,51 @@
+package com.voicestudio.mobile
+
+import android.app.Application
+import android.view.SurfaceView
+import androidx.compose.runtime.*
+import androidx.compose.ui.test.junit4.createComposeRule
+import androidx.media3.common.Player
+import androidx.test.core.app.ApplicationProvider
+import com.voicestudio.mobile.designsystem.VoiceStudioTheme
+import com.voicestudio.mobile.features.LocalAudioPlayer
+import com.voicestudio.mobile.features.LocalAudioPlayerFactory
+import com.voicestudio.mobile.features.LocalL10n
+import com.voicestudio.mobile.l10n.L10n
+import org.junit.Assert.assertSame
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.ArgumentCaptor
+import org.mockito.Mockito.*
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.annotation.Config
+import java.io.File
+
+@RunWith(RobolectricTestRunner::class)
+@Config(sdk=[34])
+class LocalVideoPlaybackTest {
+ @get:Rule val compose=createComposeRule()
+
+ @Test fun switchingVideosAttachesTheRetainedSurfaceToTheNewPlayer() {
+ val first=mock(Player::class.java)
+ val second=mock(Player::class.java)
+ val players=ArrayDeque(listOf(first,second))
+ var file by mutableStateOf(File("first.mp4"))
+ val app=ApplicationProvider.getApplicationContext<Application>()
+ compose.setContent {
+ CompositionLocalProvider(LocalAudioPlayerFactory provides {players.removeFirst()},LocalL10n provides L10n.load(app,"en")) {
+ VoiceStudioTheme {LocalAudioPlayer(file) {throw AssertionError(it)}}
+ }
+ }
+ compose.waitForIdle()
+ val initial=ArgumentCaptor.forClass(SurfaceView::class.java)
+ verify(first,atLeastOnce()).setVideoSurfaceView(initial.capture())
+ compose.runOnIdle {file=File("second.mp4")}
+ compose.waitForIdle()
+ val replacement=ArgumentCaptor.forClass(SurfaceView::class.java)
+ verify(second,atLeastOnce()).setVideoSurfaceView(replacement.capture())
+ assertSame(initial.value,replacement.value)
+ verify(first).release()
+ verify(second,never()).release()
+ }
+}
diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt
index 6909e8cc3..2aba4b63a 100644
--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt
+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/Media.kt
@@ -142,9 +142,13 @@ val LocalHostPlayerFactory=staticCompositionLocalOf<(Context,VoiceStudioApi)->Pl
}
}
+val LocalAudioPlayerFactory=staticCompositionLocalOf<(Context)->Player> {
+ { context -> ExoPlayer.Builder(context).build() }
+}
@Composable fun LocalAudioPlayer(file: File, failure: (Throwable)->Unit) {
val context=LocalContext.current
- val player=remember(file) {ExoPlayer.Builder(context).build()}
+ val factory=LocalAudioPlayerFactory.current
+ val player=remember(file) {factory(context)}
var playing by remember(file) {mutableStateOf(false)}
var position by remember(file) {mutableLongStateOf(0)}
var duration by remember(file) {mutableLongStateOf(0)}
@@ -162,7 +166,7 @@ val LocalHostPlayerFactory=staticCompositionLocalOf<(Context,VoiceStudioApi)->Pl
Group {
Column(Modifier.fillMaxWidth().padding(VSTheme.inset)) {
if(file.extension=="mp4") Box(Modifier.fillMaxWidth().heightIn(max=(com.voicestudio.mobile.designsystem.VSTokens.SPACING_GAP*25).dp),contentAlignment=Alignment.Center) {
- androidx.compose.ui.viewinterop.AndroidView(factory={android.view.SurfaceView(it).also(player::setVideoSurfaceView)},modifier=Modifier.aspectRatio(videoRatio,matchHeightConstraintsFirst=true))
+ androidx.compose.ui.viewinterop.AndroidView(factory={android.view.SurfaceView(it)},update={player.setVideoSurfaceView(it)},modifier=Modifier.aspectRatio(videoRatio,matchHeightConstraintsFirst=true))
}
IconAction(if(playing) "player.pause" else "player.play",if(playing) VSIcons.Pause else Icons.Default.PlayArrow) {
if(playing) player.pause() else {if(player.playbackState==Player.STATE_ENDED) player.seekTo(0);player.play()}
diff --git a/mobile/ios/VoiceStudio/Features/PhoneDubView.swift b/mobile/ios/VoiceStudio/Features/PhoneDubView.swift
index 4ac455f54..df2af1bcf 100644
--- a/mobile/ios/VoiceStudio/Features/PhoneDubView.swift
+++ b/mobile/ios/VoiceStudio/Features/PhoneDubView.swift
@@ -133,10 +133,20 @@ struct PhoneDubView: View {
Task { await model.performPhone { try await model.phone.dubStore?.delete(record.id); await model.phone.refresh() } }
}
}
-private struct PhoneDubPlayback: View {
+struct PhoneDubPlayback: View {
+ @Environment(AppModel.self) private var model
let url: URL
@State private var player: AVPlayer?
+ @MainActor static func startPlayback(url: URL) throws -> AVPlayer {
+ try activatePlaybackSession()
+ let player = AVPlayer(url: url)
+ player.play()
+ return player
+ }
var body: some View {
- VideoPlayer(player: player).onAppear { player = AVPlayer(url: url); player?.play() }.onDisappear { player?.pause(); player = nil }
+ VideoPlayer(player: player).onAppear {
+ do { player = try Self.startPlayback(url: url) }
+ catch { model.error = ClientError("mobile.errors.audio") }
+ }.onDisappear { player?.pause(); player = nil }
}
}
diff --git a/mobile/ios/VoiceStudio/Platform/Audio.swift b/mobile/ios/VoiceStudio/Platform/Audio.swift
index 78ca53fb3..37bb456d3 100644
--- a/mobile/ios/VoiceStudio/Platform/Audio.swift
+++ b/mobile/ios/VoiceStudio/Platform/Audio.swift
@@ -5,6 +5,11 @@ import Observation
import VoiceStudioKit
import UniformTypeIdentifiers
+@MainActor func activatePlaybackSession() throws {
+ try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)
+ try AVAudioSession.sharedInstance().setActive(true)
+}
+
@MainActor @Observable final class AudioPlayer {
private var player: AVPlayer?
private var ticker: Task<Void, Never>?
@@ -64,8 +69,7 @@ import UniformTypeIdentifiers
func play() {
guard let player else { return }
do {
- try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)
- try AVAudioSession.sharedInstance().setActive(true)
+ try activatePlaybackSession()
} catch { self.error = ClientError("mobile.errors.audio"); return }
if duration > 0 && position >= duration { seek(0) }
player.play(); playing = true
diff --git a/mobile/ios/VoiceStudio/project.yml b/mobile/ios/VoiceStudio/project.yml
index 7231ee93b..af8e0ca58 100644
--- a/mobile/ios/VoiceStudio/project.yml
+++ b/mobile/ios/VoiceStudio/project.yml
@@ -32,7 +32,6 @@ targets:
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
GENERATE_INFOPLIST_FILE: 'NO'
INFOPLIST_FILE: $(DERIVED_FILE_DIR)/VoiceStudio-Info.plist
- ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
dependencies:
- package: VoiceStudioKit
- framework: ../../native/ios/build/xcframeworks/sherpa-onnx.xcframework
diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swift
new file mode 100644
index 000000000..d4f0cfc8d
--- /dev/null
+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhonePlaybackTests.swift
@@ -0,0 +1,17 @@
+#if os(iOS) && canImport(VoiceStudio)
+import AVFoundation
+import Foundation
+import Testing
+@testable import VoiceStudio
+
+@Test @MainActor func phoneDubPlaybackRestoresPlaybackAfterDictation() throws {
+ let session = AVAudioSession.sharedInstance()
+ try session.setCategory(.record, mode: .measurement)
+ defer { try? session.setActive(false) }
+ let player = try PhoneDubPlayback.startPlayback(url: URL(fileURLWithPath: "/tmp/dub.mp4"))
+ defer { player.pause() }
+ #expect(session.category == .playback)
+ #expect(session.mode == .default)
+ #expect(player.currentItem != nil)
+}
+#endif