VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(android): serialize dub cancel/resume/delete, place raw PCM by timestamp, split dub utterances at pauses, keep separate speak and clone model choices
debpalash931139ede4f9
diff --git a/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt b/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.ktindex 163860486..4888b9425 100644--- a/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt+++ b/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt@@ -42,6 +42,10 @@ class PhoneDubWorker(context: Context,params: WorkerParameters): CoroutineWorkeroverride suspend fun getForegroundInfo(): ForegroundInfo = notification()override suspend fun doWork(): Result {val job=inputData.getString("job") ?: return Result.failure()+ // Valid ids only: an invalid one fails inside run() with the checkpoint error as before.+ return if(runCatching {jobs.directory(job)}.isSuccess) jobs.exclusive(job) {execute(job)} else execute(job)+ }+ private suspend fun execute(job: String): Result {try {setForeground(notification())val runtime=PhoneRuntime.get(applicationContext)diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.ktindex 585c26b50..9b750b125 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt@@ -183,7 +183,14 @@ class AndroidAudioDecoder: AudioDecoder {var sum=0f;repeat(channels) {sum+=if(encoding==AudioFormat.ENCODING_PCM_FLOAT) buffer.float else buffer.short/32768f}(sum/channels).takeIf {it.isFinite()} ?: 0f}- val samples=converter.accept(mono);written+=samples.size+ var samples=converter.accept(mono)+ // Container timestamps are the truth: a late start or a dropped packet must stay silent, overlap must not drift.+ val shift=timelineShift(extractor.sampleTime,written)+ if(shift>0) {+ if(written+shift>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")+ appendSilence(out,shift);written+=shift+ } else if(shift<0) samples=samples.copyOfRange(minOf(-shift,samples.size.toLong()).toInt(),samples.size)+ written+=samples.sizeif(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)samples.forEach {bytes.putFloat(it)};out.write(bytes.array())@@ -201,6 +208,17 @@ class AndroidAudioDecoder: AudioDecoder {}+/**+ * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a raw-PCM buffer+ * stamped [sampleTimeUs] lands at its container time given [written] frames already output.+ * Jitter under 10 ms is ignored; a missing timestamp changes nothing.+ */+internal fun timelineShift(sampleTimeUs: Long,written: Long): Long {+ if(sampleTimeUs<0) return 0+ val delta=sampleTimeUs*24000/1_000_000-written+ return if(kotlin.math.abs(delta)<240) 0 else delta+}+/** Real recordings have no synthetic marker; generated audio uses markSynthetic. */internal fun writeReferenceWave(audio: LocalAudio, file: File) {val header=ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN)diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.ktindex 39eae1a00..e9bd33f21 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt@@ -97,13 +97,8 @@ class PhoneLocalEngine(private val models: ModelDownloads): LocalEngine {val text=result.text.trim()if(text.isNotEmpty()) {val duration=(limit-start).toDouble()/audio.sampleRate- val valid=result.timestamps.filter {it.isFinite()&&it>=0&&it<duration}- val first=(valid.minOrNull()?.toDouble() ?: 0.0).coerceIn(0.0,(duration-.02).coerceAtLeast(0.0))- val last=valid.maxOrNull()?.toDouble()- val tail=result.durations.lastOrNull()?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5- val end=if(last==null) duration else (last+tail).coerceIn(minOf(first+.01,duration),duration)val offset=start.toDouble()/audio.sampleRate- segments+=SpeechSegment(offset+first,offset+end,text)+ segments+=splitUtterances(text,result.tokens.toList(),result.timestamps.toList(),result.durations.toList(),duration).map {SpeechSegment(offset+it.start,offset+it.end,it.text)}}} finally {stream.release()}progress(limit.toFloat()/audio.samples.size)@@ -198,3 +193,42 @@ suspend fun verifyNativeRuntimes(): Map<String,Int> = withContext(Dispatchers.IOinternal class NativeTtsCallback(private val onChunk: () -> Int) : (FloatArray) -> Int {override fun invoke(samples: FloatArray): Int = onChunk()}++/** Pause that starts a new utterance; shorter after sentence-final punctuation. */+private const val UTTERANCE_GAP=1.0+private const val SENTENCE_GAP=.4+/**+ * One recognition window can hold several sentences with long pauses. Dubbing speaks each segment at its+ * start, so later sentences would play early; split at pauses so each keeps its own timing.+ * Falls back to one segment when tokens and timestamps do not line up.+ */+internal fun splitUtterances(text: String,tokens: List<String>,timestamps: List<Float>,durations: List<Float>,windowSeconds: Double): List<SpeechSegment> {+ val valid=tokens.size==timestamps.size&&tokens.isNotEmpty()&×tamps.all {it.isFinite()&&it>=0&&it<windowSeconds}+ fun single(): List<SpeechSegment> {+ val times=timestamps.filter {it.isFinite()&&it>=0&&it<windowSeconds}+ val first=(times.minOrNull()?.toDouble() ?: 0.0).coerceIn(0.0,(windowSeconds-.02).coerceAtLeast(0.0))+ val tail=durations.lastOrNull()?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5+ val end=times.maxOrNull()?.let {(it+tail).coerceIn(minOf(first+.01,windowSeconds),windowSeconds)} ?: windowSeconds+ return listOf(SpeechSegment(first,end,text))+ }+ if(!valid) return single()+ fun endOf(i: Int): Double {+ val tail=durations.getOrNull(i)?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5+ return (timestamps[i]+tail).coerceAtMost(windowSeconds)+ }+ val groups=mutableListOf(mutableListOf(0))+ for(i in 1 until tokens.size) {+ val gap=timestamps[i]-(if(durations.size==tokens.size) endOf(i-1) else timestamps[i-1].toDouble())+ val sentenceEnd=tokens[i-1].trimEnd().lastOrNull()?.let {it in ".!?。!?"}==true+ if(gap>=UTTERANCE_GAP||(sentenceEnd&&gap>=SENTENCE_GAP)) groups+=mutableListOf(i) else groups.last()+=i+ }+ val pieces=groups.mapIndexedNotNull {n,g ->+ val piece=g.joinToString("") {tokens[it]}.trim()+ if(piece.isEmpty()) null else {+ val start=timestamps[g.first()].toDouble()+ val nextStart=groups.getOrNull(n+1)?.let {timestamps[it.first()].toDouble()} ?: windowSeconds+ SpeechSegment(start,endOf(g.last()).coerceIn(start+.01,maxOf(start+.01,nextStart)).coerceAtMost(windowSeconds),piece)+ }+ }+ return if(pieces.size<=1) single() else pieces+}diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.ktindex 3c24d54fa..89eb2278e 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt@@ -2,6 +2,8 @@ package com.voicestudio.mobile.localimport com.voicestudio.mobile.network.ClientFailureimport kotlinx.coroutines.*+import kotlinx.coroutines.sync.Mutex+import kotlinx.coroutines.sync.withLockimport kotlinx.serialization.Serializableimport java.io.Fileimport java.nio.file.Files@@ -76,7 +78,17 @@ class PhoneDubStore(private val root: File) {catch(e:CancellationException) {throw e} catch(_:Exception) {null}}.sortedByDescending {it.request.createdAt}}- suspend fun delete(id: String) = withContext(Dispatchers.IO) {if(!directory(id).deleteRecursively()) throw ClientFailure("mobile.simple.checkpoint_failed")}+ /**+ * Serialises everything that writes one job (a running worker, its cancel cleanup, a resumed worker,+ * deletion). WorkManager reports a cancelled job finished before its native call has returned, so+ * the lock, not WorkInfo, is what makes resume and delete wait for the old run to finish saving.+ */+ suspend fun <T> exclusive(id: String,block: suspend ()->T): T {+ val key=directory(id).absolutePath+ return locks.getOrPut(key) {Mutex()}.withLock {block()}+ }+ suspend fun delete(id: String) = exclusive(id) {withContext(Dispatchers.IO) {if(!directory(id).deleteRecursively()) throw ClientFailure("mobile.simple.checkpoint_failed")}}+ private companion object {val locks=java.util.concurrent.ConcurrentHashMap<String,Mutex>()}}interface DubMedia {/** Returns the output file; a video export replaces its original soundtrack. */diff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.ktnew file mode 100644index 000000000..35f694e08--- /dev/null+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt@@ -0,0 +1,66 @@+package com.voicestudio.mobile.local++import kotlinx.coroutines.*+import org.junit.Assert.*+import org.junit.Rule+import org.junit.Test+import org.junit.rules.TemporaryFolder+import java.io.File+import java.util.UUID++class DubConcurrencyAndTimingTest {+ @get:Rule val temporary=TemporaryFolder()++ @Test fun exclusiveRunsForTheSameJobNeverInterleave() = runBlocking {+ val store=PhoneDubStore(File(temporary.root,"jobs"));val id=UUID.randomUUID().toString()+ val events=java.util.Collections.synchronizedList(mutableListOf<String>())+ val first=launch(Dispatchers.Default) {store.exclusive(id) {events+="a-start";delay(150);events+="a-end"}}+ delay(50)+ val second=launch(Dispatchers.Default) {store.exclusive(id) {events+="b-start";events+="b-end"}}+ joinAll(first,second)+ assertEquals(listOf("a-start","a-end","b-start","b-end"),events)+ }+ @Test fun deleteWaitsForTheRunningJobToFinishSaving() = runBlocking {+ val store=PhoneDubStore(File(temporary.root,"jobs"));val id=UUID.randomUUID().toString()+ store.directory(id).mkdirs()+ var finished=false;var sawDirectory=false+ val running=launch(Dispatchers.Default) {store.exclusive(id) {delay(150);sawDirectory=store.directory(id).exists();finished=true}}+ delay(50)+ store.delete(id)+ assertTrue(finished);assertTrue(sawDirectory);assertFalse(store.directory(id).exists())+ running.join()+ }+ @Test fun differentJobsDoNotBlockEachOther() = runBlocking {+ val store=PhoneDubStore(File(temporary.root,"jobs"))+ val gate=CompletableDeferred<Unit>()+ val holder=launch(Dispatchers.Default) {store.exclusive(UUID.randomUUID().toString()) {gate.await()}}+ delay(50)+ withTimeout(2000) {store.exclusive(UUID.randomUUID().toString()) {}}+ gate.complete(Unit);holder.join()+ }++ @Test fun rawTimelineInsertsSilenceForLateStartAndGaps() {+ assertEquals(24000L,timelineShift(1_000_000,0))+ assertEquals(12000L,timelineShift(1_500_000,12000))+ }+ @Test fun rawTimelineDropsOverlapAndIgnoresJitterAndMissingTime() {+ assertEquals(-2400L,timelineShift(900_000,24000))+ assertEquals(0L,timelineShift(1_004_000,24000))+ assertEquals(0L,timelineShift(-1,500))+ }++ @Test fun pausesSplitOneWindowIntoTimedUtterances() {+ val tokens=listOf(" Hello"," there","."," Much"," later",".")+ val ts=listOf(0.5f,0.9f,1.2f,9.0f,9.4f,9.8f)+ val parts=splitUtterances("Hello there. Much later.",tokens,ts,listOf(.3f,.3f,.2f,.3f,.3f,.2f),20.0)+ assertEquals(listOf("Hello there.","Much later."),parts.map {it.text})+ assertEquals(.5,parts[0].start,1e-6);assertEquals(9.0,parts[1].start,1e-6)+ assertTrue(parts[0].end<=9.0&&parts[1].end<=20.0&&parts[1].end>parts[1].start)+ }+ @Test fun continuousSpeechAndMismatchedTimestampsStayOneSegment() {+ val one=splitUtterances("a b c",listOf("a"," b"," c"),listOf(0f,.4f,.8f),emptyList(),20.0)+ assertEquals(1,one.size);assertEquals("a b c",one[0].text)+ val bad=splitUtterances("a b",listOf("a"," b"),listOf(0f),emptyList(),20.0)+ assertEquals(1,bad.size)+ }+}diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.ktindex 042f0fa20..3c8bc218c 100644--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt@@ -148,11 +148,11 @@ import com.voicestudio.mobile.local.*}}-@Composable fun TaskModelPicker(vm: StudioViewModel, task: String, capabilities: Set<String>, selected: PhoneModel?, compact: Boolean = false) {+@Composable fun TaskModelPicker(vm: StudioViewModel, task: String, capabilities: Set<String>, selected: PhoneModel?, compact: Boolean = false, key: String = task) {val phone by vm.phone.state.collectAsState()ChoicePicker("mobile.local.model",selected?.id.orEmpty(),phone.models.filter {it.capability in capabilities}.map {Choice(it.id,it.name,enabled=it.fits(vm.phone.ramBytes),supporting=if(it.fits(vm.phone.ramBytes)) tr("mobile.simple.${it.capability}") else tr("mobile.local.ram_required",mapOf("ram" to it.minRamGb)))- },compact=compact) {id -> vm.phone.cancelInference();vm.preference {it.copy(phoneModels=it.phoneModels+(task to id))}}+ },compact=compact) {id -> vm.phone.cancelInference();vm.preference {it.copy(phoneModels=it.phoneModels+(key to id))}}}/** One explicit Get action starts the dependency list; each gated model keeps its own sheet. */diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.ktindex 0e3be511b..d65650243 100644--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt@@ -12,6 +12,7 @@ import com.voicestudio.mobile.designsystem.*import androidx.compose.runtime.*import androidx.compose.runtime.saveable.rememberSaveableimport androidx.compose.ui.platform.LocalContext+import com.voicestudio.mobile.local.PhoneModelimport com.voicestudio.mobile.features.*import java.io.Fileimport kotlinx.coroutines.*@@ -79,6 +80,13 @@ import kotlinx.coroutines.*}}+/** Preset speech and saved-voice cloning use different models, so each keeps its own remembered choice. */+internal fun speakModelKey(capability: String)=if(capability=="clone") "speak-clone" else "speak"+internal fun chooseSpeakModel(models: List<PhoneModel>,installed: Set<String>,choices: Map<String,String>,capability: String): PhoneModel? {+ val matching=models.filter {it.capability==capability}+ return matching.firstOrNull {it.id==choices[speakModelKey(capability)]} ?: matching.firstOrNull {it.id in installed} ?: matching.firstOrNull()+}+@Composable fun SpeakScreen(vm: StudioViewModel, initialPhoneVoiceID: String? = null, initialComputerVoiceID: String? = null) {com.voicestudio.mobile.features.models.TaskLocation(vm,"speak",setOf("tts","clone")) {location,model ->if(location==com.voicestudio.mobile.local.RunLocation.COMPUTER) ComputerSpeakScreen(vm,initialComputerVoiceID)@@ -90,7 +98,8 @@ import kotlinx.coroutines.*var speaker by rememberSaveable(initialPhoneVoiceID) {mutableStateOf(initialPhoneVoiceID?.let {"phone:"+it} ?: "0")}val voice=phone.voices.firstOrNull {"phone:"+it.id==speaker}val capability=if(voice==null) "tts" else "clone"- val selected=model?.takeIf {it.capability==capability} ?: phone.models.firstOrNull {it.capability==capability&&it.id in phone.installed} ?: phone.models.firstOrNull {it.capability==capability}+ val prefs by vm.preferences.collectAsState()+ val selected=chooseSpeakModel(phone.models,phone.installed,prefs.phoneModels,capability)LaunchedEffect(phone.voices) {if(speaker.startsWith("phone:")&&voice==null) speaker="0"}val save=rememberLauncherForActivityResult(ActivityResultContracts.CreateDocument("audio/wav")) {uri ->val source=phone.audio@@ -111,7 +120,7 @@ import kotlinx.coroutines.*val languageChoices=if(voice!=null) listOf(Choice("auto",tr("languagePicker.auto"))) else selected?.languages.orEmpty().map {Choice(it,languageName(it,LocalL10n.current.locale))}Row(Modifier.fillMaxWidth(),horizontalArrangement=Arrangement.spacedBy(VSTheme.gap)) {Box(Modifier.weight(1f)) {ChoicePicker("mobile.speak.voice",speaker,(0 until speakers).map {Choice(it.toString(),tr("mobile.local.preset_voice",mapOf("number" to it+1)))}+phone.voices.map {Choice("phone:"+it.id,it.name)},compact=true) {speaker=it}}- Box(Modifier.weight(1f)) {com.voicestudio.mobile.features.models.TaskModelPicker(vm,"speak",setOf(capability),selected,compact=true)}+ Box(Modifier.weight(1f)) {com.voicestudio.mobile.features.models.TaskModelPicker(vm,"speak",setOf(capability),selected,compact=true,key=speakModelKey(capability))}Box(Modifier.weight(1f)) {ChoicePicker("mobile.speak.language",language,languageChoices,compact=true) {language=it}}}if(selected?.let {!phone.canRun(it,vm.phone.ramBytes)}!=false) selected?.let {com.voicestudio.mobile.features.models.MissingModelsCard(vm,"mobile.tabs.speak",listOf(it))}diff --git a/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.kt b/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.ktnew file mode 100644index 000000000..89a3f2f4f--- /dev/null+++ b/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.kt@@ -0,0 +1,24 @@+package com.voicestudio.mobile.features++import com.voicestudio.mobile.features.speak.chooseSpeakModel+import com.voicestudio.mobile.features.speak.speakModelKey+import com.voicestudio.mobile.local.ModelLicence+import com.voicestudio.mobile.local.PhoneModel+import org.junit.Assert.*+import org.junit.Test++class SpeakModelChoiceTest {+ private fun model(id: String,capability: String)=PhoneModel(id,capability,"sherpa-onnx","org/m","a".repeat(40),emptyList(),1,listOf("en"),1.0,ModelLicence("mit","commercial-ok",false,"","","b".repeat(64)))+ private val models=listOf(model("tts-a","tts"),model("tts-b","tts"),model("clone-a","clone"),model("clone-b","clone"))+ @Test fun presetAndCloneChoicesDoNotOverwriteEachOther() {+ var choices=emptyMap<String,String>()+ choices=choices+(speakModelKey("tts") to "tts-b")+ choices=choices+(speakModelKey("clone") to "clone-b")+ assertEquals("tts-b",chooseSpeakModel(models,setOf("tts-a"),choices,"tts")?.id)+ assertEquals("clone-b",chooseSpeakModel(models,emptySet(),choices,"clone")?.id)+ }+ @Test fun withoutAChoiceFirstInstalledThenFirstModelOfThatCapability() {+ assertEquals("clone-b",chooseSpeakModel(models,setOf("clone-b"),emptyMap(),"clone")?.id)+ assertEquals("tts-a",chooseSpeakModel(models,emptySet(),emptyMap(),"tts")?.id)+ }+}