Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(android): serialize dub cancel/resume/delete, place raw PCM by timestamp, split dub utterances at pauses, keep separate speak and clone model choices

debpalash931139ede4f9
diff --git a/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt b/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt
index 163860486..4888b9425 100644
--- a/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt
+++ b/mobile/android/core/data/src/main/kotlin/com/voicestudio/mobile/data/PhoneDubWorker.kt
@@ -42,6 +42,10 @@ class PhoneDubWorker(context: Context,params: WorkerParameters): CoroutineWorker
override suspend fun getForegroundInfo(): ForegroundInfo = notification()
override suspend fun doWork(): Result {
val job=inputData.getString("job") ?: return Result.failure()
+ // Valid ids only: an invalid one fails inside run() with the checkpoint error as before.
+ return if(runCatching {jobs.directory(job)}.isSuccess) jobs.exclusive(job) {execute(job)} else execute(job)
+ }
+ private suspend fun execute(job: String): Result {
try {
setForeground(notification())
val runtime=PhoneRuntime.get(applicationContext)
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
index 585c26b50..9b750b125 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
@@ -183,7 +183,14 @@ class AndroidAudioDecoder: AudioDecoder {
var sum=0f;repeat(channels) {sum+=if(encoding==AudioFormat.ENCODING_PCM_FLOAT) buffer.float else buffer.short/32768f}
(sum/channels).takeIf {it.isFinite()} ?: 0f
}
- val samples=converter.accept(mono);written+=samples.size
+ var samples=converter.accept(mono)
+ // Container timestamps are the truth: a late start or a dropped packet must stay silent, overlap must not drift.
+ val shift=timelineShift(extractor.sampleTime,written)
+ if(shift>0) {
+ if(written+shift>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
+ appendSilence(out,shift);written+=shift
+ } else if(shift<0) samples=samples.copyOfRange(minOf(-shift,samples.size.toLong()).toInt(),samples.size)
+ written+=samples.size
if(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)
samples.forEach {bytes.putFloat(it)};out.write(bytes.array())
@@ -201,6 +208,17 @@ class AndroidAudioDecoder: AudioDecoder {
}
+/**
+ * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a raw-PCM buffer
+ * stamped [sampleTimeUs] lands at its container time given [written] frames already output.
+ * Jitter under 10 ms is ignored; a missing timestamp changes nothing.
+ */
+internal fun timelineShift(sampleTimeUs: Long,written: Long): Long {
+ if(sampleTimeUs<0) return 0
+ val delta=sampleTimeUs*24000/1_000_000-written
+ return if(kotlin.math.abs(delta)<240) 0 else delta
+}
+
/** Real recordings have no synthetic marker; generated audio uses markSynthetic. */
internal fun writeReferenceWave(audio: LocalAudio, file: File) {
val header=ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN)
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt
index 39eae1a00..e9bd33f21 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/LocalEngine.kt
@@ -97,13 +97,8 @@ class PhoneLocalEngine(private val models: ModelDownloads): LocalEngine {
val text=result.text.trim()
if(text.isNotEmpty()) {
val duration=(limit-start).toDouble()/audio.sampleRate
- val valid=result.timestamps.filter {it.isFinite()&&it>=0&&it<duration}
- val first=(valid.minOrNull()?.toDouble() ?: 0.0).coerceIn(0.0,(duration-.02).coerceAtLeast(0.0))
- val last=valid.maxOrNull()?.toDouble()
- val tail=result.durations.lastOrNull()?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5
- val end=if(last==null) duration else (last+tail).coerceIn(minOf(first+.01,duration),duration)
val offset=start.toDouble()/audio.sampleRate
- segments+=SpeechSegment(offset+first,offset+end,text)
+ segments+=splitUtterances(text,result.tokens.toList(),result.timestamps.toList(),result.durations.toList(),duration).map {SpeechSegment(offset+it.start,offset+it.end,it.text)}
}
} finally {stream.release()}
progress(limit.toFloat()/audio.samples.size)
@@ -198,3 +193,42 @@ suspend fun verifyNativeRuntimes(): Map<String,Int> = withContext(Dispatchers.IO
internal class NativeTtsCallback(private val onChunk: () -> Int) : (FloatArray) -> Int {
override fun invoke(samples: FloatArray): Int = onChunk()
}
+
+/** Pause that starts a new utterance; shorter after sentence-final punctuation. */
+private const val UTTERANCE_GAP=1.0
+private const val SENTENCE_GAP=.4
+/**
+ * One recognition window can hold several sentences with long pauses. Dubbing speaks each segment at its
+ * start, so later sentences would play early; split at pauses so each keeps its own timing.
+ * Falls back to one segment when tokens and timestamps do not line up.
+ */
+internal fun splitUtterances(text: String,tokens: List<String>,timestamps: List<Float>,durations: List<Float>,windowSeconds: Double): List<SpeechSegment> {
+ val valid=tokens.size==timestamps.size&&tokens.isNotEmpty()&&timestamps.all {it.isFinite()&&it>=0&&it<windowSeconds}
+ fun single(): List<SpeechSegment> {
+ val times=timestamps.filter {it.isFinite()&&it>=0&&it<windowSeconds}
+ val first=(times.minOrNull()?.toDouble() ?: 0.0).coerceIn(0.0,(windowSeconds-.02).coerceAtLeast(0.0))
+ val tail=durations.lastOrNull()?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5
+ val end=times.maxOrNull()?.let {(it+tail).coerceIn(minOf(first+.01,windowSeconds),windowSeconds)} ?: windowSeconds
+ return listOf(SpeechSegment(first,end,text))
+ }
+ if(!valid) return single()
+ fun endOf(i: Int): Double {
+ val tail=durations.getOrNull(i)?.takeIf {it.isFinite()&&it>0}?.toDouble() ?: .5
+ return (timestamps[i]+tail).coerceAtMost(windowSeconds)
+ }
+ val groups=mutableListOf(mutableListOf(0))
+ for(i in 1 until tokens.size) {
+ val gap=timestamps[i]-(if(durations.size==tokens.size) endOf(i-1) else timestamps[i-1].toDouble())
+ val sentenceEnd=tokens[i-1].trimEnd().lastOrNull()?.let {it in ".!?。!?"}==true
+ if(gap>=UTTERANCE_GAP||(sentenceEnd&&gap>=SENTENCE_GAP)) groups+=mutableListOf(i) else groups.last()+=i
+ }
+ val pieces=groups.mapIndexedNotNull {n,g ->
+ val piece=g.joinToString("") {tokens[it]}.trim()
+ if(piece.isEmpty()) null else {
+ val start=timestamps[g.first()].toDouble()
+ val nextStart=groups.getOrNull(n+1)?.let {timestamps[it.first()].toDouble()} ?: windowSeconds
+ SpeechSegment(start,endOf(g.last()).coerceIn(start+.01,maxOf(start+.01,nextStart)).coerceAtMost(windowSeconds),piece)
+ }
+ }
+ return if(pieces.size<=1) single() else pieces
+}
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
index 3c24d54fa..89eb2278e 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
@@ -2,6 +2,8 @@ package com.voicestudio.mobile.local
import com.voicestudio.mobile.network.ClientFailure
import kotlinx.coroutines.*
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
import kotlinx.serialization.Serializable
import java.io.File
import java.nio.file.Files
@@ -76,7 +78,17 @@ class PhoneDubStore(private val root: File) {
catch(e:CancellationException) {throw e} catch(_:Exception) {null}
}.sortedByDescending {it.request.createdAt}
}
- suspend fun delete(id: String) = withContext(Dispatchers.IO) {if(!directory(id).deleteRecursively()) throw ClientFailure("mobile.simple.checkpoint_failed")}
+ /**
+ * Serialises everything that writes one job (a running worker, its cancel cleanup, a resumed worker,
+ * deletion). WorkManager reports a cancelled job finished before its native call has returned, so
+ * the lock, not WorkInfo, is what makes resume and delete wait for the old run to finish saving.
+ */
+ suspend fun <T> exclusive(id: String,block: suspend ()->T): T {
+ val key=directory(id).absolutePath
+ return locks.getOrPut(key) {Mutex()}.withLock {block()}
+ }
+ suspend fun delete(id: String) = exclusive(id) {withContext(Dispatchers.IO) {if(!directory(id).deleteRecursively()) throw ClientFailure("mobile.simple.checkpoint_failed")}}
+ private companion object {val locks=java.util.concurrent.ConcurrentHashMap<String,Mutex>()}
}
interface DubMedia {
/** Returns the output file; a video export replaces its original soundtrack. */
diff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt
new file mode 100644
index 000000000..35f694e08
--- /dev/null
+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt
@@ -0,0 +1,66 @@
+package com.voicestudio.mobile.local
+
+import kotlinx.coroutines.*
+import org.junit.Assert.*
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import java.io.File
+import java.util.UUID
+
+class DubConcurrencyAndTimingTest {
+ @get:Rule val temporary=TemporaryFolder()
+
+ @Test fun exclusiveRunsForTheSameJobNeverInterleave() = runBlocking {
+ val store=PhoneDubStore(File(temporary.root,"jobs"));val id=UUID.randomUUID().toString()
+ val events=java.util.Collections.synchronizedList(mutableListOf<String>())
+ val first=launch(Dispatchers.Default) {store.exclusive(id) {events+="a-start";delay(150);events+="a-end"}}
+ delay(50)
+ val second=launch(Dispatchers.Default) {store.exclusive(id) {events+="b-start";events+="b-end"}}
+ joinAll(first,second)
+ assertEquals(listOf("a-start","a-end","b-start","b-end"),events)
+ }
+ @Test fun deleteWaitsForTheRunningJobToFinishSaving() = runBlocking {
+ val store=PhoneDubStore(File(temporary.root,"jobs"));val id=UUID.randomUUID().toString()
+ store.directory(id).mkdirs()
+ var finished=false;var sawDirectory=false
+ val running=launch(Dispatchers.Default) {store.exclusive(id) {delay(150);sawDirectory=store.directory(id).exists();finished=true}}
+ delay(50)
+ store.delete(id)
+ assertTrue(finished);assertTrue(sawDirectory);assertFalse(store.directory(id).exists())
+ running.join()
+ }
+ @Test fun differentJobsDoNotBlockEachOther() = runBlocking {
+ val store=PhoneDubStore(File(temporary.root,"jobs"))
+ val gate=CompletableDeferred<Unit>()
+ val holder=launch(Dispatchers.Default) {store.exclusive(UUID.randomUUID().toString()) {gate.await()}}
+ delay(50)
+ withTimeout(2000) {store.exclusive(UUID.randomUUID().toString()) {}}
+ gate.complete(Unit);holder.join()
+ }
+
+ @Test fun rawTimelineInsertsSilenceForLateStartAndGaps() {
+ assertEquals(24000L,timelineShift(1_000_000,0))
+ assertEquals(12000L,timelineShift(1_500_000,12000))
+ }
+ @Test fun rawTimelineDropsOverlapAndIgnoresJitterAndMissingTime() {
+ assertEquals(-2400L,timelineShift(900_000,24000))
+ assertEquals(0L,timelineShift(1_004_000,24000))
+ assertEquals(0L,timelineShift(-1,500))
+ }
+
+ @Test fun pausesSplitOneWindowIntoTimedUtterances() {
+ val tokens=listOf(" Hello"," there","."," Much"," later",".")
+ val ts=listOf(0.5f,0.9f,1.2f,9.0f,9.4f,9.8f)
+ val parts=splitUtterances("Hello there. Much later.",tokens,ts,listOf(.3f,.3f,.2f,.3f,.3f,.2f),20.0)
+ assertEquals(listOf("Hello there.","Much later."),parts.map {it.text})
+ assertEquals(.5,parts[0].start,1e-6);assertEquals(9.0,parts[1].start,1e-6)
+ assertTrue(parts[0].end<=9.0&&parts[1].end<=20.0&&parts[1].end>parts[1].start)
+ }
+ @Test fun continuousSpeechAndMismatchedTimestampsStayOneSegment() {
+ val one=splitUtterances("a b c",listOf("a"," b"," c"),listOf(0f,.4f,.8f),emptyList(),20.0)
+ assertEquals(1,one.size);assertEquals("a b c",one[0].text)
+ val bad=splitUtterances("a b",listOf("a"," b"),listOf(0f),emptyList(),20.0)
+ assertEquals(1,bad.size)
+ }
+}
diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt
index 042f0fa20..3c8bc218c 100644
--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt
+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/models/ModelsScreen.kt
@@ -148,11 +148,11 @@ import com.voicestudio.mobile.local.*
}
}
-@Composable fun TaskModelPicker(vm: StudioViewModel, task: String, capabilities: Set<String>, selected: PhoneModel?, compact: Boolean = false) {
+@Composable fun TaskModelPicker(vm: StudioViewModel, task: String, capabilities: Set<String>, selected: PhoneModel?, compact: Boolean = false, key: String = task) {
val phone by vm.phone.state.collectAsState()
ChoicePicker("mobile.local.model",selected?.id.orEmpty(),phone.models.filter {it.capability in capabilities}.map {
Choice(it.id,it.name,enabled=it.fits(vm.phone.ramBytes),supporting=if(it.fits(vm.phone.ramBytes)) tr("mobile.simple.${it.capability}") else tr("mobile.local.ram_required",mapOf("ram" to it.minRamGb)))
- },compact=compact) {id -> vm.phone.cancelInference();vm.preference {it.copy(phoneModels=it.phoneModels+(task to id))}}
+ },compact=compact) {id -> vm.phone.cancelInference();vm.preference {it.copy(phoneModels=it.phoneModels+(key to id))}}
}
/** One explicit Get action starts the dependency list; each gated model keeps its own sheet. */
diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt
index 0e3be511b..d65650243 100644
--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt
+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/speak/SpeakScreen.kt
@@ -12,6 +12,7 @@ import com.voicestudio.mobile.designsystem.*
import androidx.compose.runtime.*
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.ui.platform.LocalContext
+import com.voicestudio.mobile.local.PhoneModel
import com.voicestudio.mobile.features.*
import java.io.File
import kotlinx.coroutines.*
@@ -79,6 +80,13 @@ import kotlinx.coroutines.*
}
}
+/** Preset speech and saved-voice cloning use different models, so each keeps its own remembered choice. */
+internal fun speakModelKey(capability: String)=if(capability=="clone") "speak-clone" else "speak"
+internal fun chooseSpeakModel(models: List<PhoneModel>,installed: Set<String>,choices: Map<String,String>,capability: String): PhoneModel? {
+ val matching=models.filter {it.capability==capability}
+ return matching.firstOrNull {it.id==choices[speakModelKey(capability)]} ?: matching.firstOrNull {it.id in installed} ?: matching.firstOrNull()
+}
+
@Composable fun SpeakScreen(vm: StudioViewModel, initialPhoneVoiceID: String? = null, initialComputerVoiceID: String? = null) {
com.voicestudio.mobile.features.models.TaskLocation(vm,"speak",setOf("tts","clone")) {location,model ->
if(location==com.voicestudio.mobile.local.RunLocation.COMPUTER) ComputerSpeakScreen(vm,initialComputerVoiceID)
@@ -90,7 +98,8 @@ import kotlinx.coroutines.*
var speaker by rememberSaveable(initialPhoneVoiceID) {mutableStateOf(initialPhoneVoiceID?.let {"phone:"+it} ?: "0")}
val voice=phone.voices.firstOrNull {"phone:"+it.id==speaker}
val capability=if(voice==null) "tts" else "clone"
- val selected=model?.takeIf {it.capability==capability} ?: phone.models.firstOrNull {it.capability==capability&&it.id in phone.installed} ?: phone.models.firstOrNull {it.capability==capability}
+ val prefs by vm.preferences.collectAsState()
+ val selected=chooseSpeakModel(phone.models,phone.installed,prefs.phoneModels,capability)
LaunchedEffect(phone.voices) {if(speaker.startsWith("phone:")&&voice==null) speaker="0"}
val save=rememberLauncherForActivityResult(ActivityResultContracts.CreateDocument("audio/wav")) {uri ->
val source=phone.audio
@@ -111,7 +120,7 @@ import kotlinx.coroutines.*
val languageChoices=if(voice!=null) listOf(Choice("auto",tr("languagePicker.auto"))) else selected?.languages.orEmpty().map {Choice(it,languageName(it,LocalL10n.current.locale))}
Row(Modifier.fillMaxWidth(),horizontalArrangement=Arrangement.spacedBy(VSTheme.gap)) {
Box(Modifier.weight(1f)) {ChoicePicker("mobile.speak.voice",speaker,(0 until speakers).map {Choice(it.toString(),tr("mobile.local.preset_voice",mapOf("number" to it+1)))}+phone.voices.map {Choice("phone:"+it.id,it.name)},compact=true) {speaker=it}}
- Box(Modifier.weight(1f)) {com.voicestudio.mobile.features.models.TaskModelPicker(vm,"speak",setOf(capability),selected,compact=true)}
+ Box(Modifier.weight(1f)) {com.voicestudio.mobile.features.models.TaskModelPicker(vm,"speak",setOf(capability),selected,compact=true,key=speakModelKey(capability))}
Box(Modifier.weight(1f)) {ChoicePicker("mobile.speak.language",language,languageChoices,compact=true) {language=it}}
}
if(selected?.let {!phone.canRun(it,vm.phone.ramBytes)}!=false) selected?.let {com.voicestudio.mobile.features.models.MissingModelsCard(vm,"mobile.tabs.speak",listOf(it))}
diff --git a/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.kt b/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.kt
new file mode 100644
index 000000000..89a3f2f4f
--- /dev/null
+++ b/mobile/android/features/src/test/kotlin/com/voicestudio/mobile/features/SpeakModelChoiceTest.kt
@@ -0,0 +1,24 @@
+package com.voicestudio.mobile.features
+
+import com.voicestudio.mobile.features.speak.chooseSpeakModel
+import com.voicestudio.mobile.features.speak.speakModelKey
+import com.voicestudio.mobile.local.ModelLicence
+import com.voicestudio.mobile.local.PhoneModel
+import org.junit.Assert.*
+import org.junit.Test
+
+class SpeakModelChoiceTest {
+ private fun model(id: String,capability: String)=PhoneModel(id,capability,"sherpa-onnx","org/m","a".repeat(40),emptyList(),1,listOf("en"),1.0,ModelLicence("mit","commercial-ok",false,"","","b".repeat(64)))
+ private val models=listOf(model("tts-a","tts"),model("tts-b","tts"),model("clone-a","clone"),model("clone-b","clone"))
+ @Test fun presetAndCloneChoicesDoNotOverwriteEachOther() {
+ var choices=emptyMap<String,String>()
+ choices=choices+(speakModelKey("tts") to "tts-b")
+ choices=choices+(speakModelKey("clone") to "clone-b")
+ assertEquals("tts-b",chooseSpeakModel(models,setOf("tts-a"),choices,"tts")?.id)
+ assertEquals("clone-b",chooseSpeakModel(models,emptySet(),choices,"clone")?.id)
+ }
+ @Test fun withoutAChoiceFirstInstalledThenFirstModelOfThatCapability() {
+ assertEquals("clone-b",chooseSpeakModel(models,setOf("clone-b"),emptyMap(),"clone")?.id)
+ assertEquals("tts-a",chooseSpeakModel(models,emptySet(),emptyMap(),"tts")?.id)
+ }
+}