VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(mobile): preserve offline model access and decoded audio timing
debpalash6150584753e8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 853c3799a..8c8a9244e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -29,6 +29,8 @@ metadata and the backend fallback mirror it.### Fixed+- Phone models can be reaccepted offline without downloading again, and Android dubbing preserves short audio gaps and overlaps (#2969)+- Bug reports no longer risk freezing the app while looking for your tools: the Node-version folder scan now runs off the main thread (#3180)- Bug reports no longer freeze the app when the tool search or a drive stalls, and free disk is only borrowed from a parent folder when the install folder doesn't exist yet (#3179)- Model licence dialog is wider and scrolls as one area with the buttons pinned, and follows macOS, Windows and Linux dialog conventions (#3160)diff --git a/docs/mobile.md b/docs/mobile.mdindex 7f1c75595..87deac70a 100644--- a/docs/mobile.md+++ b/docs/mobile.md@@ -60,11 +60,13 @@ Plain `http://` works only for addresses on your own network (home Wi-Fi,- **"Accept the licence on your computer"**: some models need their licenceaccepted first. Open VoiceStudio on the computer → **Model Catalogue** and- accept it there; the phone can't accept licences.-- **Engine shown as unavailable**: install it on the computer. The reason the- app shows comes from the computer.+ accept it there for the **My computer** route. For **This phone**, review and+ accept the licence in the phone’s **Models** screen.+- **Computer engine shown as unavailable**: install it on the paired computer.+ For phone tasks, install the model from the phone’s **Models** screen.- **Dictate isn't ready**: install the speech-to-text model the app names, on- the computer. Remote dictation also needs an API-key session or a trusted+ the phone for **This phone**, or the computer for **My computer**. Remote+ dictation also needs an API-key session or a trustednetwork set up on the computer; the share PIN alone isn't enough.- **Can't connect**: make sure the phone and computer are on the same network(or both on Tailscale), sharing is on, and you used the IP address the paneldiff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.ktindex 9b750b125..eaf576427 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt@@ -132,13 +132,7 @@ class AndroidAudioDecoder: AudioDecoder {}val converter=resampler ?: MonoResampler(rate,24000).also {resampler=it}val samples=converter.accept(mono)- val expected=info.presentationTimeUs.coerceAtLeast(0)*24000/1_000_000- val gap=(expected-written).takeIf {it>2400} ?: 0L- if(written+gap+samples.size>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")- appendSilence(out,gap);written+=gap+samples.size- if(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")- val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)- samples.forEach {bytes.putFloat(it)};out.write(bytes.array())+ written=appendDecodedPcm(out,samples,info.presentationTimeUs,written,maxSeconds)if(durationUs>0) progress((info.presentationTimeUs.toFloat()/durationUs).coerceIn(0f,1f))}outputDone=info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM!=0@@ -183,17 +177,7 @@ class AndroidAudioDecoder: AudioDecoder {var sum=0f;repeat(channels) {sum+=if(encoding==AudioFormat.ENCODING_PCM_FLOAT) buffer.float else buffer.short/32768f}(sum/channels).takeIf {it.isFinite()} ?: 0f}- var samples=converter.accept(mono)- // Container timestamps are the truth: a late start or a dropped packet must stay silent, overlap must not drift.- val shift=timelineShift(extractor.sampleTime,written)- if(shift>0) {- if(written+shift>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")- appendSilence(out,shift);written+=shift- } else if(shift<0) samples=samples.copyOfRange(minOf(-shift,samples.size.toLong()).toInt(),samples.size)- written+=samples.size- if(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")- val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)- samples.forEach {bytes.putFloat(it)};out.write(bytes.array())+ written=appendDecodedPcm(out,converter.accept(mono),extractor.sampleTime,written,maxSeconds)if(durationUs>0) progress((extractor.sampleTime.toFloat()/durationUs).coerceIn(0f,1f))extractor.advance()}@@ -208,8 +192,24 @@ class AndroidAudioDecoder: AudioDecoder {}+/** Write one decoded buffer at its container timestamp. */+internal suspend fun appendDecodedPcm(out: FileOutputStream,samples: FloatArray,timeUs: Long,written: Long,maxSeconds: Int): Long {+ val shift=timelineShift(timeUs,written)+ val gap=shift.coerceAtLeast(0)+ val skip=minOf((-shift).coerceAtLeast(0),samples.size.toLong()).toInt()+ val end=written+gap+samples.size-skip+ if(end>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")+ var remaining=gap*4+ val zeros=ByteArray(32768)+ while(remaining>0) {currentCoroutineContext().ensureActive();val count=minOf(zeros.size.toLong(),remaining).toInt();out.write(zeros,0,count);remaining-=count}+ val bytes=ByteBuffer.allocate((samples.size-skip)*4).order(ByteOrder.LITTLE_ENDIAN)+ for(i in skip until samples.size) bytes.putFloat(samples[i])+ out.write(bytes.array())+ return end+}+/**- * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a raw-PCM buffer+ * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a decoded audio buffer* stamped [sampleTimeUs] lands at its container time given [written] frames already output.* Jitter under 10 ms is ignored; a missing timestamp changes nothing.*/diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.ktindex 716d46fe9..370a79c01 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt@@ -60,7 +60,7 @@ class ModelDownloads internal constructor(private fun fit(model: PhoneModel) {if(!model.fits(ramBytes)) throw ClientFailure("mobile.local.ram_required",mapOf("ram" to model.minRamGb.toString()))}suspend fun download(model: PhoneModel, allowMobile: Boolean) = withContext(Dispatchers.IO) {lock(model).withLock {- fit(model);licence.requireAccepted(model);networkAllowed(allowMobile)+ fit(model);licence.requireAccepted(model)val job=currentCoroutineContext().jobval directory=dir(model).apply {mkdirs()}val catalogue=modelPin(model).identity@@ -69,9 +69,10 @@ class ModelDownloads internal constructor(val partial=File(directory,f.path+".part").length().takeIf {it<=f.size} ?: 0Lf.size-partial}+ if(needed>0) networkAllowed(allowMobile)// Concurrent downloads each pass the check alone; reserve what the others still need.synchronized(reservations) {- if(freeBytes()-reservations.values.sum()<needed+16*1024*1024) throw ClientFailure("mobile.local.storage_required")+ if(needed>0 && freeBytes()-reservations.values.sum()<needed+16*1024*1024) throw ClientFailure("mobile.local.storage_required")reservations[model.id]=needed}mutable.update {it+(model.id to DownloadProgress(total=model.totalBytes,running=true))}diff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.ktindex 8628aaef3..b4a02e342 100644--- a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt@@ -39,6 +39,24 @@ class DubConcurrencyAndTimingTest {gate.complete(Unit);holder.join()}+ @Test fun decodedBuffersPreserveShortGapsAndTrimOverlap() = runBlocking {+ val output=File(temporary.root,"decoded.pcm")+ java.io.FileOutputStream(output).use {stream ->+ var written=appendDecodedPcm(stream,FloatArray(1200) {.25f},50_000,0,1)+ assertEquals(2400L,written) // 50 ms of silence then 50 ms of speech+ written=appendDecodedPcm(stream,FloatArray(1200) {.5f},150_000,written,1)+ assertEquals(4800L,written)+ written=appendDecodedPcm(stream,FloatArray(1200) {.75f},175_000,written,1)+ assertEquals(5400L,written) // overlapping 25 ms is discarded+ }+ val samples=PcmFile(output).read(0,5400).samples+ assertTrue(samples.sliceArray(0 until 1200).all {it==0f})+ assertTrue(samples.sliceArray(1200 until 2400).all {it==.25f})+ assertTrue(samples.sliceArray(2400 until 3600).all {it==0f})+ assertTrue(samples.sliceArray(3600 until 4800).all {it==.5f})+ assertTrue(samples.sliceArray(4800 until 5400).all {it==.75f})+ }+@Test fun rawTimelineInsertsSilenceForLateStartAndGaps() {assertEquals(24000L,timelineShift(1_000_000,0))assertEquals(24000L,timelineShift(1_500_000,12000)) // 1.5 s = 36000 samples, 12000 writtendiff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.ktindex 78d7331c6..299d3dab8 100644--- a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt@@ -98,6 +98,20 @@ class LocalPolicyTest {assertEquals(0,server.requestCount)}}+ @Test fun verifiedInstalledModelCanBeReacceptedOfflineWithoutDownloadSpace() = runTest {+ MockWebServer().use {server -> server.start();server.enqueue(MockResponse().setBody(Buffer().write(payload)))+ val m=model(true);val g=gate();g.record(m,true)+ downloads(server,g).download(m,false)+ g.record(m,false);g.record(m,true)+ // A fresh downloader verifies disk contents rather than trusting an in-memory stamp.+ val offline=downloads(server,g,wifi={false},free={0})+ offline.download(m,false)+ assertTrue(offline.installed(m));assertEquals(1,server.requestCount)+ File(temporary.root,"models/fixture/weights.bin").writeBytes(ByteArray(payload.size))+ failure("mobile.local.wifi_required") {downloads(server,g,wifi={false},free={0}).download(m,false)}+ assertEquals(1,server.requestCount)+ }+ }@Test fun fallbackAndMobileDataRequireAnExplicitDownload() = runTest {MockWebServer().use {primary ->MockWebServer().use {mirror ->primary.start();mirror.start();primary.enqueue(MockResponse().setResponseCode(503));mirror.enqueue(MockResponse().setBody(Buffer().write(payload)))@@ -116,6 +130,22 @@ class LocalPolicyTest {d.delete(model());assertEquals(0L,d.usedBytes())}}+ @Test fun concurrentDownloadsReserveDiskAndCancellationReleasesIt() = runBlocking {+ MockWebServer().use {server -> server.start()+ server.enqueue(MockResponse().setSocketPolicy(okhttp3.mockwebserver.SocketPolicy.NO_RESPONSE))+ server.enqueue(MockResponse().setBody(Buffer().write(payload)))+ val d=downloads(server,free={16*1024*1024+payload.size.toLong()})+ val first=launch(Dispatchers.IO) {d.download(model(),false)}+ try {+ withContext(Dispatchers.IO) {assertNotNull(server.takeRequest(5,TimeUnit.SECONDS))}+ failure("mobile.local.storage_required") {withTimeout(5000) {d.download(model().copy(id="second"),false)}}+ assertEquals(1,server.requestCount)+ } finally {first.cancelAndJoin()}+ withTimeout(5000) {d.download(model().copy(id="second"),false)}+ assertTrue(d.installed(model().copy(id="second")))+ assertEquals(2,server.requestCount)+ }+ }@Test fun withdrawalBlocksPreviouslyVerifiedWeights() = runTest {MockWebServer().use {server ->server.start();server.enqueue(MockResponse().setBody(Buffer().write(payload)))val m=model(true);val g=gate();g.record(m,true);val d=downloads(server,g);d.download(m,false)diff --git a/mobile/contract/SPEC.md b/mobile/contract/SPEC.mdindex 93860dc5e..91cc680f2 100644--- a/mobile/contract/SPEC.md+++ b/mobile/contract/SPEC.md@@ -255,8 +255,9 @@ All tabs work without pairing once on-device models are installed (see M2); the### On-device dubbingExtract audio (AVFoundation / MediaExtractor), transcribe (sherpa Whisper),-translate per segment (llama.cpp), synthesize per segment cloning the original-speaker from a reference clip cut from the source (omnivoice.cpp), fit timing,+translate per segment (llama.cpp), synthesize per segment using either the+original speaker from a source reference clip or a saved phone voice+(omnivoice.cpp), fit timing,mix, and mux back into the video (AVAssetWriter / MediaMuxer). Long jobs run asa foreground task with progress (Android foreground service; iOS keepsprogress visible and resumes from a saved checkpoint after interruption).diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swiftindex 2c4033797..03ebe3e40 100644--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift@@ -185,6 +185,9 @@ private actor RangeServer: HTTPTransport {// iOS links the runtimes statically, so a missing library path cannot make them unavailable there.#expect(!NativeLocalEngine.runtimeAvailable(.omni, library: "/does-not-exist"))#expect(!NativeLocalEngine.runtimeAvailable(.llama, library: "/does-not-exist"))+ #else+ #expect(NativeLocalEngine.runtimeAvailable(.omni))+ #expect(NativeLocalEngine.runtimeAvailable(.llama))#endifawait #expect(throws: (any Error).self) { try await engine.clone(model: clone, text: "Hello", reference: PCMChunk(samples: [0], sampleRate: 16000), referenceText: "Reference") }try await store.download(translate, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))diff --git a/mobile/native/ios/build.py b/mobile/native/ios/build.pyindex 596f112b9..2f39acbae 100644--- a/mobile/native/ios/build.py+++ b/mobile/native/ios/build.py@@ -196,6 +196,8 @@ def stamp_omni_source(verified, output):stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:run('git', 'apply', '--check', patch, cwd=target, env=env)run('git', 'apply', patch, cwd=target, env=env)+ # Fail closed if a tool invocation ever succeeds without applying all hunks.+ run('git', 'apply', '--reverse', '--check', patch, cwd=target, env=env)return target