Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(mobile): preserve offline model access and decoded audio timing

debpalash6150584753e8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 853c3799a..8c8a9244e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,8 @@ metadata and the backend fallback mirror it.
### Fixed
+- Phone models can be reaccepted offline without downloading again, and Android dubbing preserves short audio gaps and overlaps (#2969)
+
- Bug reports no longer risk freezing the app while looking for your tools: the Node-version folder scan now runs off the main thread (#3180)
- Bug reports no longer freeze the app when the tool search or a drive stalls, and free disk is only borrowed from a parent folder when the install folder doesn't exist yet (#3179)
- Model licence dialog is wider and scrolls as one area with the buttons pinned, and follows macOS, Windows and Linux dialog conventions (#3160)
diff --git a/docs/mobile.md b/docs/mobile.md
index 7f1c75595..87deac70a 100644
--- a/docs/mobile.md
+++ b/docs/mobile.md
@@ -60,11 +60,13 @@ Plain `http://` works only for addresses on your own network (home Wi-Fi,
- **"Accept the licence on your computer"**: some models need their licence
accepted first. Open VoiceStudio on the computer → **Model Catalogue** and
- accept it there; the phone can't accept licences.
-- **Engine shown as unavailable**: install it on the computer. The reason the
- app shows comes from the computer.
+ accept it there for the **My computer** route. For **This phone**, review and
+ accept the licence in the phone’s **Models** screen.
+- **Computer engine shown as unavailable**: install it on the paired computer.
+ For phone tasks, install the model from the phone’s **Models** screen.
- **Dictate isn't ready**: install the speech-to-text model the app names, on
- the computer. Remote dictation also needs an API-key session or a trusted
+ the phone for **This phone**, or the computer for **My computer**. Remote
+ dictation also needs an API-key session or a trusted
network set up on the computer; the share PIN alone isn't enough.
- **Can't connect**: make sure the phone and computer are on the same network
(or both on Tailscale), sharing is on, and you used the IP address the panel
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
index 9b750b125..eaf576427 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/AudioFiles.kt
@@ -132,13 +132,7 @@ class AndroidAudioDecoder: AudioDecoder {
}
val converter=resampler ?: MonoResampler(rate,24000).also {resampler=it}
val samples=converter.accept(mono)
- val expected=info.presentationTimeUs.coerceAtLeast(0)*24000/1_000_000
- val gap=(expected-written).takeIf {it>2400} ?: 0L
- if(written+gap+samples.size>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
- appendSilence(out,gap);written+=gap+samples.size
- if(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
- val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)
- samples.forEach {bytes.putFloat(it)};out.write(bytes.array())
+ written=appendDecodedPcm(out,samples,info.presentationTimeUs,written,maxSeconds)
if(durationUs>0) progress((info.presentationTimeUs.toFloat()/durationUs).coerceIn(0f,1f))
}
outputDone=info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM!=0
@@ -183,17 +177,7 @@ class AndroidAudioDecoder: AudioDecoder {
var sum=0f;repeat(channels) {sum+=if(encoding==AudioFormat.ENCODING_PCM_FLOAT) buffer.float else buffer.short/32768f}
(sum/channels).takeIf {it.isFinite()} ?: 0f
}
- var samples=converter.accept(mono)
- // Container timestamps are the truth: a late start or a dropped packet must stay silent, overlap must not drift.
- val shift=timelineShift(extractor.sampleTime,written)
- if(shift>0) {
- if(written+shift>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
- appendSilence(out,shift);written+=shift
- } else if(shift<0) samples=samples.copyOfRange(minOf(-shift,samples.size.toLong()).toInt(),samples.size)
- written+=samples.size
- if(written>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
- val bytes=ByteBuffer.allocate(samples.size*4).order(ByteOrder.LITTLE_ENDIAN)
- samples.forEach {bytes.putFloat(it)};out.write(bytes.array())
+ written=appendDecodedPcm(out,converter.accept(mono),extractor.sampleTime,written,maxSeconds)
if(durationUs>0) progress((extractor.sampleTime.toFloat()/durationUs).coerceIn(0f,1f))
extractor.advance()
}
@@ -208,8 +192,24 @@ class AndroidAudioDecoder: AudioDecoder {
}
+/** Write one decoded buffer at its container timestamp. */
+internal suspend fun appendDecodedPcm(out: FileOutputStream,samples: FloatArray,timeUs: Long,written: Long,maxSeconds: Int): Long {
+ val shift=timelineShift(timeUs,written)
+ val gap=shift.coerceAtLeast(0)
+ val skip=minOf((-shift).coerceAtLeast(0),samples.size.toLong()).toInt()
+ val end=written+gap+samples.size-skip
+ if(end>maxSeconds*24000L) throw ClientFailure("mobile.simple.audio_long")
+ var remaining=gap*4
+ val zeros=ByteArray(32768)
+ while(remaining>0) {currentCoroutineContext().ensureActive();val count=minOf(zeros.size.toLong(),remaining).toInt();out.write(zeros,0,count);remaining-=count}
+ val bytes=ByteBuffer.allocate((samples.size-skip)*4).order(ByteOrder.LITTLE_ENDIAN)
+ for(i in skip until samples.size) bytes.putFloat(samples[i])
+ out.write(bytes.array())
+ return end
+}
+
/**
- * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a raw-PCM buffer
+ * Frames of 24 kHz silence to insert (positive) or leading frames to drop (negative) so a decoded audio buffer
* stamped [sampleTimeUs] lands at its container time given [written] frames already output.
* Jitter under 10 ms is ignored; a missing timestamp changes nothing.
*/
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt
index 716d46fe9..370a79c01 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/ModelDownloads.kt
@@ -60,7 +60,7 @@ class ModelDownloads internal constructor(
private fun fit(model: PhoneModel) {if(!model.fits(ramBytes)) throw ClientFailure("mobile.local.ram_required",mapOf("ram" to model.minRamGb.toString()))}
suspend fun download(model: PhoneModel, allowMobile: Boolean) = withContext(Dispatchers.IO) {
lock(model).withLock {
- fit(model);licence.requireAccepted(model);networkAllowed(allowMobile)
+ fit(model);licence.requireAccepted(model)
val job=currentCoroutineContext().job
val directory=dir(model).apply {mkdirs()}
val catalogue=modelPin(model).identity
@@ -69,9 +69,10 @@ class ModelDownloads internal constructor(
val partial=File(directory,f.path+".part").length().takeIf {it<=f.size} ?: 0L
f.size-partial
}
+ if(needed>0) networkAllowed(allowMobile)
// Concurrent downloads each pass the check alone; reserve what the others still need.
synchronized(reservations) {
- if(freeBytes()-reservations.values.sum()<needed+16*1024*1024) throw ClientFailure("mobile.local.storage_required")
+ if(needed>0 && freeBytes()-reservations.values.sum()<needed+16*1024*1024) throw ClientFailure("mobile.local.storage_required")
reservations[model.id]=needed
}
mutable.update {it+(model.id to DownloadProgress(total=model.totalBytes,running=true))}
diff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt
index 8628aaef3..b4a02e342 100644
--- a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt
+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/DubConcurrencyAndTimingTest.kt
@@ -39,6 +39,24 @@ class DubConcurrencyAndTimingTest {
gate.complete(Unit);holder.join()
}
+ @Test fun decodedBuffersPreserveShortGapsAndTrimOverlap() = runBlocking {
+ val output=File(temporary.root,"decoded.pcm")
+ java.io.FileOutputStream(output).use {stream ->
+ var written=appendDecodedPcm(stream,FloatArray(1200) {.25f},50_000,0,1)
+ assertEquals(2400L,written) // 50 ms of silence then 50 ms of speech
+ written=appendDecodedPcm(stream,FloatArray(1200) {.5f},150_000,written,1)
+ assertEquals(4800L,written)
+ written=appendDecodedPcm(stream,FloatArray(1200) {.75f},175_000,written,1)
+ assertEquals(5400L,written) // overlapping 25 ms is discarded
+ }
+ val samples=PcmFile(output).read(0,5400).samples
+ assertTrue(samples.sliceArray(0 until 1200).all {it==0f})
+ assertTrue(samples.sliceArray(1200 until 2400).all {it==.25f})
+ assertTrue(samples.sliceArray(2400 until 3600).all {it==0f})
+ assertTrue(samples.sliceArray(3600 until 4800).all {it==.5f})
+ assertTrue(samples.sliceArray(4800 until 5400).all {it==.75f})
+ }
+
@Test fun rawTimelineInsertsSilenceForLateStartAndGaps() {
assertEquals(24000L,timelineShift(1_000_000,0))
assertEquals(24000L,timelineShift(1_500_000,12000)) // 1.5 s = 36000 samples, 12000 written
diff --git a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt
index 78d7331c6..299d3dab8 100644
--- a/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt
+++ b/mobile/android/core/local/src/test/kotlin/com/voicestudio/mobile/local/LocalPolicyTest.kt
@@ -98,6 +98,20 @@ class LocalPolicyTest {
assertEquals(0,server.requestCount)
}
}
+ @Test fun verifiedInstalledModelCanBeReacceptedOfflineWithoutDownloadSpace() = runTest {
+ MockWebServer().use {server -> server.start();server.enqueue(MockResponse().setBody(Buffer().write(payload)))
+ val m=model(true);val g=gate();g.record(m,true)
+ downloads(server,g).download(m,false)
+ g.record(m,false);g.record(m,true)
+ // A fresh downloader verifies disk contents rather than trusting an in-memory stamp.
+ val offline=downloads(server,g,wifi={false},free={0})
+ offline.download(m,false)
+ assertTrue(offline.installed(m));assertEquals(1,server.requestCount)
+ File(temporary.root,"models/fixture/weights.bin").writeBytes(ByteArray(payload.size))
+ failure("mobile.local.wifi_required") {downloads(server,g,wifi={false},free={0}).download(m,false)}
+ assertEquals(1,server.requestCount)
+ }
+ }
@Test fun fallbackAndMobileDataRequireAnExplicitDownload() = runTest {
MockWebServer().use {primary ->MockWebServer().use {mirror ->
primary.start();mirror.start();primary.enqueue(MockResponse().setResponseCode(503));mirror.enqueue(MockResponse().setBody(Buffer().write(payload)))
@@ -116,6 +130,22 @@ class LocalPolicyTest {
d.delete(model());assertEquals(0L,d.usedBytes())
}
}
+ @Test fun concurrentDownloadsReserveDiskAndCancellationReleasesIt() = runBlocking {
+ MockWebServer().use {server -> server.start()
+ server.enqueue(MockResponse().setSocketPolicy(okhttp3.mockwebserver.SocketPolicy.NO_RESPONSE))
+ server.enqueue(MockResponse().setBody(Buffer().write(payload)))
+ val d=downloads(server,free={16*1024*1024+payload.size.toLong()})
+ val first=launch(Dispatchers.IO) {d.download(model(),false)}
+ try {
+ withContext(Dispatchers.IO) {assertNotNull(server.takeRequest(5,TimeUnit.SECONDS))}
+ failure("mobile.local.storage_required") {withTimeout(5000) {d.download(model().copy(id="second"),false)}}
+ assertEquals(1,server.requestCount)
+ } finally {first.cancelAndJoin()}
+ withTimeout(5000) {d.download(model().copy(id="second"),false)}
+ assertTrue(d.installed(model().copy(id="second")))
+ assertEquals(2,server.requestCount)
+ }
+ }
@Test fun withdrawalBlocksPreviouslyVerifiedWeights() = runTest {
MockWebServer().use {server ->server.start();server.enqueue(MockResponse().setBody(Buffer().write(payload)))
val m=model(true);val g=gate();g.record(m,true);val d=downloads(server,g);d.download(m,false)
diff --git a/mobile/contract/SPEC.md b/mobile/contract/SPEC.md
index 93860dc5e..91cc680f2 100644
--- a/mobile/contract/SPEC.md
+++ b/mobile/contract/SPEC.md
@@ -255,8 +255,9 @@ All tabs work without pairing once on-device models are installed (see M2); the
### On-device dubbing
Extract audio (AVFoundation / MediaExtractor), transcribe (sherpa Whisper),
-translate per segment (llama.cpp), synthesize per segment cloning the original
-speaker from a reference clip cut from the source (omnivoice.cpp), fit timing,
+translate per segment (llama.cpp), synthesize per segment using either the
+original speaker from a source reference clip or a saved phone voice
+(omnivoice.cpp), fit timing,
mix, and mux back into the video (AVAssetWriter / MediaMuxer). Long jobs run as
a foreground task with progress (Android foreground service; iOS keeps
progress visible and resumes from a saved checkpoint after interruption).
diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
index 2c4033797..03ebe3e40 100644
--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
@@ -185,6 +185,9 @@ private actor RangeServer: HTTPTransport {
// iOS links the runtimes statically, so a missing library path cannot make them unavailable there.
#expect(!NativeLocalEngine.runtimeAvailable(.omni, library: "/does-not-exist"))
#expect(!NativeLocalEngine.runtimeAvailable(.llama, library: "/does-not-exist"))
+ #else
+ #expect(NativeLocalEngine.runtimeAvailable(.omni))
+ #expect(NativeLocalEngine.runtimeAvailable(.llama))
#endif
await #expect(throws: (any Error).self) { try await engine.clone(model: clone, text: "Hello", reference: PCMChunk(samples: [0], sampleRate: 16000), referenceText: "Reference") }
try await store.download(translate, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))
diff --git a/mobile/native/ios/build.py b/mobile/native/ios/build.py
index 596f112b9..2f39acbae 100644
--- a/mobile/native/ios/build.py
+++ b/mobile/native/ios/build.py
@@ -196,6 +196,8 @@ def stamp_omni_source(verified, output):
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
run('git', 'apply', '--check', patch, cwd=target, env=env)
run('git', 'apply', patch, cwd=target, env=env)
+ # Fail closed if a tool invocation ever succeeds without applying all hunks.
+ run('git', 'apply', '--reverse', '--check', patch, cwd=target, env=env)
return target