Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(mobile): pair blank-recognition dub references with their own text; connect after Android QR scan

debpalash4f666100620d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 873a1a74f..f98ac5eae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,7 +27,8 @@ metadata and the backend fallback mirror it.
- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)
### Fixed
-- Refined iOS navigation, voice libraries, grouped creation forms and licence sheets with native controls, accessible text and explicit cancellation. (#2969)
+-- Phone dubbing no longer speaks the wrong words when a quiet reference clip is not recognised, and Android pairing connects right after a QR scan
+ Refined iOS navigation, voice libraries, grouped creation forms and licence sheets with native controls, accessible text and explicit cancellation. (#2969)
- Phone apps open on an Electron-style Home dashboard with shared artwork, workspace shortcuts, recent audio, saved voices and resumable dubbing projects (#2969)
diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
index 846bc1df7..3c24d54fa 100644
--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt
@@ -146,12 +146,20 @@ class PhoneDubPipeline(private val store: PhoneDubStore,private val engine: Loca
val previous=checkpoint.renders.firstOrNull {it.index==index}
if(previous!=null&&previous.frames==frames.toLong()&&rendered.length()==frames*4L&&sha256(rendered)==previous.sha256) continue
stage("mobile.android.stage_render",base)
- val reference=if(request.referenceText!=null) {
+ val duration=segment.speech.end-segment.speech.start
+ // A saved text equal to the segment's own transcript means recognition of the 8 s clip came back
+ // blank: the reference is then the whole segment, so a resumed job pairs the same audio with the same words.
+ val wholeSegment=request.referenceText==null&&duration>8.0&&!segment.referenceText.isNullOrBlank()&&segment.referenceText==segment.speech.text
+ var reference=if(request.referenceText!=null) {
val saved=PcmFile(store.file(id,"voice-reference.pcm"));saved.read(0,saved.frames.toInt())
- } else pcm.clip(segment.speech.start,minOf(segment.speech.end,segment.speech.start+8.0))
+ } else pcm.clip(segment.speech.start,if(wholeSegment) segment.speech.end else minOf(segment.speech.end,segment.speech.start+8.0))
val referenceText=request.referenceText ?: segment.referenceText?.takeIf {it.isNotBlank()} ?: run {
- val text=if(segment.speech.end-segment.speech.start<=8.0) segment.speech.text
- else engine.transcribe(asr,reference).trim().ifBlank {segment.speech.text}
+ val recognized=if(duration<=8.0) segment.speech.text else engine.transcribe(asr,reference).trim()
+ val text=recognized.ifBlank {
+ // Quiet or short clip: use the whole segment with its own transcript, never a clip with someone else's words.
+ reference=pcm.clip(segment.speech.start,segment.speech.end)
+ segment.speech.text
+ }
segment=segment.copy(referenceText=text)
save(checkpoint.copy(segments=checkpoint.segments.toMutableList().also {it[index]=segment}))
text
diff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt
index e4d9bbd01..d56990370 100644
--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt
+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt
@@ -25,7 +25,7 @@ import java.util.Date
ScreenTitle("mobile.connect.title")
Action("mobile.connect.scan_qr",!state.busy,onClick=camera)
if(scanner) scanContent({raw ->
- try {val address=PairingAddress.parse(raw);url=address.baseUrl.toString();pin=address.pin.orEmpty();scanner=false;manual=true}
+ try {val address=PairingAddress.parse(raw);url=address.baseUrl.toString();pin=address.pin.orEmpty();scanner=false;manual=true;vm.connect(url,pin,"")}
catch(e:Exception) {scanner=false;vm.fail(e)}
},{scanner=false},{scanner=false;vm.fail(it)})
Secondary("mobile.simple.address") {manual=!manual}
diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
index 2182ed1e1..b2ce50b88 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
@@ -204,18 +204,25 @@ public struct PhoneDubPipeline: Sendable {
let rendered = try await store.file(id,"render-\(targetIndex)-\(index).pcm")
if let saved = cp.renders.first(where: { $0.target == target.code && $0.index == index }), saved.frames == frames, (try? PCMFile(rendered).frames) == frames, (try? ModelHash.file(rendered)) == saved.hash { continue }
try await stage("mobile.android.stage_render",progress)
- let reference: PCMChunk
+ let savedText = (request.referenceText ?? segment.referenceText)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+ let segmentText = segment.speech.text.trimmingCharacters(in: .whitespacesAndNewlines)
+ // When recognition of the 8 s clip came back blank, the saved text is the segment's own transcript and
+ // the reference is the whole segment, so a resumed job pairs the same audio with the same words.
+ let wholeSegment = request.referenceText == nil && !savedText.isEmpty && savedText == segmentText && segment.speech.end - segment.speech.start > 8
+ var reference: PCMChunk
if request.referenceText != nil {
let pcm = PCMFile(try await store.file(id,"reference.pcm")); reference = try pcm.read(start: 0,count: pcm.frames)
- } else { reference = try original.clip(start: segment.speech.start,end: min(segment.speech.end,segment.speech.start+8)) }
- let savedText = (request.referenceText ?? segment.referenceText)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+ } else { reference = try original.clip(start: segment.speech.start,end: wholeSegment ? segment.speech.end : min(segment.speech.end,segment.speech.start+8)) }
let referenceText: String
if !savedText.isEmpty { referenceText = savedText }
else {
let recognized = try await engine.transcribe(model: asr,audio: reference,progress: { _ in }).trimmingCharacters(in: .whitespacesAndNewlines)
- // Quiet/short clips still have the segment's original ASR transcript.
- referenceText = recognized.isEmpty ? segment.speech.text.trimmingCharacters(in: .whitespacesAndNewlines) : recognized
- guard !referenceText.isEmpty else { throw ClientError("mobile.simple.no_speech") }
+ if recognized.isEmpty {
+ guard !segmentText.isEmpty else { throw ClientError("mobile.simple.no_speech") }
+ // Quiet or short clip: use the whole segment with its own transcript, never a clip with someone else's words.
+ reference = try original.clip(start: segment.speech.start,end: segment.speech.end)
+ referenceText = segmentText
+ } else { referenceText = recognized }
}
if segment.referenceText != referenceText {
segment.referenceText = referenceText; cp.segments[index] = segment; try await save(cp)
diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift
index 948dcb7fc..714659d4c 100644
--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift
+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift
@@ -75,6 +75,9 @@ private actor FixtureEngine: LocalEngine {
private(set) var translations = 0
private(set) var clones = 0
private var failClone = 0
+ private var longSegment = false
+ private(set) var references: [(frames: Int, text: String)] = []
+ func useLongSegment() { longSegment = true }
func failOnClone(_ count: Int) { failClone = count }
func empty() { transcript = "" }
func synthesize(model: PhoneModel,text: String,speaker: Int,speed: Float,progress: @escaping EngineProgressHandler) async throws -> LocalAudio { LocalAudio(samples: Array(repeating: 0.25,count: 24000),sampleRate: 24000) }
@@ -82,11 +85,13 @@ private actor FixtureEngine: LocalEngine {
clones += 1
if clones == failClone { throw CancellationError() }
#expect(reference.sampleRate == 24000); #expect(!referenceText.isEmpty)
+ references.append((reference.samples.count, referenceText))
return LocalAudio(samples: Array(repeating: 0.4,count: 12000),sampleRate: 24000)
}
func transcribe(model: PhoneModel,audio: PCMChunk,progress: @escaping EngineProgressHandler) async throws -> String { try Task.checkCancellation(); transcriptions += 1; return transcript }
func transcribeSegments(model: PhoneModel,audio: PCMChunk,progress: @escaping EngineProgressHandler) async throws -> [SpeechSegment] {
transcriptions += 1
+ if longSegment { return [SpeechSegment(start: 0.5,end: 11.5,text: "Hello long")] }
return [SpeechSegment(start: 0.1,end: 0.35,text: "Hello"), SpeechSegment(start: 0.5,end: 0.85,text: "World")]
}
func streamTranscribe(model: PhoneModel,chunks: AsyncThrowingStream<PCMChunk,any Error>,progress: @escaping EngineProgressHandler) async throws -> AsyncThrowingStream<TranscriptUpdate,any Error> { AsyncThrowingStream { $0.finish() } }
@@ -295,6 +300,23 @@ func realAuxiliaryPhoneDubOfflineAndResume() async throws {
#expect(result.segments.map(\.referenceText) == ["Hello","World"])
}
+@Test func blankRecognitionOfALongSegmentUsesTheWholeSegmentWithItsOwnText() async throws {
+ let root = try fixtureRoot(); defer { try? FileManager.default.removeItem(at: root) }
+ let source = root.appendingPathComponent("source.wav"); try Data("fixture".utf8).write(to: source)
+ let models = try [fixtureModel(.asr),fixtureModel(.translate),fixtureModel(.clone)]
+ let store = try PhoneDubStore(root: root.appendingPathComponent("dubs")), engine = FixtureEngine()
+ await engine.empty(); await engine.useLongSegment(); await engine.failOnClone(1)
+ let request = try await store.create(source: source,title: "Source",targets: [PhoneDubTarget(code: "fr",name: "French")],models: models,consent: true,audioOnly: true)
+ let pipeline = PhoneDubPipeline(store: store,engine: engine,decoder: FixtureDecoder(frames: 24000 * 12),media: FixtureMedia())
+ await #expect(throws: CancellationError.self) { try await pipeline.run(id: request.id,models: models) }
+ // The resumed job must pair the same audio with the same words as the interrupted one would have.
+ #expect(try await pipeline.run(id: request.id,models: models).done)
+ let references = await engine.references
+ #expect(references.count == 1)
+ #expect(references.first?.text == "Hello long")
+ #expect(references.first?.frames == 11 * 24000)
+}
+
@Test func reviewCorruptRecordsDoNotHideHealthyLibraries() async throws {
let root = try fixtureRoot(); defer { try? FileManager.default.removeItem(at: root) }
let voicesRoot = root.appendingPathComponent("voices"), dubsRoot = root.appendingPathComponent("dubs")