VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(mobile): pair blank-recognition dub references with their own text; connect after Android QR scan
debpalash4f666100620d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 873a1a74f..f98ac5eae 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,7 +27,8 @@ metadata and the backend fallback mirror it.- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)### Fixed-- Refined iOS navigation, voice libraries, grouped creation forms and licence sheets with native controls, accessible text and explicit cancellation. (#2969)+-- Phone dubbing no longer speaks the wrong words when a quiet reference clip is not recognised, and Android pairing connects right after a QR scan+ Refined iOS navigation, voice libraries, grouped creation forms and licence sheets with native controls, accessible text and explicit cancellation. (#2969)- Phone apps open on an Electron-style Home dashboard with shared artwork, workspace shortcuts, recent audio, saved voices and resumable dubbing projects (#2969)diff --git a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.ktindex 846bc1df7..3c24d54fa 100644--- a/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt+++ b/mobile/android/core/local/src/main/kotlin/com/voicestudio/mobile/local/PhoneDub.kt@@ -146,12 +146,20 @@ class PhoneDubPipeline(private val store: PhoneDubStore,private val engine: Locaval previous=checkpoint.renders.firstOrNull {it.index==index}if(previous!=null&&previous.frames==frames.toLong()&&rendered.length()==frames*4L&&sha256(rendered)==previous.sha256) continuestage("mobile.android.stage_render",base)- val reference=if(request.referenceText!=null) {+ val duration=segment.speech.end-segment.speech.start+ // A saved text equal to the segment's own transcript means recognition of the 8 s clip came back+ // blank: the reference is then the whole segment, so a resumed job pairs the same audio with the same words.+ val wholeSegment=request.referenceText==null&&duration>8.0&&!segment.referenceText.isNullOrBlank()&&segment.referenceText==segment.speech.text+ var reference=if(request.referenceText!=null) {val saved=PcmFile(store.file(id,"voice-reference.pcm"));saved.read(0,saved.frames.toInt())- } else pcm.clip(segment.speech.start,minOf(segment.speech.end,segment.speech.start+8.0))+ } else pcm.clip(segment.speech.start,if(wholeSegment) segment.speech.end else minOf(segment.speech.end,segment.speech.start+8.0))val referenceText=request.referenceText ?: segment.referenceText?.takeIf {it.isNotBlank()} ?: run {- val text=if(segment.speech.end-segment.speech.start<=8.0) segment.speech.text- else engine.transcribe(asr,reference).trim().ifBlank {segment.speech.text}+ val recognized=if(duration<=8.0) segment.speech.text else engine.transcribe(asr,reference).trim()+ val text=recognized.ifBlank {+ // Quiet or short clip: use the whole segment with its own transcript, never a clip with someone else's words.+ reference=pcm.clip(segment.speech.start,segment.speech.end)+ segment.speech.text+ }segment=segment.copy(referenceText=text)save(checkpoint.copy(segments=checkpoint.segments.toMutableList().also {it[index]=segment}))textdiff --git a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.ktindex e4d9bbd01..d56990370 100644--- a/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt+++ b/mobile/android/features/src/main/kotlin/com/voicestudio/mobile/features/connect/ConnectScreen.kt@@ -25,7 +25,7 @@ import java.util.DateScreenTitle("mobile.connect.title")Action("mobile.connect.scan_qr",!state.busy,onClick=camera)if(scanner) scanContent({raw ->- try {val address=PairingAddress.parse(raw);url=address.baseUrl.toString();pin=address.pin.orEmpty();scanner=false;manual=true}+ try {val address=PairingAddress.parse(raw);url=address.baseUrl.toString();pin=address.pin.orEmpty();scanner=false;manual=true;vm.connect(url,pin,"")}catch(e:Exception) {scanner=false;vm.fail(e)}},{scanner=false},{scanner=false;vm.fail(it)})Secondary("mobile.simple.address") {manual=!manual}diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swiftindex 2182ed1e1..b2ce50b88 100644--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift@@ -204,18 +204,25 @@ public struct PhoneDubPipeline: Sendable {let rendered = try await store.file(id,"render-\(targetIndex)-\(index).pcm")if let saved = cp.renders.first(where: { $0.target == target.code && $0.index == index }), saved.frames == frames, (try? PCMFile(rendered).frames) == frames, (try? ModelHash.file(rendered)) == saved.hash { continue }try await stage("mobile.android.stage_render",progress)- let reference: PCMChunk+ let savedText = (request.referenceText ?? segment.referenceText)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""+ let segmentText = segment.speech.text.trimmingCharacters(in: .whitespacesAndNewlines)+ // When recognition of the 8 s clip came back blank, the saved text is the segment's own transcript and+ // the reference is the whole segment, so a resumed job pairs the same audio with the same words.+ let wholeSegment = request.referenceText == nil && !savedText.isEmpty && savedText == segmentText && segment.speech.end - segment.speech.start > 8+ var reference: PCMChunkif request.referenceText != nil {let pcm = PCMFile(try await store.file(id,"reference.pcm")); reference = try pcm.read(start: 0,count: pcm.frames)- } else { reference = try original.clip(start: segment.speech.start,end: min(segment.speech.end,segment.speech.start+8)) }- let savedText = (request.referenceText ?? segment.referenceText)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""+ } else { reference = try original.clip(start: segment.speech.start,end: wholeSegment ? segment.speech.end : min(segment.speech.end,segment.speech.start+8)) }let referenceText: Stringif !savedText.isEmpty { referenceText = savedText }else {let recognized = try await engine.transcribe(model: asr,audio: reference,progress: { _ in }).trimmingCharacters(in: .whitespacesAndNewlines)- // Quiet/short clips still have the segment's original ASR transcript.- referenceText = recognized.isEmpty ? segment.speech.text.trimmingCharacters(in: .whitespacesAndNewlines) : recognized- guard !referenceText.isEmpty else { throw ClientError("mobile.simple.no_speech") }+ if recognized.isEmpty {+ guard !segmentText.isEmpty else { throw ClientError("mobile.simple.no_speech") }+ // Quiet or short clip: use the whole segment with its own transcript, never a clip with someone else's words.+ reference = try original.clip(start: segment.speech.start,end: segment.speech.end)+ referenceText = segmentText+ } else { referenceText = recognized }}if segment.referenceText != referenceText {segment.referenceText = referenceText; cp.segments[index] = segment; try await save(cp)diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swiftindex 948dcb7fc..714659d4c 100644--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/PhoneWorkflowTests.swift@@ -75,6 +75,9 @@ private actor FixtureEngine: LocalEngine {private(set) var translations = 0private(set) var clones = 0private var failClone = 0+ private var longSegment = false+ private(set) var references: [(frames: Int, text: String)] = []+ func useLongSegment() { longSegment = true }func failOnClone(_ count: Int) { failClone = count }func empty() { transcript = "" }func synthesize(model: PhoneModel,text: String,speaker: Int,speed: Float,progress: @escaping EngineProgressHandler) async throws -> LocalAudio { LocalAudio(samples: Array(repeating: 0.25,count: 24000),sampleRate: 24000) }@@ -82,11 +85,13 @@ private actor FixtureEngine: LocalEngine {clones += 1if clones == failClone { throw CancellationError() }#expect(reference.sampleRate == 24000); #expect(!referenceText.isEmpty)+ references.append((reference.samples.count, referenceText))return LocalAudio(samples: Array(repeating: 0.4,count: 12000),sampleRate: 24000)}func transcribe(model: PhoneModel,audio: PCMChunk,progress: @escaping EngineProgressHandler) async throws -> String { try Task.checkCancellation(); transcriptions += 1; return transcript }func transcribeSegments(model: PhoneModel,audio: PCMChunk,progress: @escaping EngineProgressHandler) async throws -> [SpeechSegment] {transcriptions += 1+ if longSegment { return [SpeechSegment(start: 0.5,end: 11.5,text: "Hello long")] }return [SpeechSegment(start: 0.1,end: 0.35,text: "Hello"), SpeechSegment(start: 0.5,end: 0.85,text: "World")]}func streamTranscribe(model: PhoneModel,chunks: AsyncThrowingStream<PCMChunk,any Error>,progress: @escaping EngineProgressHandler) async throws -> AsyncThrowingStream<TranscriptUpdate,any Error> { AsyncThrowingStream { $0.finish() } }@@ -295,6 +300,23 @@ func realAuxiliaryPhoneDubOfflineAndResume() async throws {#expect(result.segments.map(\.referenceText) == ["Hello","World"])}+@Test func blankRecognitionOfALongSegmentUsesTheWholeSegmentWithItsOwnText() async throws {+ let root = try fixtureRoot(); defer { try? FileManager.default.removeItem(at: root) }+ let source = root.appendingPathComponent("source.wav"); try Data("fixture".utf8).write(to: source)+ let models = try [fixtureModel(.asr),fixtureModel(.translate),fixtureModel(.clone)]+ let store = try PhoneDubStore(root: root.appendingPathComponent("dubs")), engine = FixtureEngine()+ await engine.empty(); await engine.useLongSegment(); await engine.failOnClone(1)+ let request = try await store.create(source: source,title: "Source",targets: [PhoneDubTarget(code: "fr",name: "French")],models: models,consent: true,audioOnly: true)+ let pipeline = PhoneDubPipeline(store: store,engine: engine,decoder: FixtureDecoder(frames: 24000 * 12),media: FixtureMedia())+ await #expect(throws: CancellationError.self) { try await pipeline.run(id: request.id,models: models) }+ // The resumed job must pair the same audio with the same words as the interrupted one would have.+ #expect(try await pipeline.run(id: request.id,models: models).done)+ let references = await engine.references+ #expect(references.count == 1)+ #expect(references.first?.text == "Hello long")+ #expect(references.first?.frames == 11 * 24000)+}+@Test func reviewCorruptRecordsDoNotHideHealthyLibraries() async throws {let root = try fixtureRoot(); defer { try? FileManager.default.removeItem(at: root) }let voicesRoot = root.appendingPathComponent("voices"), dubsRoot = root.appendingPathComponent("dubs")