VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(ios): start dictation mic after the inference slot, split dub windows at pauses, place PCM by timestamp, serialize dub jobs
debpalash5a575565dab6
diff --git a/mobile/ios/VoiceStudio/Features/DictateView.swift b/mobile/ios/VoiceStudio/Features/DictateView.swiftindex 80c5c9995..61bba7b7c 100644--- a/mobile/ios/VoiceStudio/Features/DictateView.swift+++ b/mobile/ios/VoiceStudio/Features/DictateView.swift@@ -145,10 +145,14 @@ struct DictateView: View {defer { audio.stop(); listening = false; finalizing = false }do {guard let phoneModel, let engine = model.phone.engine else { throw ClientError("mobile.local.model_required") }- let bytes = try await audio.start()- guard listening, !Task.isCancelled else { audio.stop(); return }let (chunks, continuation) = AsyncThrowingStream<PCMChunk, any Error>.makeStream(bufferingPolicy: .bufferingOldest(64))+ // The inference slot and recognizer come first; the bounded capture buffer would overflow while waiting.let updates = try await engine.streamTranscribe(model: phoneModel, chunks: chunks, progress: { _ in })+ let bytes: AsyncThrowingStream<Data, any Error>+ do {+ bytes = try await audio.start()+ guard listening, !Task.isCancelled else { audio.stop(); continuation.finish(); return }+ } catch { continuation.finish(throwing: error); throw error } // releases the slotreceiving = Task {do {for try await update in updates {diff --git a/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c b/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.cindex 7a3eaa1e0..15e23407b 100644--- a/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c+++ b/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c@@ -103,24 +103,22 @@ static int transcribe_impl(const VSConfig *c, const float *samples, int32_t n, iif(r && r->text) {size_t size=strlen(r->text);if(segments && size) {- double duration=(double)count/rate, start=0, end=duration, first=duration, last=-1;- if(r->timestamps) for(int32_t i=0;i<r->count;++i) {- double t=r->timestamps[i];- if(isfinite(t) && t>=0 && t<duration) { if(t<first) first=t; if(t>last) last=t; }- }- if(last>=0) {- start=fmax(0,fmin(first,duration-.02));- double tail=(r->durations && r->count>0 && isfinite(r->durations[r->count-1]) && r->durations[r->count-1]>0) ? r->durations[r->count-1] : .5;- end=fmin(duration,fmax(start+.01,last+tail));- }- VSSegment *next_segments=realloc(segments->segments,(size_t)(segments->count+1)*sizeof(VSSegment));- if(!next_segments) rc=-3;+ // Window-relative token timing goes to Swift, which splits the window at pauses.+ // Without aligned tokens and timestamps the window is one untimed segment (token=0).+ int aligned=r->timestamps && r->tokens_arr && r->count>0;+ int32_t emit=aligned ? r->count : 1;+ VSSegment *grown=realloc(segments->segments,(size_t)(segments->count+emit)*sizeof(VSSegment));+ if(!grown) rc=-3;else {- segments->segments=next_segments;- VSSegment *segment=&segments->segments[segments->count];- segment->start=(double)offset/rate+start; segment->end=(double)offset/rate+end;- segment->text=strdup(r->text);- if(!segment->text) rc=-3; else segments->count++;+ segments->segments=grown;+ for(int32_t i=0;i<emit && rc==0;++i) {+ VSSegment *segment=&segments->segments[segments->count];+ segment->window=(int32_t)(offset/((int64_t)rate*20)); segment->token=aligned ? 1 : 0;+ segment->start=aligned ? r->timestamps[i] : 0;+ segment->end=aligned && r->durations ? r->timestamps[i]+r->durations[i] : 0;+ segment->text=strdup(aligned ? r->tokens_arr[i] : r->text);+ if(!segment->text) rc=-3; else segments->count++;+ }}}char *next=realloc(combined,length+size+2);diff --git a/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h b/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.hindex d5629d71f..8ef532fdb 100644--- a/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h+++ b/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h@@ -33,7 +33,8 @@ void vs_text_free(char *text);int vs_runtime_available(const char *runtime, const char *library);int vs_synthesize(const VSConfig *config, const char *text, int32_t speaker, float speed, VSCancel *cancel, VSAudio *audio);int vs_transcribe(const VSConfig *config, const float *samples, int32_t count, int32_t rate, VSCancel *cancel, char **text);-typedef struct { double start; double end; char *text; } VSSegment;+/// token=1: one recognised token with window-relative start/end (end<=start when untimed); token=0: a whole window.+typedef struct { double start; double end; char *text; int32_t window; int32_t token; } VSSegment;typedef struct { VSSegment *segments; int32_t count; } VSSegments;void vs_segments_free(VSSegments *segments);int vs_transcribe_segments(const VSConfig *config, const float *samples, int32_t count, int32_t rate, VSCancel *cancel, VSSegments *segments);diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swiftindex 4b6d24948..fdf5434a4 100644--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift@@ -42,8 +42,9 @@ public struct AVPhoneAudioDecoder: PhoneAudioDecoder {guard rc == kCMBlockBufferNoErr, bytes.count.isMultiple(of: 4) else { throw ClientError("mobile.simple.media_failed") }let timestamp = CMSampleBufferGetPresentationTimeStamp(buffer).secondsguard timestamp.isFinite else { throw ClientError("mobile.simple.media_failed") }- let position = Int((timestamp*24000).rounded())+ var position = Int((timestamp*24000).rounded())if position >= totalFrames { continue }+ position = count+timelineShift(timestamp: timestamp,written: count)if position > count { try pad(position-count); count = position }let skip = max(0,count-position)let frames = min(bytes.count/4-skip,totalFrames-count)@@ -162,3 +163,11 @@ public struct AVPhoneDubMedia: PhoneDubMedia {return try await withTaskCancellationHandler { try await work.value } onCancel: { work.cancel() }}}++/// Frames to insert (positive) or leading frames to drop (negative) so a buffer stamped `timestamp`+/// seconds lands at its container time given `written` 24 kHz frames already output. Under 10 ms is jitter.+func timelineShift(timestamp: Double,written: Int) -> Int {+ guard timestamp.isFinite, timestamp >= 0 else { return 0 }+ let delta = Int((timestamp*24000).rounded())-written+ return abs(delta) < 240 ? 0 : delta+}diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swiftindex 178c22eed..e521a9890 100644--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift@@ -54,6 +54,41 @@ public struct SpeechSegment: Codable, Sendable, Equatable {public let text: Stringpublic init(start: Double,end: Double,text: String) { self.start = start; self.end = end; self.text = text }}+/// Pause that starts a new utterance; shorter after sentence-final punctuation.+let utteranceGap = 1.0, sentenceGap = 0.4+/// One 20 s recognition window can hold several sentences with long pauses. Dubbing speaks each segment+/// at its start, so later sentences would play early; split at pauses so each keeps its own timing.+/// `ends[i] <= timestamps[i]` means token i has no duration. Falls back to one segment when tokens and+/// timestamps do not line up.+func splitUtterances(text: String,tokens: [String],timestamps: [Double],ends: [Double],window: Double) -> [SpeechSegment] {+ let ok = !tokens.isEmpty && tokens.count == timestamps.count && timestamps.allSatisfy { $0.isFinite && $0 >= 0 && $0 < window }+ let valid = timestamps.filter { $0.isFinite && $0 >= 0 && $0 < window }+ func tail(_ i: Int) -> Double { i < ends.count && ends.count == timestamps.count && ends[i].isFinite && ends[i] > timestamps[i] ? ends[i]-timestamps[i] : 0.5 }+ func single() -> [SpeechSegment] {+ let clean = text.trimmingCharacters(in: .whitespacesAndNewlines)+ guard !clean.isEmpty else { return [] }+ let first = min(max(valid.min() ?? 0,0),max(0,window-0.02))+ let end = valid.max().map { min(window,max(first+0.01,$0+(ok ? tail(timestamps.count-1) : 0.5))) } ?? window+ return [SpeechSegment(start: first,end: end,text: clean)]+ }+ guard ok else { return single() }+ func endOf(_ i: Int) -> Double { min(window,timestamps[i]+tail(i)) }+ var groups: [[Int]] = [[0]]+ for i in 1..<max(1,tokens.count) {+ let gap = timestamps[i]-endOf(i-1)+ let sentenceEnd = tokens[i-1].trimmingCharacters(in: .whitespaces).last.map { ".!?。!?".contains($0) } ?? false+ if gap >= utteranceGap || (sentenceEnd && gap >= sentenceGap) { groups.append([i]) } else { groups[groups.count-1].append(i) }+ }+ var pieces: [SpeechSegment] = []+ for (n,group) in groups.enumerated() {+ let piece = group.map { tokens[$0] }.joined().trimmingCharacters(in: .whitespacesAndNewlines)+ if piece.isEmpty { continue }+ let start = timestamps[group[0]]+ let nextStart = n+1 < groups.count ? timestamps[groups[n+1][0]] : window+ pieces.append(SpeechSegment(start: start,end: min(window,min(max(endOf(group[group.count-1]),start+0.01),max(start+0.01,nextStart))),text: piece))+ }+ return pieces.count <= 1 ? single() : pieces+}public protocol LocalEngine: Sendable {func synthesize(model: PhoneModel, text: String, speaker: Int, speed: Float, progress: @escaping EngineProgressHandler) async throws -> LocalAudiofunc clone(model: PhoneModel, text: String, reference: PCMChunk, referenceText: String, progress: @escaping EngineProgressHandler) async throws -> LocalAudio@@ -209,11 +244,29 @@ public actor NativeLocalEngine: LocalEngine {var output = VSSegments(); defer { vs_segments_free(&output) }try audio.samples.withUnsafeBufferPointer { try Self.check(vs_transcribe_segments(&config.value,$0.baseAddress,Int32($0.count),Int32(audio.sampleRate),token.pointer,&output)) }guard output.count == 0 || output.segments != nil else { throw ClientError("mobile.local.inference_failed") }- return (0..<Int(output.count)).compactMap { index in+ let total = Double(audio.samples.count)/Double(audio.sampleRate)+ var result: [SpeechSegment] = [], window = -1, tokens: [(String,Double,Double)] = []+ func flush() {+ guard window >= 0, !tokens.isEmpty else { tokens = []; return }+ let base = Double(window)*20, length = min(20,max(0,total-base))+ result += splitUtterances(text: tokens.map(\.0).joined(),tokens: tokens.map(\.0),timestamps: tokens.map(\.1),ends: tokens.map(\.2),window: length)+ .map { SpeechSegment(start: base+$0.start,end: base+$0.end,text: $0.text) }+ tokens = []+ }+ for index in 0..<Int(output.count) {let segment = output.segments![index]- let text = segment.text.map { String(cString: $0).trimmingCharacters(in: .whitespacesAndNewlines) } ?? ""- return text.isEmpty ? nil : SpeechSegment(start: segment.start,end: segment.end,text: text)+ let text = segment.text.map { String(cString: $0) } ?? ""+ if segment.token == 0 {+ flush()+ let clean = text.trimmingCharacters(in: .whitespacesAndNewlines)+ if !clean.isEmpty { result.append(SpeechSegment(start: Double(segment.window)*20,end: min(total,Double(segment.window)*20+20),text: clean)) }+ continue+ }+ if Int(segment.window) != window { flush(); window = Int(segment.window) }+ tokens.append((text,segment.start,segment.end))}+ flush()+ return result}}public func clone(model: PhoneModel, text: String, reference: PCMChunk, referenceText: String, progress: @escaping EngineProgressHandler = { _ in }) async throws -> LocalAudio {@@ -246,10 +299,13 @@ public actor NativeLocalEngine: LocalEngine {let library = libraries[model.runtime] ?? ""let store = storelet (output, continuation) = AsyncThrowingStream<TranscriptUpdate, any Error>.makeStream(bufferingPolicy: .bufferingNewest(32))+ // Signals once the worker holds the inference slot and the recognizer exists, so the+ // caller starts the microphone only then (the capture buffer is bounded).+ let (ready, readiness) = AsyncThrowingStream<Void, any Error>.makeStream(bufferingPolicy: .bufferingNewest(1))let work = Task.detached(priority: .userInitiated) {do {try await NativeLease.shared.acquire()- } catch { continuation.finish(throwing: error); await store.release(model); return }+ } catch { readiness.finish(throwing: error); continuation.finish(throwing: error); await store.release(model); return }do {let root = try await store.authorize(model)try Task.checkCancellation()@@ -257,6 +313,7 @@ public actor NativeLocalEngine: LocalEngine {let config = NativeConfiguration(model: model, root: root, library: library)guard let stream = vs_stream_create(&config.value) else { throw ClientError("mobile.local.inference_failed") }defer { vs_stream_free(stream) }+ readiness.yield(); readiness.finish()var finals: [String] = []func accept(_ chunk: PCMChunk, finished: Bool) throws {try Task.checkCancellation()@@ -281,6 +338,7 @@ public actor NativeLocalEngine: LocalEngine {continuation.finish()} catch {if !(error is CancellationError) { await store.invalidateVerification(model) }+ readiness.finish(throwing: error) // no-op once ready was signalledcontinuation.finish(throwing: error)}// The stream and recognizer have been freed by the inner scope's defer.@@ -288,6 +346,13 @@ public actor NativeLocalEngine: LocalEngine {await store.release(model)}continuation.onTermination = { _ in token.cancel(); work.cancel() }+ do {+ try await withTaskCancellationHandler {+ for try await _ in ready { return }+ try Task.checkCancellation()+ throw ClientError("mobile.local.inference_failed")+ } onCancel: { token.cancel(); work.cancel() }+ } catch { token.cancel(); work.cancel(); throw error }return output}private static func validate(_ pcm: PCMChunk) throws {diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swiftindex b2ce50b88..269ed6fca 100644--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift@@ -70,11 +70,16 @@ public actor PhoneDubStore {public init(root: URL) throws { self.root = root; try PrivateFiles.directory(root); try PrivateFiles.removeUnpublished(in: root,metadata: "request.json") }public func directory(_ id: UUID) throws -> URL { try PrivateFiles.child(root,id.uuidString) }public func file(_ id: UUID,_ name: String) throws -> URL { try PrivateFiles.child(directory(id),name) }- public func acquire(_ id: UUID) throws {- guard !active.contains(id) else { throw ClientError("sidebarTools.wait") }- active.insert(id)+ private var waiters: [UUID: [CheckedContinuation<Void,Never>]] = [:]+ /// One lock per job. A resumed run or a delete waits for the previous run (including its cancel+ /// cleanup and final checkpoint save) instead of racing it; the lock is handed straight to the next waiter.+ public func acquire(_ id: UUID) async throws {+ if active.contains(id) { await withCheckedContinuation { waiters[id,default: []].append($0) } } else { active.insert(id) }+ if Task.isCancelled { release(id); throw CancellationError() }+ }+ public func release(_ id: UUID) {+ if var queue = waiters[id], !queue.isEmpty { let next = queue.removeFirst(); waiters[id] = queue.isEmpty ? nil : queue; next.resume() } else { active.remove(id) }}- public func release(_ id: UUID) { active.remove(id) }public func create(source: URL,title: String,targets: [PhoneDubTarget],models: [PhoneModel],consent: Bool,reference: PCMChunk? = nil,referenceText: String? = nil,audioOnly: Bool = false) throws -> PhoneDubRequest {guard consent, !targets.isEmpty, targets.count <= 20, Set(targets.map(\.code)).count == targets.count,targets.allSatisfy({ !$0.code.isEmpty && $0.code.count <= 80 && $0.code.utf8.allSatisfy({ (48...57).contains($0) || (65...90).contains($0) || (97...122).contains($0) || [45,95].contains($0) }) && (2...120).contains($0.name.count) && !$0.name.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }) }),@@ -125,8 +130,8 @@ public actor PhoneDubStore {guard try ModelHash.file(url) == output.hash else { throw ClientError("mobile.simple.media_failed") }return url}- public func delete(_ id: UUID) throws {- guard !active.contains(id) else { throw ClientError("sidebarTools.wait") }+ public func delete(_ id: UUID) async throws {+ try await acquire(id); defer { release(id) }try FileManager.default.removeItem(at: directory(id))}}diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swiftindex fc6504c4e..9f2ecd941 100644--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift@@ -2,6 +2,7 @@ import Foundationimport CryptoKitimport Darwinimport Testing+import VSNative@testable import VoiceStudioKitprivate func localDirectory() throws -> URL {@@ -513,3 +514,72 @@ extension NativeLeaseRegressionTests {await #expect(throws: ClientError("mobile.local.checksum_failed")) { try await queued.value }}}++@Test func pausesSplitOneWindowIntoTimedUtterances() {+ let tokens = [" Hello"," there","."," Much"," later","."]+ let ts: [Double] = [0.5,0.9,1.2,9.0,9.4,9.8]+ let ends: [Double] = [0.8,1.2,1.4,9.3,9.7,10.0]+ let parts = splitUtterances(text: "Hello there. Much later.",tokens: tokens,timestamps: ts,ends: ends,window: 20)+ #expect(parts.map(\.text) == ["Hello there.","Much later."])+ #expect(parts[0].start == 0.5 && parts[1].start == 9.0)+ #expect(parts[0].end <= 9.0 && parts[1].end <= 20 && parts[1].end > parts[1].start)+ // 0.4 s after sentence-final punctuation is enough; mid-sentence needs 1.0 s.+ #expect(splitUtterances(text: "a. b",tokens: ["a","."," b"],timestamps: [0,0.3,1.0],ends: [0.3,0.5,1.3],window: 20).count == 2)+ #expect(splitUtterances(text: "a b c",tokens: ["a"," b"," c"],timestamps: [0,0.4,1.2],ends: [0.3,0.7,1.5],window: 20).count == 1)+}+@Test func continuousSpeechAndMismatchedTimestampsStayOneSegment() {+ #expect(splitUtterances(text: "a b c",tokens: ["a"," b"," c"],timestamps: [0,0.4,0.8],ends: [],window: 20).count == 1)+ let bad = splitUtterances(text: "a b",tokens: ["a"," b"],timestamps: [0],ends: [],window: 20)+ #expect(bad.count == 1 && bad[0].text == "a b")+}+@Test func rawPCMIsPlacedByTimestampWithJitterTolerance() {+ #expect(timelineShift(timestamp: 1,written: 0) == 24000)+ #expect(timelineShift(timestamp: 1.5,written: 12000) == 24000)+ #expect(timelineShift(timestamp: 0.9,written: 24000) == -2400)+ #expect(timelineShift(timestamp: 1.004,written: 24000) == 0)+ #expect(timelineShift(timestamp: .nan,written: 500) == 0)+}+@Test func dubJobLockMakesResumeAndDeleteWaitForTheOldRun() async throws {+ let root = try localDirectory(); defer { try? FileManager.default.removeItem(at: root) }+ let store = try PhoneDubStore(root: root.appendingPathComponent("dubs")), id = UUID()+ try FileManager.default.createDirectory(at: try await store.directory(id),withIntermediateDirectories: true)+ try await store.acquire(id)+ let finished = Flag()+ let deleting = Task { try await store.delete(id); finished.set() }+ try await Task.sleep(for: .milliseconds(150))+ #expect(!finished.value)+ #expect(FileManager.default.fileExists(atPath: try await store.directory(id).path))+ await store.release(id)+ try await deleting.value+ #expect(!FileManager.default.fileExists(atPath: try await store.directory(id).path))+ // A different job is never blocked.+ try await store.acquire(UUID())+}+@Test func dictationMicrophoneWaitsForInferenceSlot() async throws {+ let root = try localDirectory(); defer { try? FileManager.default.removeItem(at: root) }+ let model = try localFixture(), stream = try localFixture(capability: "asr-stream")+ let store = try ModelStore(root: root, available: { _ in 100_000_000 })+ try await store.download(model, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))+ try await store.download(stream, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))+ let engine = NativeLocalEngine(store: store), gate = NativeWorkGate()+ defer { gate.finish.signal() }+ let holder = Task { try await engine.execute(model, capability: .tts, progress: { _ in }) { _, _, _, _ in gate.run(); return 1 } }+ let deadline = ContinuousClock.now + .seconds(10)+ while gate.count == 0 && ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(5)) }+ try #require(gate.count == 1)+ let (chunks, feed) = AsyncThrowingStream<PCMChunk, any Error>.makeStream()+ defer { feed.finish() }+ let returned = Flag()+ let waiting = Task { _ = try? await engine.streamTranscribe(model: stream, chunks: chunks); returned.set() }+ try await Task.sleep(for: .milliseconds(200))+ #expect(!returned.value, "streamTranscribe must not return (so the mic must not start) while the slot is held")+ gate.finish.signal()+ _ = try await holder.value+ await waiting.value+ #expect(returned.value)+}+private final class Flag: @unchecked Sendable {+ private let lock = NSLock(); private var flag = false+ func set() { lock.withLock { flag = true } }+ var value: Bool { lock.withLock { flag } }+}