Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(ios): start dictation mic after the inference slot, split dub windows at pauses, place PCM by timestamp, serialize dub jobs

debpalash5a575565dab6
diff --git a/mobile/ios/VoiceStudio/Features/DictateView.swift b/mobile/ios/VoiceStudio/Features/DictateView.swift
index 80c5c9995..61bba7b7c 100644
--- a/mobile/ios/VoiceStudio/Features/DictateView.swift
+++ b/mobile/ios/VoiceStudio/Features/DictateView.swift
@@ -145,10 +145,14 @@ struct DictateView: View {
defer { audio.stop(); listening = false; finalizing = false }
do {
guard let phoneModel, let engine = model.phone.engine else { throw ClientError("mobile.local.model_required") }
- let bytes = try await audio.start()
- guard listening, !Task.isCancelled else { audio.stop(); return }
let (chunks, continuation) = AsyncThrowingStream<PCMChunk, any Error>.makeStream(bufferingPolicy: .bufferingOldest(64))
+ // The inference slot and recognizer come first; the bounded capture buffer would overflow while waiting.
let updates = try await engine.streamTranscribe(model: phoneModel, chunks: chunks, progress: { _ in })
+ let bytes: AsyncThrowingStream<Data, any Error>
+ do {
+ bytes = try await audio.start()
+ guard listening, !Task.isCancelled else { audio.stop(); continuation.finish(); return }
+ } catch { continuation.finish(throwing: error); throw error } // releases the slot
receiving = Task {
do {
for try await update in updates {
diff --git a/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c b/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c
index 7a3eaa1e0..15e23407b 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c
+++ b/mobile/ios/VoiceStudioKit/Sources/VSNative/VSNative.c
@@ -103,24 +103,22 @@ static int transcribe_impl(const VSConfig *c, const float *samples, int32_t n, i
if(r && r->text) {
size_t size=strlen(r->text);
if(segments && size) {
- double duration=(double)count/rate, start=0, end=duration, first=duration, last=-1;
- if(r->timestamps) for(int32_t i=0;i<r->count;++i) {
- double t=r->timestamps[i];
- if(isfinite(t) && t>=0 && t<duration) { if(t<first) first=t; if(t>last) last=t; }
- }
- if(last>=0) {
- start=fmax(0,fmin(first,duration-.02));
- double tail=(r->durations && r->count>0 && isfinite(r->durations[r->count-1]) && r->durations[r->count-1]>0) ? r->durations[r->count-1] : .5;
- end=fmin(duration,fmax(start+.01,last+tail));
- }
- VSSegment *next_segments=realloc(segments->segments,(size_t)(segments->count+1)*sizeof(VSSegment));
- if(!next_segments) rc=-3;
+ // Window-relative token timing goes to Swift, which splits the window at pauses.
+ // Without aligned tokens and timestamps the window is one untimed segment (token=0).
+ int aligned=r->timestamps && r->tokens_arr && r->count>0;
+ int32_t emit=aligned ? r->count : 1;
+ VSSegment *grown=realloc(segments->segments,(size_t)(segments->count+emit)*sizeof(VSSegment));
+ if(!grown) rc=-3;
else {
- segments->segments=next_segments;
- VSSegment *segment=&segments->segments[segments->count];
- segment->start=(double)offset/rate+start; segment->end=(double)offset/rate+end;
- segment->text=strdup(r->text);
- if(!segment->text) rc=-3; else segments->count++;
+ segments->segments=grown;
+ for(int32_t i=0;i<emit && rc==0;++i) {
+ VSSegment *segment=&segments->segments[segments->count];
+ segment->window=(int32_t)(offset/((int64_t)rate*20)); segment->token=aligned ? 1 : 0;
+ segment->start=aligned ? r->timestamps[i] : 0;
+ segment->end=aligned && r->durations ? r->timestamps[i]+r->durations[i] : 0;
+ segment->text=strdup(aligned ? r->tokens_arr[i] : r->text);
+ if(!segment->text) rc=-3; else segments->count++;
+ }
}
}
char *next=realloc(combined,length+size+2);
diff --git a/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h b/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h
index d5629d71f..8ef532fdb 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h
+++ b/mobile/ios/VoiceStudioKit/Sources/VSNative/include/VSNative.h
@@ -33,7 +33,8 @@ void vs_text_free(char *text);
int vs_runtime_available(const char *runtime, const char *library);
int vs_synthesize(const VSConfig *config, const char *text, int32_t speaker, float speed, VSCancel *cancel, VSAudio *audio);
int vs_transcribe(const VSConfig *config, const float *samples, int32_t count, int32_t rate, VSCancel *cancel, char **text);
-typedef struct { double start; double end; char *text; } VSSegment;
+/// token=1: one recognised token with window-relative start/end (end<=start when untimed); token=0: a whole window.
+typedef struct { double start; double end; char *text; int32_t window; int32_t token; } VSSegment;
typedef struct { VSSegment *segments; int32_t count; } VSSegments;
void vs_segments_free(VSSegments *segments);
int vs_transcribe_segments(const VSConfig *config, const float *samples, int32_t count, int32_t rate, VSCancel *cancel, VSSegments *segments);
diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift
index 4b6d24948..fdf5434a4 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift
+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/AVDubMedia.swift
@@ -42,8 +42,9 @@ public struct AVPhoneAudioDecoder: PhoneAudioDecoder {
guard rc == kCMBlockBufferNoErr, bytes.count.isMultiple(of: 4) else { throw ClientError("mobile.simple.media_failed") }
let timestamp = CMSampleBufferGetPresentationTimeStamp(buffer).seconds
guard timestamp.isFinite else { throw ClientError("mobile.simple.media_failed") }
- let position = Int((timestamp*24000).rounded())
+ var position = Int((timestamp*24000).rounded())
if position >= totalFrames { continue }
+ position = count+timelineShift(timestamp: timestamp,written: count)
if position > count { try pad(position-count); count = position }
let skip = max(0,count-position)
let frames = min(bytes.count/4-skip,totalFrames-count)
@@ -162,3 +163,11 @@ public struct AVPhoneDubMedia: PhoneDubMedia {
return try await withTaskCancellationHandler { try await work.value } onCancel: { work.cancel() }
}
}
+
+/// Frames to insert (positive) or leading frames to drop (negative) so a buffer stamped `timestamp`
+/// seconds lands at its container time given `written` 24 kHz frames already output. Under 10 ms is jitter.
+func timelineShift(timestamp: Double,written: Int) -> Int {
+ guard timestamp.isFinite, timestamp >= 0 else { return 0 }
+ let delta = Int((timestamp*24000).rounded())-written
+ return abs(delta) < 240 ? 0 : delta
+}
diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift
index 178c22eed..e521a9890 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift
+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/LocalEngine.swift
@@ -54,6 +54,41 @@ public struct SpeechSegment: Codable, Sendable, Equatable {
public let text: String
public init(start: Double,end: Double,text: String) { self.start = start; self.end = end; self.text = text }
}
+/// Pause that starts a new utterance; shorter after sentence-final punctuation.
+let utteranceGap = 1.0, sentenceGap = 0.4
+/// One 20 s recognition window can hold several sentences with long pauses. Dubbing speaks each segment
+/// at its start, so later sentences would play early; split at pauses so each keeps its own timing.
+/// `ends[i] <= timestamps[i]` means token i has no duration. Falls back to one segment when tokens and
+/// timestamps do not line up.
+func splitUtterances(text: String,tokens: [String],timestamps: [Double],ends: [Double],window: Double) -> [SpeechSegment] {
+ let ok = !tokens.isEmpty && tokens.count == timestamps.count && timestamps.allSatisfy { $0.isFinite && $0 >= 0 && $0 < window }
+ let valid = timestamps.filter { $0.isFinite && $0 >= 0 && $0 < window }
+ func tail(_ i: Int) -> Double { i < ends.count && ends.count == timestamps.count && ends[i].isFinite && ends[i] > timestamps[i] ? ends[i]-timestamps[i] : 0.5 }
+ func single() -> [SpeechSegment] {
+ let clean = text.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !clean.isEmpty else { return [] }
+ let first = min(max(valid.min() ?? 0,0),max(0,window-0.02))
+ let end = valid.max().map { min(window,max(first+0.01,$0+(ok ? tail(timestamps.count-1) : 0.5))) } ?? window
+ return [SpeechSegment(start: first,end: end,text: clean)]
+ }
+ guard ok else { return single() }
+ func endOf(_ i: Int) -> Double { min(window,timestamps[i]+tail(i)) }
+ var groups: [[Int]] = [[0]]
+ for i in 1..<max(1,tokens.count) {
+ let gap = timestamps[i]-endOf(i-1)
+ let sentenceEnd = tokens[i-1].trimmingCharacters(in: .whitespaces).last.map { ".!?。!?".contains($0) } ?? false
+ if gap >= utteranceGap || (sentenceEnd && gap >= sentenceGap) { groups.append([i]) } else { groups[groups.count-1].append(i) }
+ }
+ var pieces: [SpeechSegment] = []
+ for (n,group) in groups.enumerated() {
+ let piece = group.map { tokens[$0] }.joined().trimmingCharacters(in: .whitespacesAndNewlines)
+ if piece.isEmpty { continue }
+ let start = timestamps[group[0]]
+ let nextStart = n+1 < groups.count ? timestamps[groups[n+1][0]] : window
+ pieces.append(SpeechSegment(start: start,end: min(window,min(max(endOf(group[group.count-1]),start+0.01),max(start+0.01,nextStart))),text: piece))
+ }
+ return pieces.count <= 1 ? single() : pieces
+}
public protocol LocalEngine: Sendable {
func synthesize(model: PhoneModel, text: String, speaker: Int, speed: Float, progress: @escaping EngineProgressHandler) async throws -> LocalAudio
func clone(model: PhoneModel, text: String, reference: PCMChunk, referenceText: String, progress: @escaping EngineProgressHandler) async throws -> LocalAudio
@@ -209,11 +244,29 @@ public actor NativeLocalEngine: LocalEngine {
var output = VSSegments(); defer { vs_segments_free(&output) }
try audio.samples.withUnsafeBufferPointer { try Self.check(vs_transcribe_segments(&config.value,$0.baseAddress,Int32($0.count),Int32(audio.sampleRate),token.pointer,&output)) }
guard output.count == 0 || output.segments != nil else { throw ClientError("mobile.local.inference_failed") }
- return (0..<Int(output.count)).compactMap { index in
+ let total = Double(audio.samples.count)/Double(audio.sampleRate)
+ var result: [SpeechSegment] = [], window = -1, tokens: [(String,Double,Double)] = []
+ func flush() {
+ guard window >= 0, !tokens.isEmpty else { tokens = []; return }
+ let base = Double(window)*20, length = min(20,max(0,total-base))
+ result += splitUtterances(text: tokens.map(\.0).joined(),tokens: tokens.map(\.0),timestamps: tokens.map(\.1),ends: tokens.map(\.2),window: length)
+ .map { SpeechSegment(start: base+$0.start,end: base+$0.end,text: $0.text) }
+ tokens = []
+ }
+ for index in 0..<Int(output.count) {
let segment = output.segments![index]
- let text = segment.text.map { String(cString: $0).trimmingCharacters(in: .whitespacesAndNewlines) } ?? ""
- return text.isEmpty ? nil : SpeechSegment(start: segment.start,end: segment.end,text: text)
+ let text = segment.text.map { String(cString: $0) } ?? ""
+ if segment.token == 0 {
+ flush()
+ let clean = text.trimmingCharacters(in: .whitespacesAndNewlines)
+ if !clean.isEmpty { result.append(SpeechSegment(start: Double(segment.window)*20,end: min(total,Double(segment.window)*20+20),text: clean)) }
+ continue
+ }
+ if Int(segment.window) != window { flush(); window = Int(segment.window) }
+ tokens.append((text,segment.start,segment.end))
}
+ flush()
+ return result
}
}
public func clone(model: PhoneModel, text: String, reference: PCMChunk, referenceText: String, progress: @escaping EngineProgressHandler = { _ in }) async throws -> LocalAudio {
@@ -246,10 +299,13 @@ public actor NativeLocalEngine: LocalEngine {
let library = libraries[model.runtime] ?? ""
let store = store
let (output, continuation) = AsyncThrowingStream<TranscriptUpdate, any Error>.makeStream(bufferingPolicy: .bufferingNewest(32))
+ // Signals once the worker holds the inference slot and the recognizer exists, so the
+ // caller starts the microphone only then (the capture buffer is bounded).
+ let (ready, readiness) = AsyncThrowingStream<Void, any Error>.makeStream(bufferingPolicy: .bufferingNewest(1))
let work = Task.detached(priority: .userInitiated) {
do {
try await NativeLease.shared.acquire()
- } catch { continuation.finish(throwing: error); await store.release(model); return }
+ } catch { readiness.finish(throwing: error); continuation.finish(throwing: error); await store.release(model); return }
do {
let root = try await store.authorize(model)
try Task.checkCancellation()
@@ -257,6 +313,7 @@ public actor NativeLocalEngine: LocalEngine {
let config = NativeConfiguration(model: model, root: root, library: library)
guard let stream = vs_stream_create(&config.value) else { throw ClientError("mobile.local.inference_failed") }
defer { vs_stream_free(stream) }
+ readiness.yield(); readiness.finish()
var finals: [String] = []
func accept(_ chunk: PCMChunk, finished: Bool) throws {
try Task.checkCancellation()
@@ -281,6 +338,7 @@ public actor NativeLocalEngine: LocalEngine {
continuation.finish()
} catch {
if !(error is CancellationError) { await store.invalidateVerification(model) }
+ readiness.finish(throwing: error) // no-op once ready was signalled
continuation.finish(throwing: error)
}
// The stream and recognizer have been freed by the inner scope's defer.
@@ -288,6 +346,13 @@ public actor NativeLocalEngine: LocalEngine {
await store.release(model)
}
continuation.onTermination = { _ in token.cancel(); work.cancel() }
+ do {
+ try await withTaskCancellationHandler {
+ for try await _ in ready { return }
+ try Task.checkCancellation()
+ throw ClientError("mobile.local.inference_failed")
+ } onCancel: { token.cancel(); work.cancel() }
+ } catch { token.cancel(); work.cancel(); throw error }
return output
}
private static func validate(_ pcm: PCMChunk) throws {
diff --git a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
index b2ce50b88..269ed6fca 100644
--- a/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
+++ b/mobile/ios/VoiceStudioKit/Sources/VoiceStudioKit/Local/PhoneDub.swift
@@ -70,11 +70,16 @@ public actor PhoneDubStore {
public init(root: URL) throws { self.root = root; try PrivateFiles.directory(root); try PrivateFiles.removeUnpublished(in: root,metadata: "request.json") }
public func directory(_ id: UUID) throws -> URL { try PrivateFiles.child(root,id.uuidString) }
public func file(_ id: UUID,_ name: String) throws -> URL { try PrivateFiles.child(directory(id),name) }
- public func acquire(_ id: UUID) throws {
- guard !active.contains(id) else { throw ClientError("sidebarTools.wait") }
- active.insert(id)
+ private var waiters: [UUID: [CheckedContinuation<Void,Never>]] = [:]
+ /// One lock per job. A resumed run or a delete waits for the previous run (including its cancel
+ /// cleanup and final checkpoint save) instead of racing it; the lock is handed straight to the next waiter.
+ public func acquire(_ id: UUID) async throws {
+ if active.contains(id) { await withCheckedContinuation { waiters[id,default: []].append($0) } } else { active.insert(id) }
+ if Task.isCancelled { release(id); throw CancellationError() }
+ }
+ public func release(_ id: UUID) {
+ if var queue = waiters[id], !queue.isEmpty { let next = queue.removeFirst(); waiters[id] = queue.isEmpty ? nil : queue; next.resume() } else { active.remove(id) }
}
- public func release(_ id: UUID) { active.remove(id) }
public func create(source: URL,title: String,targets: [PhoneDubTarget],models: [PhoneModel],consent: Bool,reference: PCMChunk? = nil,referenceText: String? = nil,audioOnly: Bool = false) throws -> PhoneDubRequest {
guard consent, !targets.isEmpty, targets.count <= 20, Set(targets.map(\.code)).count == targets.count,
targets.allSatisfy({ !$0.code.isEmpty && $0.code.count <= 80 && $0.code.utf8.allSatisfy({ (48...57).contains($0) || (65...90).contains($0) || (97...122).contains($0) || [45,95].contains($0) }) && (2...120).contains($0.name.count) && !$0.name.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }) }),
@@ -125,8 +130,8 @@ public actor PhoneDubStore {
guard try ModelHash.file(url) == output.hash else { throw ClientError("mobile.simple.media_failed") }
return url
}
- public func delete(_ id: UUID) throws {
- guard !active.contains(id) else { throw ClientError("sidebarTools.wait") }
+ public func delete(_ id: UUID) async throws {
+ try await acquire(id); defer { release(id) }
try FileManager.default.removeItem(at: directory(id))
}
}
diff --git a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
index fc6504c4e..9f2ecd941 100644
--- a/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
+++ b/mobile/ios/VoiceStudioKit/Tests/VoiceStudioKitTests/LocalInferenceTests.swift
@@ -2,6 +2,7 @@ import Foundation
import CryptoKit
import Darwin
import Testing
+import VSNative
@testable import VoiceStudioKit
private func localDirectory() throws -> URL {
@@ -513,3 +514,72 @@ extension NativeLeaseRegressionTests {
await #expect(throws: ClientError("mobile.local.checksum_failed")) { try await queued.value }
}
}
+
+@Test func pausesSplitOneWindowIntoTimedUtterances() {
+ let tokens = [" Hello"," there","."," Much"," later","."]
+ let ts: [Double] = [0.5,0.9,1.2,9.0,9.4,9.8]
+ let ends: [Double] = [0.8,1.2,1.4,9.3,9.7,10.0]
+ let parts = splitUtterances(text: "Hello there. Much later.",tokens: tokens,timestamps: ts,ends: ends,window: 20)
+ #expect(parts.map(\.text) == ["Hello there.","Much later."])
+ #expect(parts[0].start == 0.5 && parts[1].start == 9.0)
+ #expect(parts[0].end <= 9.0 && parts[1].end <= 20 && parts[1].end > parts[1].start)
+ // 0.4 s after sentence-final punctuation is enough; mid-sentence needs 1.0 s.
+ #expect(splitUtterances(text: "a. b",tokens: ["a","."," b"],timestamps: [0,0.3,1.0],ends: [0.3,0.5,1.3],window: 20).count == 2)
+ #expect(splitUtterances(text: "a b c",tokens: ["a"," b"," c"],timestamps: [0,0.4,1.2],ends: [0.3,0.7,1.5],window: 20).count == 1)
+}
+@Test func continuousSpeechAndMismatchedTimestampsStayOneSegment() {
+ #expect(splitUtterances(text: "a b c",tokens: ["a"," b"," c"],timestamps: [0,0.4,0.8],ends: [],window: 20).count == 1)
+ let bad = splitUtterances(text: "a b",tokens: ["a"," b"],timestamps: [0],ends: [],window: 20)
+ #expect(bad.count == 1 && bad[0].text == "a b")
+}
+@Test func rawPCMIsPlacedByTimestampWithJitterTolerance() {
+ #expect(timelineShift(timestamp: 1,written: 0) == 24000)
+ #expect(timelineShift(timestamp: 1.5,written: 12000) == 24000)
+ #expect(timelineShift(timestamp: 0.9,written: 24000) == -2400)
+ #expect(timelineShift(timestamp: 1.004,written: 24000) == 0)
+ #expect(timelineShift(timestamp: .nan,written: 500) == 0)
+}
+@Test func dubJobLockMakesResumeAndDeleteWaitForTheOldRun() async throws {
+ let root = try localDirectory(); defer { try? FileManager.default.removeItem(at: root) }
+ let store = try PhoneDubStore(root: root.appendingPathComponent("dubs")), id = UUID()
+ try FileManager.default.createDirectory(at: try await store.directory(id),withIntermediateDirectories: true)
+ try await store.acquire(id)
+ let finished = Flag()
+ let deleting = Task { try await store.delete(id); finished.set() }
+ try await Task.sleep(for: .milliseconds(150))
+ #expect(!finished.value)
+ #expect(FileManager.default.fileExists(atPath: try await store.directory(id).path))
+ await store.release(id)
+ try await deleting.value
+ #expect(!FileManager.default.fileExists(atPath: try await store.directory(id).path))
+ // A different job is never blocked.
+ try await store.acquire(UUID())
+}
+@Test func dictationMicrophoneWaitsForInferenceSlot() async throws {
+ let root = try localDirectory(); defer { try? FileManager.default.removeItem(at: root) }
+ let model = try localFixture(), stream = try localFixture(capability: "asr-stream")
+ let store = try ModelStore(root: root, available: { _ in 100_000_000 })
+ try await store.download(model, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))
+ try await store.download(stream, wifi: { true }, transport: RangeServer(bytes: Data("model".utf8)))
+ let engine = NativeLocalEngine(store: store), gate = NativeWorkGate()
+ defer { gate.finish.signal() }
+ let holder = Task { try await engine.execute(model, capability: .tts, progress: { _ in }) { _, _, _, _ in gate.run(); return 1 } }
+ let deadline = ContinuousClock.now + .seconds(10)
+ while gate.count == 0 && ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(5)) }
+ try #require(gate.count == 1)
+ let (chunks, feed) = AsyncThrowingStream<PCMChunk, any Error>.makeStream()
+ defer { feed.finish() }
+ let returned = Flag()
+ let waiting = Task { _ = try? await engine.streamTranscribe(model: stream, chunks: chunks); returned.set() }
+ try await Task.sleep(for: .milliseconds(200))
+ #expect(!returned.value, "streamTranscribe must not return (so the mic must not start) while the slot is held")
+ gate.finish.signal()
+ _ = try await holder.value
+ await waiting.value
+ #expect(returned.value)
+}
+private final class Flag: @unchecked Sendable {
+ private let lock = NSLock(); private var flag = false
+ func set() { lock.withLock { flag = true } }
+ var value: Bool { lock.withLock { flag } }
+}