Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(audio): don't read a blocked sample queue as silent sources on macOS

EtienneLescotdaf23f7a5fb9
diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift
index 18895cb66..85331e7c7 100644
--- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift
+++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift
@@ -86,6 +86,10 @@ public final class AudioTrackMixer {
/// jitter never trips it, and the file is written offline, so the latency it costs at a
/// transition into silence costs the recording nothing.
static let emissionGraceFrames = sampleRate / 4
+ /// A gap between two passes of the sample queue longer than this means the queue itself
+ /// was blocked, not that the sources went quiet. The ticker fires every 10 ms, so ten
+ /// times that is never ordinary scheduling jitter. See `discountQueueStall(until:)`.
+ static let queueStallFrames = sampleRate / 10
/// Longest hole a source may silence-pad across. Past this the source has been dead long
/// enough that padding would materialize however long the outage lasted — the cursor
/// carries the track across the gap instead.
@@ -121,6 +125,8 @@ public final class AudioTrackMixer {
private var deliveredThroughFrame = [Int64?](repeating: nil, count: Source.allCases.count)
private var undeliveredFrames = [Int64](repeating: 0, count: Source.allCases.count)
private var longestHoleFrames = [Int64](repeating: 0, count: Source.allCases.count)
+ /// Where the clock stood at the previous pass of the sample queue, in absolute frames.
+ private var lastPassFrame: Int64?
private var pending: [CMSampleBuffer] = []
private var didWarnAboutBacklog = false
private var didWarnAboutDecode: Set<Int> = []
@@ -178,6 +184,7 @@ public final class AudioTrackMixer {
}
let now = clock()
+ discountQueueStall(until: now)
let startFrame = frameIndex(of: presentationTime, from: anchor)
noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount))
sources[source.rawValue].ingest(frames, atFrame: startFrame)
@@ -194,7 +201,33 @@ public final class AudioTrackMixer {
guard anchor != nil else {
return
}
- drain(upTo: clock(), grace: Int64(MixFormat.emissionGraceFrames))
+ let now = clock()
+ discountQueueStall(until: now)
+ drain(upTo: now, grace: Int64(MixFormat.emissionGraceFrames))
+ }
+
+ /// Keeps a blocked sample queue from reading as silent sources (#1117).
+ ///
+ /// The grace is measured on the clock, but the clock keeps running while the queue cannot:
+ /// a writer or a busy queue that holds it past `emissionGraceFrames` would otherwise make
+ /// the first pass after it — a tick, or the first of the buffers waiting behind it — see
+ /// every source as quiet for the whole stall, emit silence up to now, and then drop the
+ /// audio already queued for that span as `droppedFrames`. The sources were delivering the
+ /// whole time; it is the mixer that was not listening. So a pass that comes much later than
+ /// the ticker allows moves each source's last delivery forward by the gap, which measures
+ /// the grace against the time the queue was actually running.
+ private func discountQueueStall(until now: CMTime) {
+ guard let anchor, now.isValid, now.isNumeric else {
+ return
+ }
+ let nowFrame = frameIndex(of: now, from: anchor)
+ if let lastPassFrame, nowFrame - lastPassFrame > Int64(MixFormat.queueStallFrames) {
+ let gap = nowFrame - lastPassFrame
+ for index in sources.indices {
+ sources[index].lastDeliveryFrame += gap
+ }
+ }
+ lastPassFrame = nowFrame
}
/// Carries the track out to the end of the take and writes out everything still buffered.
diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift
index baf8a7c9c..9a2a686ad 100644
--- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift
+++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift
@@ -237,8 +237,7 @@ final class AudioTrackMixerTests: XCTestCase {
makeSourceBuffer(burst(seconds: 0.2, left: 0, right: 0.5), at: clock.now, nonInterleaved: false),
from: .microphone
)
- clock.advance(seconds: 0.2)
- mixer.tick()
+ runTicker(mixer, clock, seconds: 0.2)
for _ in 0..<10 {
// Each buffer covers the 200 ms about to elapse, the way a live source delivers.
@@ -246,8 +245,7 @@ final class AudioTrackMixerTests: XCTestCase {
makeSourceBuffer(burst(seconds: 0.2, left: 0.5, right: 0), at: clock.now, nonInterleaved: true),
from: .system
)
- clock.advance(seconds: 0.2)
- mixer.tick()
+ runTicker(mixer, clock, seconds: 0.2)
}
clock.advance(seconds: 0.2)
mixer.finish(atSourceTime: clock.now)
@@ -370,9 +368,9 @@ final class AudioTrackMixerTests: XCTestCase {
makeSourceBuffer(burst(seconds: 0.5, left: 0.5, right: 0.5), at: .zero, nonInterleaved: true),
from: .system
)
- // Long enough past the grace that the source is written off and the cursor runs on.
- clock.advance(seconds: 2)
- mixer.tick()
+ // Long enough past the grace that the source is written off and the cursor runs on —
+ // with the queue running, so this is a quiet source and not a blocked queue.
+ runTicker(mixer, clock, seconds: 2)
// …and only now does it hand over the span it went quiet for, which is already behind
// the cursor and cannot be placed.
@@ -393,6 +391,67 @@ final class AudioTrackMixerTests: XCTestCase {
XCTAssertEqual(report.trimmedSeconds, 0, accuracy: 0.011)
}
+ /// A blocked sample queue is not a silent source (#1117).
+ ///
+ /// Both sources keep delivering while the queue is held for a second — by the writer, or by
+ /// anything else on it — so their buffers for that second wait behind the block. When the
+ /// queue frees up, the ticker gets there first, which is the worst order: measured on the
+ /// clock alone, both sources would look a second quiet, the cursor would run to now in
+ /// silence, and the queued second would then arrive behind it and be dropped.
+ func testABlockedQueueDoesNotTurnQueuedAudioIntoSilence() {
+ let sink = RecordingSink()
+ let clock = TestClock(.zero)
+ let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1)
+ mixer.beginTimeline(at: clock.now)
+
+ var delivered = CMTime.zero
+ func deliverBoth(seconds: Double) {
+ mixer.ingest(
+ makeSourceBuffer(burst(seconds: seconds, left: 0.5, right: 0), at: delivered, nonInterleaved: true),
+ from: .system
+ )
+ mixer.ingest(
+ makeSourceBuffer(burst(seconds: seconds, left: 0, right: 0.5), at: delivered, nonInterleaved: false),
+ from: .microphone
+ )
+ delivered = CMTimeAdd(delivered, CMTime(seconds: seconds, preferredTimescale: 48_000))
+ }
+
+ for _ in 0..<50 {
+ deliverBoth(seconds: 0.01)
+ runTicker(mixer, clock, seconds: 0.01)
+ }
+
+ // The queue is blocked for a second: no tick, no buffer, while the clock runs on.
+ clock.advance(seconds: 1)
+ mixer.tick()
+ for _ in 0..<100 {
+ deliverBoth(seconds: 0.01)
+ }
+
+ for _ in 0..<50 {
+ deliverBoth(seconds: 0.01)
+ runTicker(mixer, clock, seconds: 0.01)
+ }
+ mixer.finish(atSourceTime: clock.now)
+
+ let frames = mixedFrames(sink)
+ assertContiguous(sink)
+ XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011)
+ XCTAssertEqual(mixer.deliveryReport(for: .system).droppedSeconds, 0, accuracy: 0.0001)
+ XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, accuracy: 0.0001)
+ for step in 0..<20 {
+ let from = Double(step) * 0.1
+ for channel in 0..<channelCount {
+ XCTAssertGreaterThan(
+ peak(frames, from: from, to: from + 0.1, channel: channel),
+ audibleThreshold,
+ "silence on channel \(channel) at \(from)s means queued audio was dropped"
+ )
+ }
+ }
+ }
+
// MARK: - Pause
/// A pause stops the clock instead of resetting it. `ScreenCaptureRecorder.timelineNow()`
@@ -763,6 +822,14 @@ final class AudioTrackMixerTests: XCTestCase {
}
}
+ /// Moves the clock the way the recorder's 10 ms ticker sees it, with the queue running.
+ private func runTicker(_ mixer: AudioTrackMixer, _ clock: TestClock, seconds: Double) {
+ for _ in 0..<Int((seconds * 100).rounded()) {
+ clock.advance(seconds: 0.01)
+ mixer.tick()
+ }
+ }
+
private func peak(_ samples: [Int16], from: Double, to: Double, channel: Int? = nil) -> Int16 {
let firstFrame = Int(from * Double(sampleRate))
let lastFrame = min(Int(to * Double(sampleRate)), samples.count / channelCount)