Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(macos-capture): record from Apple's system picker in a helper session
Etienne Lescotd494a1601ffa
diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/PickerSessionCommand.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/PickerSessionCommand.swiftnew file mode 100644index 000000000..b8051bfcb--- /dev/null+++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/PickerSessionCommand.swift@@ -0,0 +1,68 @@+import Foundation++/// What Apple's system picker may offer. Mirrors the helper's own source types.+public enum PickerMode: String, Equatable {+ case display+ case window+}++/// One line of the `--picker-session` stdin protocol.+///+/// The single-take helper takes its whole request on argv and reads only `pause`, `resume`+/// and `stop` afterwards. A picker session outlives any one take, so it needs a few more+/// verbs, and `start` carries the take's request instead of argv. The bare words stay bare+/// so both modes read the same three commands the same way.+public enum PickerSessionCommand: Equatable {+ /// Show Apple's picker. `excludedWindowIDs` keeps OpenScreen's own HUD and notes out of+ /// a display capture: the filter the picker hands back cannot be edited afterwards, so+ /// the exclusion has to be part of the picker's configuration.+ case present(excludedWindowIDs: [Int], modes: [PickerMode])+ /// Start a take from the retained choice. The request JSON is kept raw for the+ /// recorder's own decoder.+ case start(request: Data)+ case pause+ case resume+ case stop+ case quit+}++public func parsePickerSessionCommand(_ line: String) -> PickerSessionCommand? {+ let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)+ switch trimmed {+ case "pause": return .pause+ case "resume": return .resume+ case "stop": return .stop+ case "quit": return .quit+ default: break+ }++ guard let data = trimmed.data(using: .utf8),+ let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any],+ let command = object["command"] as? String+ else {+ return nil+ }++ switch command {+ case "present":+ let excluded = (object["excludedWindowIds"] as? [Any] ?? []).compactMap { ($0 as? NSNumber)?.intValue }+ let requestedModes = (object["modes"] as? [String] ?? []).compactMap(PickerMode.init(rawValue:))+ return .present(+ excludedWindowIDs: excluded,+ modes: requestedModes.isEmpty ? [.display, .window] : requestedModes+ )+ case "start":+ guard let request = object["request"],+ JSONSerialization.isValidJSONObject(request),+ let requestData = try? JSONSerialization.data(withJSONObject: request)+ else {+ return nil+ }+ return .start(request: requestData)+ case "pause": return .pause+ case "resume": return .resume+ case "stop": return .stop+ case "quit": return .quit+ default: return nil+ }+}diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/PickerSession.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/PickerSession.swiftnew file mode 100644index 000000000..b7a38d789--- /dev/null+++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/PickerSession.swift@@ -0,0 +1,252 @@+import AppKit+import CoreGraphics+import Foundation+import OpenScreenCaptureCore+import ScreenCaptureKit++/// `--picker-session`: Apple's system picker, and every take recorded from what it returns.+///+/// Why a session instead of the usual one process per take:+///+/// - A capture that starts from `SCContentSharingPicker` needs no Screen Recording grant and+/// never raises macOS 15's "bypass the system private window picker" alert: the user's+/// pick is the consent. Measured on macOS 26.5 for display and window picks, from a+/// helper process like this one.+/// - That consent lives in the `SCContentFilter` the picker hands back, and a filter cannot+/// leave the process (it is not NSSecureCoding). So the process that asked has to be the+/// one that records -- and has to stay alive to record the same source again without+/// asking the user to pick it again for every take.+///+/// Each take still gets its own `ScreenCaptureRecorder` and its own `SCStream`, so pause,+/// the audio timeline and the writer behave exactly as in the single-take helper.+///+/// Protocol: one command per stdin line (`PickerSessionCommand`), events on stdout as+/// everywhere else, plus `picker-session-ready`, `picker-presented`, `picker-selected`,+/// `picker-cancelled`, `picker-failed` and `take-ended`. Closing stdin ends the session: nothing here may+/// outlive the app.+///+/// 15.2, not 14.0 where the picker API starts: `includedDisplays` and `includedWindows`+/// arrive in 15.2, and without them the pick cannot be mapped to a screen position, which+/// the cursor telemetry needs. Before 15.2 the app keeps its own picker.+///+/// `@unchecked Sendable`: every piece of state here is read and written on the main queue+/// only -- commands and picker callbacks are all hopped onto it before they touch anything.+@available(macOS 15.2, *)+final class PickerSession: NSObject, SCContentSharingPickerObserver, @unchecked Sendable {+ private var picked: PickedSource?+ private var recorder: ScreenCaptureRecorder?++ func run() -> Never {+ let app = NSApplication.shared+ // No Dock icon and no menu bar: the picker is system UI and needs no window here.+ app.setActivationPolicy(.accessory)+ SCContentSharingPicker.shared.add(self)++ let reader = Thread {+ while let line = readLine() {+ guard let command = parsePickerSessionCommand(line) else {+ emit([+ "event": "warning",+ "code": "picker-session-unknown-command",+ "message": line,+ ])+ continue+ }+ DispatchQueue.main.async { self.handle(command) }+ }+ // stdin closed: the app is gone, or never meant to keep us.+ DispatchQueue.main.async { self.shutdown() }+ }+ reader.start()++ emit(["event": "picker-session-ready"])+ app.run()+ exit(0)+ }++ private func handle(_ command: PickerSessionCommand) {+ switch command {+ case .present(let excludedWindowIDs, let modes):+ present(excludedWindowIDs: excludedWindowIDs, modes: modes)+ case .start(let request):+ startTake(request)+ case .pause:+ recorder?.pause()+ case .resume:+ recorder?.resume()+ case .stop:+ stopTake()+ case .quit:+ shutdown()+ }+ }++ // MARK: - Picker++ private func present(excludedWindowIDs: [Int], modes: [PickerMode]) {+ var configuration = SCContentSharingPickerConfiguration()+ var allowed: SCContentSharingPickerMode = []+ if modes.contains(.display) { allowed.insert(.singleDisplay) }+ if modes.contains(.window) { allowed.insert(.singleWindow) }+ configuration.allowedPickerModes = allowed+ // Swapping the source mid-take would change the frame size under a writer whose+ // dimensions are fixed when the take starts. A new source is a new pick.+ configuration.allowsChangingSelectedContent = false+ // A picked display filter hides this process's own windows, but not the app's:+ // the HUD and the notes window belong to Electron. Measured: a window owned by the+ // parent app shows in the capture unless it is listed here.+ configuration.excludedWindowIDs = excludedWindowIDs++ let picker = SCContentSharingPicker.shared+ picker.defaultConfiguration = configuration+ picker.isActive = true+ picker.present()+ emit(["event": "picker-presented"])+ }++ func contentSharingPicker(+ _ picker: SCContentSharingPicker,+ didUpdateWith filter: SCContentFilter,+ for stream: SCStream?+ ) {+ DispatchQueue.main.async { self.didPick(filter, stream: stream) }+ }++ func contentSharingPicker(_ picker: SCContentSharingPicker, didCancelFor stream: SCStream?) {+ DispatchQueue.main.async {+ // A cancel for a running stream is the user stopping it from the menu bar, which+ // the take's own stop path already reports.+ if stream == nil {+ emit(["event": "picker-cancelled"])+ }+ }+ }++ func contentSharingPickerStartDidFailWithError(_ error: Error) {+ DispatchQueue.main.async {+ emit([+ "event": "picker-failed",+ "message": "\(error)",+ ])+ }+ }++ private func didPick(_ filter: SCContentFilter, stream: SCStream?) {+ // Updates for an existing stream are Apple's "change what you share", switched off+ // by allowsChangingSelectedContent; one that arrives anyway must not swap the source.+ guard stream == nil else {+ return+ }++ let display = filter.includedDisplays.first+ let window = filter.includedWindows.first+ let frame = window?.frame ?? display?.frame ?? filter.contentRect+ let displayId = display?.displayID ?? Self.display(containing: frame)+ picked = PickedSource(filter: filter, frame: frame, displayId: displayId)++ var event: [String: Any] = [+ "event": "picker-selected",+ "kind": window != nil ? "window" : "display",+ "bounds": [+ "x": frame.origin.x,+ "y": frame.origin.y,+ "width": frame.size.width,+ "height": frame.size.height,+ ],+ "pointPixelScale": filter.pointPixelScale,+ ]+ if let displayId {+ event["displayId"] = displayId+ }+ if let window {+ event["windowId"] = window.windowID+ event["title"] = window.title ?? ""+ event["appName"] = window.owningApplication?.applicationName ?? ""+ }+ emit(event)+ }++ private static func display(containing frame: CGRect) -> CGDirectDisplayID? {+ var displays = [CGDirectDisplayID](repeating: 0, count: 8)+ var count: UInt32 = 0+ let center = CGRect(x: frame.midX, y: frame.midY, width: 1, height: 1)+ guard CGGetDisplaysWithRect(center, UInt32(displays.count), &displays, &count) == .success,+ count > 0+ else {+ return nil+ }+ return displays[0]+ }++ // MARK: - Takes++ private func startTake(_ requestData: Data) {+ guard recorder == nil else {+ emitError(code: "take-already-running", message: "A take is already recording in this session.")+ return+ }+ guard let picked else {+ emitError(code: "no-source-picked", message: "Nothing was picked in the system picker yet.")+ emit(["event": "take-ended"])+ return+ }++ let request: RecordingRequest+ do {+ request = try JSONDecoder().decode(RecordingRequest.self, from: requestData)+ } catch {+ emitError(code: "helper-error", message: "\(error)")+ emit(["event": "take-ended"])+ return+ }++ let recorder = ScreenCaptureRecorder(request: request, picked: picked)+ self.recorder = recorder+ let take = ObjectIdentifier(recorder)+ Task {+ do {+ try await recorder.start()+ } catch {+ // Same code the single-take helper reports on its way out, so the app reads a+ // failed start the same way on both paths.+ let message = (error as? HelperError)?.description ?? "\(error)"+ emitError(code: "helper-error", message: message)+ await recorder.stop()+ DispatchQueue.main.async { self.finishTake(take) }+ }+ }+ }++ private func stopTake() {+ guard let recorder else {+ emit(["event": "take-ended"])+ return+ }+ let take = ObjectIdentifier(recorder)+ Task {+ await recorder.stop()+ DispatchQueue.main.async { self.finishTake(take) }+ }+ }++ /// The session's replacement for the single-take helper exiting: the one event that+ /// says this take's output is complete.+ private func finishTake(_ finished: ObjectIdentifier) {+ guard let current = recorder, ObjectIdentifier(current) == finished else {+ return+ }+ recorder = nil+ emit(["event": "take-ended"])+ }++ private func shutdown() {+ SCContentSharingPicker.shared.isActive = false+ guard let recorder else {+ exit(0)+ }+ Task {+ await recorder.stop()+ exit(0)+ }+ }+}diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swiftindex 8b08e1554..34688e5ef 100644--- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift+++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift@@ -103,6 +103,14 @@ enum HelperError: Error, CustomStringConvertible {}}+/// A source chosen in Apple's system picker, kept by `PickerSession` for the takes after it.+struct PickedSource {+ let filter: SCContentFilter+ /// Global frame (points, top-left origin) of what was picked, for cursor mapping.+ let frame: CGRect+ let displayId: CGDirectDisplayID?+}+@available(macOS 13.0, *)final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {private struct CaptureTarget {@@ -146,19 +154,30 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {private var captureFrame = CGRect.zeroprivate let microphoneOutputTypeRawValue = 2private let hostClock = CMClockGetHostTimeClock()+ private let picked: PickedSource?- init(request: RecordingRequest) {+ /// `picked` is a choice already made in Apple's system picker (`PickerSession`). The take+ /// then records exactly that filter: no source lookup, and no Screen Recording check,+ /// because the user's pick IS the consent -- asking for the grant here would put back+ /// the very prompt the picker path exists to remove.+ init(request: RecordingRequest, picked: PickedSource? = nil) {self.request = request+ self.picked = picked}func start() async throws {- try ensureRequestedPermissions()+ try ensureRequestedPermissions(screen: picked == nil)- let content = try await SCShareableContent.excludingDesktopWindows(- false,- onScreenWindowsOnly: true- )- let target = try makeCaptureTarget(from: content)+ let target: CaptureTarget+ if let picked {+ target = makeCaptureTarget(picked: picked)+ } else {+ let content = try await SCShareableContent.excludingDesktopWindows(+ false,+ onScreenWindowsOnly: true+ )+ target = try makeCaptureTarget(from: content)+ }outputWidth = target.widthoutputHeight = target.heightcaptureFrame = target.captureFrame@@ -401,8 +420,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {])}- private func ensureRequestedPermissions() throws {- if !CGPreflightScreenCaptureAccess() {+ private func ensureRequestedPermissions(screen: Bool) throws {+ if screen && !CGPreflightScreenCaptureAccess() {let granted = CGRequestScreenCaptureAccess()if !granted {throw HelperError.permissionDenied("Screen recording permission is required for ScreenCaptureKit capture.")@@ -437,6 +456,20 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {]}+ private func makeCaptureTarget(picked: PickedSource) -> CaptureTarget {+ let size = captureSize(+ for: picked.filter,+ fallbackPointSize: picked.frame.size,+ fallbackDisplayId: picked.displayId ?? CGMainDisplayID()+ )+ return CaptureTarget(+ filter: picked.filter,+ width: size.width,+ height: size.height,+ captureFrame: picked.frame+ )+ }+private func makeCaptureTarget(from content: SCShareableContent) throws -> CaptureTarget {switch request.source.type {case "display":@@ -935,6 +968,10 @@ struct OpenScreenScreenCaptureKitHelper {/// screen", printed as the usual single JSON line and nothing else.private static let screenAccessStatusFlag = "--screen-access-status"+ /// The flag that turns this helper into a long-lived session around Apple's system+ /// picker. See `PickerSession`.+ private static let pickerSessionFlag = "--picker-session"+static func main() async {do {initializeCoreGraphicsWindowServerConnection()@@ -960,6 +997,17 @@ struct OpenScreenScreenCaptureKitHelper {exit(0)}+ if CommandLine.arguments.count == 2, CommandLine.arguments[1] == pickerSessionFlag {+ guard #available(macOS 15.2, *) else {+ emitError(+ code: "picker-session-unsupported",+ message: "The system picker session needs macOS 15.2 or later."+ )+ exit(2)+ }+ PickerSession().run()+ }+guard CommandLine.arguments.count == 2 else {throw HelperError.invalidArguments}diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/PickerSessionCommandTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/PickerSessionCommandTests.swiftnew file mode 100644index 000000000..3391da9ec--- /dev/null+++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/PickerSessionCommandTests.swift@@ -0,0 +1,43 @@+import XCTest+@testable import OpenScreenCaptureCore++final class PickerSessionCommandTests: XCTestCase {+ func testReadsTheBareWordsTheSingleTakeHelperAlreadyUses() {+ XCTAssertEqual(parsePickerSessionCommand("pause\n"), .pause)+ XCTAssertEqual(parsePickerSessionCommand("resume"), .resume)+ XCTAssertEqual(parsePickerSessionCommand(" stop "), .stop)+ XCTAssertEqual(parsePickerSessionCommand("quit"), .quit)+ }++ func testPresentCarriesTheWindowsToKeepOutOfTheCapture() {+ XCTAssertEqual(+ parsePickerSessionCommand(#"{"command":"present","excludedWindowIds":[12,34],"modes":["display"]}"#),+ .present(excludedWindowIDs: [12, 34], modes: [.display])+ )+ }++ func testPresentOffersScreensAndWindowsByDefault() {+ XCTAssertEqual(+ parsePickerSessionCommand(#"{"command":"present"}"#),+ .present(excludedWindowIDs: [], modes: [.display, .window])+ )+ }++ func testStartKeepsTheRequestForTheRecordersDecoder() throws {+ guard case .start(let data)? = parsePickerSessionCommand(+ #"{"command":"start","request":{"video":{"fps":60}}}"#+ ) else {+ return XCTFail("expected a start command")+ }+ let request = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])+ XCTAssertEqual((request["video"] as? [String: Any])?["fps"] as? Int, 60)+ }++ func testRejectsWhatItCannotRead() {+ XCTAssertNil(parsePickerSessionCommand(""))+ XCTAssertNil(parsePickerSessionCommand("record"))+ XCTAssertNil(parsePickerSessionCommand(#"{"command":"start"}"#))+ XCTAssertNil(parsePickerSessionCommand(#"{"command":"explode"}"#))+ XCTAssertNil(parsePickerSessionCommand("{not json"))+ }+}