Linux + iPhone Continuity / iMessage / SMS
fix(camera): physical rotate portrait <-> landscape
Zack Bartelbe8feafde92c
diff --git a/apple/Tether/Views/CameraView.swift b/apple/Tether/Views/CameraView.swiftindex 7b8ec2f..f6bbfcc 100644--- a/apple/Tether/Views/CameraView.swift+++ b/apple/Tether/Views/CameraView.swift@@ -11,6 +11,7 @@ import TetherFrameworkstruct CameraView: View {@Environment(TetherViewModel.self) private var viewModel+ @Environment(\.verticalSizeClass) private var verticalSizeClass@State private var position: AVCaptureDevice.Position = .backvar body: some View {@@ -34,42 +35,57 @@ struct CameraView: View {}private var streamView: some View {- VStack(spacing: 16) {- CameraPreview(session: viewModel.camera.session)- .aspectRatio(16 / 9, contentMode: .fit)- .background(Color.black)- .clipShape(RoundedRectangle(cornerRadius: 12))- .accessibilityLabel("Camera preview")-- Picker("Camera", selection: $position) {- Text("Back").tag(AVCaptureDevice.Position.back)- Text("Front").tag(AVCaptureDevice.Position.front)- }- .pickerStyle(.segmented)- .disabled(viewModel.camera.isStreaming)-- if viewModel.camera.isStreaming {- Button("Stop", role: .destructive) { viewModel.camera.stop() }- .buttonStyle(.borderedProminent)- } else {- Button("Start Streaming") { viewModel.startCamera(position: position) }- .buttonStyle(.borderedProminent)- }+ // AnyLayout keeps the preview's identity+ let landscape = verticalSizeClass == .compact+ let layout = landscape+ ? AnyLayout(HStackLayout(alignment: .top, spacing: 16))+ : AnyLayout(VStackLayout(spacing: 16))+ return layout {+ preview+ VStack(spacing: 16) { controls }+ .frame(maxWidth: landscape ? 280 : .infinity)+ }+ .padding()+ }- if let error = viewModel.camera.lastError {- Text(error)- .font(.footnote)- .foregroundStyle(.red)- }+ // 16:9 like the stream; a portrait phone shows pillarboxed, as the desktop sees it.+ private var preview: some View {+ CameraPreview(session: viewModel.camera.session, device: viewModel.camera.device)+ .aspectRatio(16 / 9, contentMode: .fit)+ .background(Color.black)+ .clipShape(RoundedRectangle(cornerRadius: 12))+ .accessibilityLabel("Camera preview")+ }- Text("Keep Tether open while streaming. On the desktop, pick the camera named after this phone.")- .font(.footnote)- .foregroundStyle(.secondary)- .multilineTextAlignment(.center)+ @ViewBuilder+ private var controls: some View {+ Picker("Camera", selection: $position) {+ Text("Back").tag(AVCaptureDevice.Position.back)+ Text("Front").tag(AVCaptureDevice.Position.front)+ }+ .pickerStyle(.segmented)+ .disabled(viewModel.camera.isStreaming)- Spacer()+ if viewModel.camera.isStreaming {+ Button("Stop", role: .destructive) { viewModel.camera.stop() }+ .buttonStyle(.borderedProminent)+ } else {+ Button("Start Streaming") { viewModel.startCamera(position: position) }+ .buttonStyle(.borderedProminent)}- .padding()++ if let error = viewModel.camera.lastError {+ Text(error)+ .font(.footnote)+ .foregroundStyle(.red)+ }++ Text("Keep Tether open while streaming. On the desktop, pick the camera named after this phone.")+ .font(.footnote)+ .foregroundStyle(.secondary)+ .multilineTextAlignment(.center)++ Spacer()}private func message(_ text: LocalizedStringKey) -> some View {@@ -90,6 +106,16 @@ struct CameraView: View {private struct CameraPreview: UIViewRepresentable {let session: AVCaptureSession+ let device: AVCaptureDevice?++ // Keeps the preview upright as the phone and the UI rotate.+ final class Coordinator {+ var device: AVCaptureDevice?+ var rotation: AVCaptureDevice.RotationCoordinator?+ var observation: NSKeyValueObservation?+ }++ func makeCoordinator() -> Coordinator { Coordinator() }final class PreviewView: UIView {override class var layerClass: AnyClass { AVCaptureVideoPreviewLayer.self }@@ -103,5 +129,25 @@ private struct CameraPreview: UIViewRepresentable {return view}- func updateUIView(_ uiView: PreviewView, context: Context) {}+ func updateUIView(_ uiView: PreviewView, context: Context) {+ let coordinator = context.coordinator+ guard coordinator.device !== device else { return }+ coordinator.device = device+ coordinator.observation = nil+ coordinator.rotation = nil+ guard let device else { return }++ let layer = uiView.previewLayer+ let rotation = AVCaptureDevice.RotationCoordinator(device: device, previewLayer: layer)+ coordinator.observation = rotation.observe(\.videoRotationAngleForHorizonLevelPreview,+ options: [.initial, .new]) { [weak layer] rotation, _ in+ let angle = rotation.videoRotationAngleForHorizonLevelPreview+ DispatchQueue.main.async {+ if let connection = layer?.connection, connection.isVideoRotationAngleSupported(angle) {+ connection.videoRotationAngle = angle+ }+ }+ }+ coordinator.rotation = rotation+ }}diff --git a/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift b/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swiftindex 05f29ba..d23ea58 100644--- a/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift+++ b/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift@@ -21,11 +21,19 @@ public final class CameraStreamer: NSObject {// Feed for a preview layer; runs only while streaming.public let session = AVCaptureSession()+ // The camera feeding `session`, for rotating a preview to match.+ public private(set) var device: AVCaptureDevice?// Everything below is confined to `queue`.private let queue = DispatchQueue(label: "tether.camera")@ObservationIgnored private var connection: NWConnection?@ObservationIgnored private var encoder: VTCompressionSession?+ // Turns frames upright and pillarboxes portrait ones into the encoder's landscape size.+ @ObservationIgnored private var fitter: FrameFitter?+ @ObservationIgnored private var rotation: AVCaptureDevice.RotationCoordinator?+ @ObservationIgnored private var rotationObservation: NSKeyValueObservation?+ // Clockwise degrees that turn the sensor's landscape frame upright.+ @ObservationIgnored private var captureAngle: Double = 0// Frames handed to the encoder whose bytes the network has not taken yet.@ObservationIgnored private var pending = 0@@ -92,10 +100,14 @@ public final class CameraStreamer: NSObject {private func teardown() {if session.isRunning { session.stopRunning() }+ rotationObservation = nil+ rotation = nilif let encoder {VTCompressionSessionInvalidate(encoder)}encoder = nil+ fitter = nil+ DispatchQueue.main.async { self.device = nil }connection?.stateUpdateHandler = nilconnection?.cancel()connection = nil@@ -132,12 +144,24 @@ public final class CameraStreamer: NSObject {output.alwaysDiscardsLateVideoFrames = trueoutput.setSampleBufferDelegate(self, queue: queue)session.addOutput(output)- // Landscape whatever the phone's orientation, so the frame size never changes mid-stream.+ // The sensor's own landscape, always. Rotating the connection while running+ // rebuilds the capture graph, which stalls the feed and the UI; FrameFitter+ // turns each frame upright instead.if let videoConnection = output.connection(with: .video), videoConnection.isVideoRotationAngleSupported(0) {videoConnection.videoRotationAngle = 0}session.commitConfiguration()++ captureAngle = 0+ let rotation = AVCaptureDevice.RotationCoordinator(device: device, previewLayer: nil)+ rotationObservation = rotation.observe(\.videoRotationAngleForHorizonLevelCapture, options: [.initial, .new]) {+ [weak self] coordinator, _ in+ let angle = coordinator.videoRotationAngleForHorizonLevelCapture+ self?.queue.async { self?.captureAngle = angle }+ }+ self.rotation = rotationsession.startRunning()+ DispatchQueue.main.async { self.device = device }}private func makeEncoder(width: Int32, height: Int32) -> VTCompressionSession? {@@ -263,18 +287,21 @@ extension CameraStreamer: AVCaptureVideoDataOutputSampleBufferDelegate {guard pending < Self.maxPending else { return }if encoder == nil {- encoder = makeEncoder(width: Int32(CVPixelBufferGetWidth(image)),- height: Int32(CVPixelBufferGetHeight(image)))+ // Always landscape, even when the stream starts in portrait.+ let w = CVPixelBufferGetWidth(image), h = CVPixelBufferGetHeight(image)+ fitter = FrameFitter(width: max(w, h), height: min(w, h))+ encoder = makeEncoder(width: Int32(max(w, h)), height: Int32(min(w, h)))}- guard let encoder else {+ guard let encoder, let fitter else {fail("H.264 encoder unavailable")return}+ guard let fitted = fitter.fit(image, rotation: captureAngle) else { return }pending += 1let status = VTCompressionSessionEncodeFrame(encoder,- imageBuffer: image,+ imageBuffer: fitted,presentationTimeStamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer),duration: .invalid,frameProperties: nil,diff --git a/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swift b/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swiftnew file mode 100644index 0000000..48f73ce--- /dev/null+++ b/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swift@@ -0,0 +1,94 @@+//+// FrameFitter.swift+// TetherFramework+//+// Turns camera frames upright and fits them into one fixed size, so the+// desktop's webcam never changes resolution when the phone rotates.+//++import CoreVideo+import VideoToolbox++final class FrameFitter {+ let width: Int+ let height: Int++ private let pixelFormat: OSType+ private let rotator: VTPixelRotationSession+ private let transfer: VTPixelTransferSession+ // Keyed by "WxH": the output size, plus the tall size for quarter turns.+ private var pools: [String: CVPixelBufferPool] = [:]+ private var quarterTurns = 0++ init?(width: Int, height: Int, pixelFormat: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {+ var rotator: VTPixelRotationSession?+ var transfer: VTPixelTransferSession?+ guard VTPixelRotationSessionCreate(nil, &rotator) == noErr,+ VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer) == noErr,+ let rotator, let transfer+ else { return nil }+ // Scale to fit, filling the rest with black: pillarbox for portrait frames.+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_ScalingMode, value: kVTScalingMode_Letterbox)++ self.width = width+ self.height = height+ self.pixelFormat = pixelFormat+ self.rotator = rotator+ self.transfer = transfer+ }++ deinit {+ VTPixelRotationSessionInvalidate(rotator)+ VTPixelTransferSessionInvalidate(transfer)+ }++ // Turns the image clockwise by `angle` degrees (a multiple of 90), then pillarboxes+ // it to the output size. Returns the image itself when it needs neither.+ func fit(_ image: CVPixelBuffer, rotation angle: Double = 0) -> CVPixelBuffer? {+ let turns = ((Int((angle / 90).rounded()) % 4) + 4) % 4+ var source = image++ if turns != 0 {+ if turns != quarterTurns {+ let values = [kVTRotation_0, kVTRotation_CW90, kVTRotation_180, kVTRotation_CCW90]+ VTSessionSetProperty(rotator, key: kVTPixelRotationPropertyKey_Rotation, value: values[turns])+ quarterTurns = turns+ }+ let w = CVPixelBufferGetWidth(image), h = CVPixelBufferGetHeight(image)+ guard let turned = makeBuffer(width: turns.isMultiple(of: 2) ? w : h,+ height: turns.isMultiple(of: 2) ? h : w),+ VTPixelRotationSessionRotateImage(rotator, image, turned) == noErr+ else { return nil }+ source = turned+ }++ if CVPixelBufferGetWidth(source) == width && CVPixelBufferGetHeight(source) == height {+ return source+ }+ guard let out = makeBuffer(width: width, height: height),+ VTPixelTransferSessionTransferImage(transfer, from: source, to: out) == noErr+ else { return nil }+ return out+ }++ private func makeBuffer(width: Int, height: Int) -> CVPixelBuffer? {+ let key = "\(width)x\(height)"+ if pools[key] == nil {+ let attributes: [String: Any] = [+ kCVPixelBufferPixelFormatTypeKey as String: pixelFormat,+ kCVPixelBufferWidthKey as String: width,+ kCVPixelBufferHeightKey as String: height,+ kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any](),+ ]+ var pool: CVPixelBufferPool?+ guard CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary, &pool) == kCVReturnSuccess+ else { return nil }+ pools[key] = pool+ }+ var buffer: CVPixelBuffer?+ guard let pool = pools[key],+ CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess+ else { return nil }+ return buffer+ }+}diff --git a/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swift b/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swiftnew file mode 100644index 0000000..8528314--- /dev/null+++ b/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swift@@ -0,0 +1,96 @@+//+// FrameFitterTests.swift+// TetherFrameworkTests+//++import CoreVideo+import Foundation+import Testing+@testable import TetherFramework++struct FrameFitterTests {+ private func makeFrame(width: Int, height: Int, luma: UInt8) throws -> CVPixelBuffer {+ var buffer: CVPixelBuffer?+ let attributes = [kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any]()] as CFDictionary+ try #require(CVPixelBufferCreate(nil, width, height, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,+ attributes, &buffer) == kCVReturnSuccess)+ let frame = try #require(buffer)+ fill(frame, luma: luma)+ return frame+ }++ private func fill(_ frame: CVPixelBuffer, luma: UInt8) {+ CVPixelBufferLockBaseAddress(frame, [])+ defer { CVPixelBufferUnlockBaseAddress(frame, []) }+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 0), Int32(luma),+ CVPixelBufferGetBytesPerRowOfPlane(frame, 0) * CVPixelBufferGetHeightOfPlane(frame, 0))+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 1), 128,+ CVPixelBufferGetBytesPerRowOfPlane(frame, 1) * CVPixelBufferGetHeightOfPlane(frame, 1))+ }++ // Top half `top`, bottom half `bottom`, so a rotation shows where the top went.+ private func makeSplitFrame(width: Int, height: Int, top: UInt8, bottom: UInt8) throws -> CVPixelBuffer {+ let frame = try makeFrame(width: width, height: height, luma: bottom)+ CVPixelBufferLockBaseAddress(frame, [])+ defer { CVPixelBufferUnlockBaseAddress(frame, []) }+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 0), Int32(top),+ CVPixelBufferGetBytesPerRowOfPlane(frame, 0) * height / 2)+ return frame+ }++ private func luma(_ frame: CVPixelBuffer, x: Int, y: Int) -> UInt8 {+ CVPixelBufferLockBaseAddress(frame, .readOnly)+ defer { CVPixelBufferUnlockBaseAddress(frame, .readOnly) }+ let base = CVPixelBufferGetBaseAddressOfPlane(frame, 0)!.assumingMemoryBound(to: UInt8.self)+ return base[y * CVPixelBufferGetBytesPerRowOfPlane(frame, 0) + x]+ }++ @Test func landscapeFramePassesThrough() throws {+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))+ let frame = try makeFrame(width: 1920, height: 1080, luma: 200)+ #expect(fitter.fit(frame) === frame)+ }++ // A phone held upright streams a tall picture with black bars, at the same size.+ @Test func portraitFrameIsPillarboxed() throws {+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))+ let frame = try makeFrame(width: 1080, height: 1920, luma: 235)++ // Dirty a pooled buffer first: recycled buffers must still get black bars.+ if let first = fitter.fit(frame) { fill(first, luma: 235) }++ let out = try #require(fitter.fit(frame))+ #expect(CVPixelBufferGetWidth(out) == 1920)+ #expect(CVPixelBufferGetHeight(out) == 1080)+ #expect(luma(out, x: 10, y: 540) <= 20)+ #expect(luma(out, x: 1909, y: 540) <= 20)+ #expect(luma(out, x: 960, y: 540) >= 200)+ }+}++extension FrameFitterTests {+ // The camera always delivers landscape; a phone held upright needs a quarter+ // turn, then the tall result is pillarboxed back into the same 16:9 size.+ @Test func quarterTurnIsRotatedThenPillarboxed() throws {+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))+ let frame = try makeSplitFrame(width: 1920, height: 1080, top: 235, bottom: 100)+ let out = try #require(fitter.fit(frame, rotation: 90))++ #expect(CVPixelBufferGetWidth(out) == 1920)+ #expect(CVPixelBufferGetHeight(out) == 1080)+ // Clockwise: the old top is now the right half of the centred picture.+ #expect(luma(out, x: 10, y: 540) <= 20)+ #expect(luma(out, x: 1100, y: 540) >= 200)+ #expect((80...120).contains(luma(out, x: 820, y: 540)))+ }++ @Test func halfTurnKeepsTheSizeAndFlips() throws {+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))+ let frame = try makeSplitFrame(width: 1920, height: 1080, top: 235, bottom: 100)+ let out = try #require(fitter.fit(frame, rotation: 180))++ #expect(CVPixelBufferGetWidth(out) == 1920)+ #expect((80...120).contains(luma(out, x: 960, y: 100)))+ #expect(luma(out, x: 960, y: 980) >= 200)+ }+}