Skip to content

Linux + iPhone Continuity / iMessage / SMS

fix(camera): physical rotate portrait <-> landscape

Zack Bartelbe8feafde92c
diff --git a/apple/Tether/Views/CameraView.swift b/apple/Tether/Views/CameraView.swift
index 7b8ec2f..f6bbfcc 100644
--- a/apple/Tether/Views/CameraView.swift
+++ b/apple/Tether/Views/CameraView.swift
@@ -11,6 +11,7 @@ import TetherFramework
struct CameraView: View {
@Environment(TetherViewModel.self) private var viewModel
+ @Environment(\.verticalSizeClass) private var verticalSizeClass
@State private var position: AVCaptureDevice.Position = .back
var body: some View {
@@ -34,42 +35,57 @@ struct CameraView: View {
}
private var streamView: some View {
- VStack(spacing: 16) {
- CameraPreview(session: viewModel.camera.session)
- .aspectRatio(16 / 9, contentMode: .fit)
- .background(Color.black)
- .clipShape(RoundedRectangle(cornerRadius: 12))
- .accessibilityLabel("Camera preview")
-
- Picker("Camera", selection: $position) {
- Text("Back").tag(AVCaptureDevice.Position.back)
- Text("Front").tag(AVCaptureDevice.Position.front)
- }
- .pickerStyle(.segmented)
- .disabled(viewModel.camera.isStreaming)
-
- if viewModel.camera.isStreaming {
- Button("Stop", role: .destructive) { viewModel.camera.stop() }
- .buttonStyle(.borderedProminent)
- } else {
- Button("Start Streaming") { viewModel.startCamera(position: position) }
- .buttonStyle(.borderedProminent)
- }
+ // AnyLayout keeps the preview's identity
+ let landscape = verticalSizeClass == .compact
+ let layout = landscape
+ ? AnyLayout(HStackLayout(alignment: .top, spacing: 16))
+ : AnyLayout(VStackLayout(spacing: 16))
+ return layout {
+ preview
+ VStack(spacing: 16) { controls }
+ .frame(maxWidth: landscape ? 280 : .infinity)
+ }
+ .padding()
+ }
- if let error = viewModel.camera.lastError {
- Text(error)
- .font(.footnote)
- .foregroundStyle(.red)
- }
+ // 16:9 like the stream; a portrait phone shows pillarboxed, as the desktop sees it.
+ private var preview: some View {
+ CameraPreview(session: viewModel.camera.session, device: viewModel.camera.device)
+ .aspectRatio(16 / 9, contentMode: .fit)
+ .background(Color.black)
+ .clipShape(RoundedRectangle(cornerRadius: 12))
+ .accessibilityLabel("Camera preview")
+ }
- Text("Keep Tether open while streaming. On the desktop, pick the camera named after this phone.")
- .font(.footnote)
- .foregroundStyle(.secondary)
- .multilineTextAlignment(.center)
+ @ViewBuilder
+ private var controls: some View {
+ Picker("Camera", selection: $position) {
+ Text("Back").tag(AVCaptureDevice.Position.back)
+ Text("Front").tag(AVCaptureDevice.Position.front)
+ }
+ .pickerStyle(.segmented)
+ .disabled(viewModel.camera.isStreaming)
- Spacer()
+ if viewModel.camera.isStreaming {
+ Button("Stop", role: .destructive) { viewModel.camera.stop() }
+ .buttonStyle(.borderedProminent)
+ } else {
+ Button("Start Streaming") { viewModel.startCamera(position: position) }
+ .buttonStyle(.borderedProminent)
}
- .padding()
+
+ if let error = viewModel.camera.lastError {
+ Text(error)
+ .font(.footnote)
+ .foregroundStyle(.red)
+ }
+
+ Text("Keep Tether open while streaming. On the desktop, pick the camera named after this phone.")
+ .font(.footnote)
+ .foregroundStyle(.secondary)
+ .multilineTextAlignment(.center)
+
+ Spacer()
}
private func message(_ text: LocalizedStringKey) -> some View {
@@ -90,6 +106,16 @@ struct CameraView: View {
private struct CameraPreview: UIViewRepresentable {
let session: AVCaptureSession
+ let device: AVCaptureDevice?
+
+ // Keeps the preview upright as the phone and the UI rotate.
+ final class Coordinator {
+ var device: AVCaptureDevice?
+ var rotation: AVCaptureDevice.RotationCoordinator?
+ var observation: NSKeyValueObservation?
+ }
+
+ func makeCoordinator() -> Coordinator { Coordinator() }
final class PreviewView: UIView {
override class var layerClass: AnyClass { AVCaptureVideoPreviewLayer.self }
@@ -103,5 +129,25 @@ private struct CameraPreview: UIViewRepresentable {
return view
}
- func updateUIView(_ uiView: PreviewView, context: Context) {}
+ func updateUIView(_ uiView: PreviewView, context: Context) {
+ let coordinator = context.coordinator
+ guard coordinator.device !== device else { return }
+ coordinator.device = device
+ coordinator.observation = nil
+ coordinator.rotation = nil
+ guard let device else { return }
+
+ let layer = uiView.previewLayer
+ let rotation = AVCaptureDevice.RotationCoordinator(device: device, previewLayer: layer)
+ coordinator.observation = rotation.observe(\.videoRotationAngleForHorizonLevelPreview,
+ options: [.initial, .new]) { [weak layer] rotation, _ in
+ let angle = rotation.videoRotationAngleForHorizonLevelPreview
+ DispatchQueue.main.async {
+ if let connection = layer?.connection, connection.isVideoRotationAngleSupported(angle) {
+ connection.videoRotationAngle = angle
+ }
+ }
+ }
+ coordinator.rotation = rotation
+ }
}
diff --git a/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift b/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift
index 05f29ba..d23ea58 100644
--- a/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift
+++ b/apple/TetherFramework/Sources/TetherFramework/CameraStreamer.swift
@@ -21,11 +21,19 @@ public final class CameraStreamer: NSObject {
// Feed for a preview layer; runs only while streaming.
public let session = AVCaptureSession()
+ // The camera feeding `session`, for rotating a preview to match.
+ public private(set) var device: AVCaptureDevice?
// Everything below is confined to `queue`.
private let queue = DispatchQueue(label: "tether.camera")
@ObservationIgnored private var connection: NWConnection?
@ObservationIgnored private var encoder: VTCompressionSession?
+ // Turns frames upright and pillarboxes portrait ones into the encoder's landscape size.
+ @ObservationIgnored private var fitter: FrameFitter?
+ @ObservationIgnored private var rotation: AVCaptureDevice.RotationCoordinator?
+ @ObservationIgnored private var rotationObservation: NSKeyValueObservation?
+ // Clockwise degrees that turn the sensor's landscape frame upright.
+ @ObservationIgnored private var captureAngle: Double = 0
// Frames handed to the encoder whose bytes the network has not taken yet.
@ObservationIgnored private var pending = 0
@@ -92,10 +100,14 @@ public final class CameraStreamer: NSObject {
private func teardown() {
if session.isRunning { session.stopRunning() }
+ rotationObservation = nil
+ rotation = nil
if let encoder {
VTCompressionSessionInvalidate(encoder)
}
encoder = nil
+ fitter = nil
+ DispatchQueue.main.async { self.device = nil }
connection?.stateUpdateHandler = nil
connection?.cancel()
connection = nil
@@ -132,12 +144,24 @@ public final class CameraStreamer: NSObject {
output.alwaysDiscardsLateVideoFrames = true
output.setSampleBufferDelegate(self, queue: queue)
session.addOutput(output)
- // Landscape whatever the phone's orientation, so the frame size never changes mid-stream.
+ // The sensor's own landscape, always. Rotating the connection while running
+ // rebuilds the capture graph, which stalls the feed and the UI; FrameFitter
+ // turns each frame upright instead.
if let videoConnection = output.connection(with: .video), videoConnection.isVideoRotationAngleSupported(0) {
videoConnection.videoRotationAngle = 0
}
session.commitConfiguration()
+
+ captureAngle = 0
+ let rotation = AVCaptureDevice.RotationCoordinator(device: device, previewLayer: nil)
+ rotationObservation = rotation.observe(\.videoRotationAngleForHorizonLevelCapture, options: [.initial, .new]) {
+ [weak self] coordinator, _ in
+ let angle = coordinator.videoRotationAngleForHorizonLevelCapture
+ self?.queue.async { self?.captureAngle = angle }
+ }
+ self.rotation = rotation
session.startRunning()
+ DispatchQueue.main.async { self.device = device }
}
private func makeEncoder(width: Int32, height: Int32) -> VTCompressionSession? {
@@ -263,18 +287,21 @@ extension CameraStreamer: AVCaptureVideoDataOutputSampleBufferDelegate {
guard pending < Self.maxPending else { return }
if encoder == nil {
- encoder = makeEncoder(width: Int32(CVPixelBufferGetWidth(image)),
- height: Int32(CVPixelBufferGetHeight(image)))
+ // Always landscape, even when the stream starts in portrait.
+ let w = CVPixelBufferGetWidth(image), h = CVPixelBufferGetHeight(image)
+ fitter = FrameFitter(width: max(w, h), height: min(w, h))
+ encoder = makeEncoder(width: Int32(max(w, h)), height: Int32(min(w, h)))
}
- guard let encoder else {
+ guard let encoder, let fitter else {
fail("H.264 encoder unavailable")
return
}
+ guard let fitted = fitter.fit(image, rotation: captureAngle) else { return }
pending += 1
let status = VTCompressionSessionEncodeFrame(
encoder,
- imageBuffer: image,
+ imageBuffer: fitted,
presentationTimeStamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer),
duration: .invalid,
frameProperties: nil,
diff --git a/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swift b/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swift
new file mode 100644
index 0000000..48f73ce
--- /dev/null
+++ b/apple/TetherFramework/Sources/TetherFramework/FrameFitter.swift
@@ -0,0 +1,94 @@
+//
+// FrameFitter.swift
+// TetherFramework
+//
+// Turns camera frames upright and fits them into one fixed size, so the
+// desktop's webcam never changes resolution when the phone rotates.
+//
+
+import CoreVideo
+import VideoToolbox
+
+final class FrameFitter {
+ let width: Int
+ let height: Int
+
+ private let pixelFormat: OSType
+ private let rotator: VTPixelRotationSession
+ private let transfer: VTPixelTransferSession
+ // Keyed by "WxH": the output size, plus the tall size for quarter turns.
+ private var pools: [String: CVPixelBufferPool] = [:]
+ private var quarterTurns = 0
+
+ init?(width: Int, height: Int, pixelFormat: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
+ var rotator: VTPixelRotationSession?
+ var transfer: VTPixelTransferSession?
+ guard VTPixelRotationSessionCreate(nil, &rotator) == noErr,
+ VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer) == noErr,
+ let rotator, let transfer
+ else { return nil }
+ // Scale to fit, filling the rest with black: pillarbox for portrait frames.
+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_ScalingMode, value: kVTScalingMode_Letterbox)
+
+ self.width = width
+ self.height = height
+ self.pixelFormat = pixelFormat
+ self.rotator = rotator
+ self.transfer = transfer
+ }
+
+ deinit {
+ VTPixelRotationSessionInvalidate(rotator)
+ VTPixelTransferSessionInvalidate(transfer)
+ }
+
+ // Turns the image clockwise by `angle` degrees (a multiple of 90), then pillarboxes
+ // it to the output size. Returns the image itself when it needs neither.
+ func fit(_ image: CVPixelBuffer, rotation angle: Double = 0) -> CVPixelBuffer? {
+ let turns = ((Int((angle / 90).rounded()) % 4) + 4) % 4
+ var source = image
+
+ if turns != 0 {
+ if turns != quarterTurns {
+ let values = [kVTRotation_0, kVTRotation_CW90, kVTRotation_180, kVTRotation_CCW90]
+ VTSessionSetProperty(rotator, key: kVTPixelRotationPropertyKey_Rotation, value: values[turns])
+ quarterTurns = turns
+ }
+ let w = CVPixelBufferGetWidth(image), h = CVPixelBufferGetHeight(image)
+ guard let turned = makeBuffer(width: turns.isMultiple(of: 2) ? w : h,
+ height: turns.isMultiple(of: 2) ? h : w),
+ VTPixelRotationSessionRotateImage(rotator, image, turned) == noErr
+ else { return nil }
+ source = turned
+ }
+
+ if CVPixelBufferGetWidth(source) == width && CVPixelBufferGetHeight(source) == height {
+ return source
+ }
+ guard let out = makeBuffer(width: width, height: height),
+ VTPixelTransferSessionTransferImage(transfer, from: source, to: out) == noErr
+ else { return nil }
+ return out
+ }
+
+ private func makeBuffer(width: Int, height: Int) -> CVPixelBuffer? {
+ let key = "\(width)x\(height)"
+ if pools[key] == nil {
+ let attributes: [String: Any] = [
+ kCVPixelBufferPixelFormatTypeKey as String: pixelFormat,
+ kCVPixelBufferWidthKey as String: width,
+ kCVPixelBufferHeightKey as String: height,
+ kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any](),
+ ]
+ var pool: CVPixelBufferPool?
+ guard CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary, &pool) == kCVReturnSuccess
+ else { return nil }
+ pools[key] = pool
+ }
+ var buffer: CVPixelBuffer?
+ guard let pool = pools[key],
+ CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess
+ else { return nil }
+ return buffer
+ }
+}
diff --git a/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swift b/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swift
new file mode 100644
index 0000000..8528314
--- /dev/null
+++ b/apple/TetherFramework/Tests/TetherFrameworkTests/FrameFitterTests.swift
@@ -0,0 +1,96 @@
+//
+// FrameFitterTests.swift
+// TetherFrameworkTests
+//
+
+import CoreVideo
+import Foundation
+import Testing
+@testable import TetherFramework
+
+struct FrameFitterTests {
+ private func makeFrame(width: Int, height: Int, luma: UInt8) throws -> CVPixelBuffer {
+ var buffer: CVPixelBuffer?
+ let attributes = [kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any]()] as CFDictionary
+ try #require(CVPixelBufferCreate(nil, width, height, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
+ attributes, &buffer) == kCVReturnSuccess)
+ let frame = try #require(buffer)
+ fill(frame, luma: luma)
+ return frame
+ }
+
+ private func fill(_ frame: CVPixelBuffer, luma: UInt8) {
+ CVPixelBufferLockBaseAddress(frame, [])
+ defer { CVPixelBufferUnlockBaseAddress(frame, []) }
+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 0), Int32(luma),
+ CVPixelBufferGetBytesPerRowOfPlane(frame, 0) * CVPixelBufferGetHeightOfPlane(frame, 0))
+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 1), 128,
+ CVPixelBufferGetBytesPerRowOfPlane(frame, 1) * CVPixelBufferGetHeightOfPlane(frame, 1))
+ }
+
+ // Top half `top`, bottom half `bottom`, so a rotation shows where the top went.
+ private func makeSplitFrame(width: Int, height: Int, top: UInt8, bottom: UInt8) throws -> CVPixelBuffer {
+ let frame = try makeFrame(width: width, height: height, luma: bottom)
+ CVPixelBufferLockBaseAddress(frame, [])
+ defer { CVPixelBufferUnlockBaseAddress(frame, []) }
+ memset(CVPixelBufferGetBaseAddressOfPlane(frame, 0), Int32(top),
+ CVPixelBufferGetBytesPerRowOfPlane(frame, 0) * height / 2)
+ return frame
+ }
+
+ private func luma(_ frame: CVPixelBuffer, x: Int, y: Int) -> UInt8 {
+ CVPixelBufferLockBaseAddress(frame, .readOnly)
+ defer { CVPixelBufferUnlockBaseAddress(frame, .readOnly) }
+ let base = CVPixelBufferGetBaseAddressOfPlane(frame, 0)!.assumingMemoryBound(to: UInt8.self)
+ return base[y * CVPixelBufferGetBytesPerRowOfPlane(frame, 0) + x]
+ }
+
+ @Test func landscapeFramePassesThrough() throws {
+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))
+ let frame = try makeFrame(width: 1920, height: 1080, luma: 200)
+ #expect(fitter.fit(frame) === frame)
+ }
+
+ // A phone held upright streams a tall picture with black bars, at the same size.
+ @Test func portraitFrameIsPillarboxed() throws {
+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))
+ let frame = try makeFrame(width: 1080, height: 1920, luma: 235)
+
+ // Dirty a pooled buffer first: recycled buffers must still get black bars.
+ if let first = fitter.fit(frame) { fill(first, luma: 235) }
+
+ let out = try #require(fitter.fit(frame))
+ #expect(CVPixelBufferGetWidth(out) == 1920)
+ #expect(CVPixelBufferGetHeight(out) == 1080)
+ #expect(luma(out, x: 10, y: 540) <= 20)
+ #expect(luma(out, x: 1909, y: 540) <= 20)
+ #expect(luma(out, x: 960, y: 540) >= 200)
+ }
+}
+
+extension FrameFitterTests {
+ // The camera always delivers landscape; a phone held upright needs a quarter
+ // turn, then the tall result is pillarboxed back into the same 16:9 size.
+ @Test func quarterTurnIsRotatedThenPillarboxed() throws {
+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))
+ let frame = try makeSplitFrame(width: 1920, height: 1080, top: 235, bottom: 100)
+ let out = try #require(fitter.fit(frame, rotation: 90))
+
+ #expect(CVPixelBufferGetWidth(out) == 1920)
+ #expect(CVPixelBufferGetHeight(out) == 1080)
+ // Clockwise: the old top is now the right half of the centred picture.
+ #expect(luma(out, x: 10, y: 540) <= 20)
+ #expect(luma(out, x: 1100, y: 540) >= 200)
+ #expect((80...120).contains(luma(out, x: 820, y: 540)))
+ }
+
+ @Test func halfTurnKeepsTheSizeAndFlips() throws {
+ let fitter = try #require(FrameFitter(width: 1920, height: 1080))
+ let frame = try makeSplitFrame(width: 1920, height: 1080, top: 235, bottom: 100)
+ let out = try #require(fitter.fit(frame, rotation: 180))
+
+ #expect(CVPixelBufferGetWidth(out) == 1920)
+ #expect((80...120).contains(luma(out, x: 960, y: 100)))
+ #expect(luma(out, x: 960, y: 980) >= 200)
+ }
+}