Run a virtual iPhone on an Apple Silicon Mac.
Swift
Serve the VM display as encoded video at /v1/host/screen
Jackycd624be5f2bc
diff --git a/Research/vphoned_http_api.md b/Research/vphoned_http_api.mdindex 1655553d..6b235b87 100644--- a/Research/vphoned_http_api.md+++ b/Research/vphoned_http_api.md@@ -7,7 +7,9 @@ upgrade at `/v1/events` uses the same port. `vphone-vm` can expose that bytestream on a host TCP address. After it admits a connection's first requesthead with the API token (see [Access control](#access-control)), it opens oneVSOCK connection for that TCP connection and forwards bytes in both-directions; it does not translate HTTP or WebSocket messages. The host+directions; it does not translate HTTP or WebSocket messages. The one+exception is `GET /v1/host/screen`, which `vphone-vm` answers itself (see+[Host screen](#host-screen)). The hostlistener is absent unless boot receives `--api-listen host:port`. Port `0`asks the OS for an available host port and the actual address is printedafter the VM starts. The API is also usable from guest and host code that@@ -202,6 +204,52 @@ installed record remains so the operation can be retried. Irisin's mobileDocuments data outside the bootstrap is retained. Apps > Uninstall Bootstrap…shows every path in a destructive confirmation alert before sending the request.+## Host screen++`GET /v1/host/screen` on the host proxy is a WebSocket that `vphone-vm`+serves itself; it never reaches vphoned. It is admitted with the same token+as every other request, and a `vphone-token.<token>` subprotocol is selected+in the 101 reply as vphoned does. A plain GET gets `426 Upgrade Required`.++Frames come from the VM display that Virtualization keeps for its window+(`_takeScreenshotWithCompletionHandler:`), with or without a window, so the+guest does no work for them: no screenshot, no JPEG, no base64. The guest is+asked only for `display.orientation`, at most once a second while a client is+connected; the first frame waits up to 0.5 s for that answer. `vphone-vm`+turns the frame upright, fits it into the requested box (the screen helper's+own rule: no enlarging, short side at least 240, even sides) and encodes it+with the hardware encoder in its own process: real time, low-latency rate+control, no frame reordering, key frames only on request, Display P3+primaries with the sRGB transfer and the BT.709 matrix in video range.++The client pulls one frame per text message:++```json+{"id":7,"codec":"hevc","box":[1280,1280],"fps":60,"bitrate":4000000,"key":false}+```++`codec` is `hevc` or `h264`; `box` is 1 to 8192 on each side, `fps` 1 to+240, `bitrate` 1 to 100 000 000. Each request gets one binary answer, in the+order the requests came: a 4-byte little-endian header length, the JSON+header, then the payload. The header always has `id` and then either+`error` (`{code, message}`, no payload) or `status`:++| `status` | Payload | Meaning |+| --- | --- | --- |+| `frame` | one access unit, 4-byte length-prefixed NAL units, no parameter sets | `key`, and on a key frame `record`: the `hvcC` / `avcC`, base64 |+| `unchanged` | none | every pixel is what the last frame encoded showed, and no key frame was asked for |+| `dropped` | none | the rate controller skipped the frame |++Every header with a `status` also has `session` (it changes with the codec,+the output size or the orientation, and the first frame of a session is a+key frame), `width` and `height` (the encoded size), `sourceWidth`,+`sourceHeight` and `scale` (the upright display in pixels and its pixels per+interface point, the VM's screen scale), and `captureUs` and `encodeUs`.+Each connection has its own encoder; a new connection starts at session 1.+Error codes: `bad_request`, `codec_unavailable` (no hardware encoder for that+codec or size), `display_unavailable`, `encoder_failed` (the next request+starts a new session) and `too_many_requests` (more than 16 waiting).+## HTTP and WebSocket contractJSON resource routes cover device state, apps, input, location, Developer Mode, low powerdiff --git a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneAPIProxy.swift b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneAPIProxy.swiftindex e25f45f2..f8aa04a9 100644--- a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneAPIProxy.swift+++ b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneAPIProxy.swift@@ -13,6 +13,9 @@ import VPhoneCoreKit/// vphoned runs as root and cannot tell a proxied connection from the VM's/// own client, so the proxy admits a connection only after its first request/// head carries the per-launch token (see `VPhoneAPIRequestGate`).+///+/// One admitted route is answered here instead of in the guest:+/// `/v1/host/screen` (`VPhoneHostScreenService`).@MainActorpublic final class VPhoneAPIProxy {public enum ProxyError: Error, CustomStringConvertible {@@ -36,9 +39,10 @@ public final class VPhoneAPIProxy {private let port: Intprivate let group = MultiThreadedEventLoopGroup(numberOfThreads: 2)private let provider: GuestSocketProvider+ private let hostScreen: VPhoneHostScreenService?private var server: Channel?- public init(device: VZVirtioSocketDevice, listen: String) throws {+ init(device: VZVirtioSocketDevice, listen: String, hostScreen: VPhoneHostScreenService? = nil) throws {guard let components = URLComponents(string: "tcp://\(listen)"),let host = components.host, !host.isEmpty,let port = components.port, (0 ... 65535).contains(port),@@ -47,6 +51,7 @@ public final class VPhoneAPIProxy {self.host = hostself.port = portprovider = GuestSocketProvider(device: device, group: group)+ self.hostScreen = hostScreenif !Self.isLoopback(host) {print("[api] warning: \(host) is not a loopback address; other machines can reach the guest API with the token")}@@ -65,11 +70,14 @@ public final class VPhoneAPIProxy {// with `Host: localhost`.let forwardedHost = "localhost"let provider = provider+ let hostScreen = hostScreenlet channel = try await ServerBootstrap(group: group).serverChannelOption(ChannelOptions.backlog, value: 128).childChannelOption(ChannelOptions.autoRead, value: false).childChannelInitializer { channel in- channel.pipeline.addHandler(APIAdmission(token: token, host: forwardedHost, provider: provider))+ channel.pipeline.addHandler(APIAdmission(+ token: token, host: forwardedHost, provider: provider, hostScreen: hostScreen,+ ))}.bind(host: host, port: port).get()@@ -145,15 +153,17 @@ private final class APIAdmission: ChannelInboundHandler, @unchecked Sendable {private let token: Stringprivate let host: String?private let provider: GuestSocketProvider+ private let hostScreen: VPhoneHostScreenService?private var state = State.readingprivate var received = Data()private var pending = Data()private var timeout: Scheduled<Void>?- init(token: String, host: String?, provider: GuestSocketProvider) {+ init(token: String, host: String?, provider: GuestSocketProvider, hostScreen: VPhoneHostScreenService?) {self.token = tokenself.host = hostself.provider = provider+ self.hostScreen = hostScreen}func channelActive(context: ChannelHandlerContext) {@@ -190,6 +200,23 @@ private final class APIAdmission: ChannelInboundHandler, @unchecked Sendable {}case let .accept(bytes):finish()+ if let hostScreen, VPhoneAPIRequestGate.hostRoute(of: bytes) == .screen {+ // Answered in this process; later reads go to the handlers+ // the service adds behind this one.+ state = .relaying+ let channel = context.channel+ do {+ try hostScreen.attach(context, admitted: bytes)+ channel.setOption(ChannelOptions.autoRead, value: true).whenFailure { _ in+ channel.close(promise: nil)+ }+ } catch {+ print("[api] host screen: \(error)")+ state = .closed+ channel.close(promise: nil)+ }+ return+ }state = .connectingpending = bytesprovider.attach(context.channel, admission: self)diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneHostScreenService.swift b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneHostScreenService.swiftnew file mode 100644index 00000000..dcc7ab63--- /dev/null+++ b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneHostScreenService.swift@@ -0,0 +1,293 @@+import Foundation+import NIOCore+import NIOHTTP1+import NIOWebSocket+import Virtualization+import VPhoneCoreKit++// MARK: - Host Screen Service++/// Serves `GET /v1/host/screen` on the API proxy (`VPhoneHostScreenProtocol`):+/// frames of the VM's own display, encoded in this process. The guest takes no+/// screenshot for it; it is asked only for its interface orientation, about+/// once a second while a stream runs.+@MainActor+final class VPhoneHostScreenService {+ private let display: @MainActor () -> VZGraphicsDisplay?+ private let interfaceDegrees: @MainActor () async -> Int?+ /// Image pixels per interface point of the upright display.+ nonisolated let scale: Double++ private var degrees = 0+ /// The first answer, or failure, has come back.+ private var answered = false+ private var checking = false+ private var checkedAt: UInt64 = 0++ init(+ scale: Double,+ display: @escaping @MainActor () -> VZGraphicsDisplay?,+ interfaceDegrees: @escaping @MainActor () async -> Int?,+ ) {+ self.scale = scale+ self.display = display+ self.interfaceDegrees = interfaceDegrees+ }++ /// The current frame and how far it turns clockwise to stand upright.+ func capture() async -> (image: CGImage, turn: Int)? {+ await refreshOrientation()+ guard let display = display(), let image = await VPhoneGraphicsDisplayCapture.capture(display) else {+ return nil+ }+ return (image, VPhoneHostScreenProtocol.clockwiseTurn(forInterfaceDegrees: degrees))+ }++ /// The guest is asked at most once a second, behind the frames. Only the+ /// first frame waits for the answer, briefly, so a landscape guest does not+ /// start with a sideways session; a guest that does not answer keeps the+ /// last value (portrait at first).+ private func refreshOrientation() async {+ let now = DispatchTime.now().uptimeNanoseconds+ if !checking, !answered || now - checkedAt >= 1_000_000_000 {+ checking = true+ checkedAt = now+ Task { @MainActor in+ if let value = await self.interfaceDegrees() {+ self.degrees = value+ }+ self.answered = true+ self.checking = false+ }+ }+ for _ in 0 ..< 50 where !answered {+ try? await Task.sleep(for: .milliseconds(10))+ }+ }++ // MARK: Pipeline++ /// Takes over a connection the proxy admitted for the screen route: the+ /// admitted head goes through an HTTP server pipeline that upgrades it to+ /// a WebSocket. Call on the channel's event loop, before it reads again.+ nonisolated func attach(_ context: ChannelHandlerContext, admitted: Data) throws {+ let service = self+ let upgrader = NIOWebSocketServerUpgrader(+ maxFrameSize: 1 << 16,+ shouldUpgrade: { channel, request in+ // A client that gave the proxy its token as a subprotocol+ // expects the server to select that protocol, as vphoned does.+ var headers = HTTPHeaders()+ if let offered = request.headers[canonicalForm: "Sec-WebSocket-Protocol"]+ .first(where: { $0.hasPrefix(VPhoneAPIRequestGate.webSocketProtocolPrefix) })+ {+ headers.add(name: "Sec-WebSocket-Protocol", value: String(offered))+ }+ return channel.eventLoop.makeSucceededFuture(headers)+ },+ upgradePipelineHandler: { channel, _ in+ channel.eventLoop.makeCompletedFuture {+ try channel.pipeline.syncOperations.addHandlers([+ NIOWebSocketFrameAggregator(+ minNonFinalFragmentSize: 1,+ maxAccumulatedFrameCount: 8,+ maxAccumulatedFrameSize: 1 << 16,+ ),+ HostScreenSocket(service: service),+ ])+ }+ },+ )+ let refusal = HostScreenRefusal()+ let sync = context.pipeline.syncOperations+ try sync.configureHTTPServerPipeline(+ withServerUpgrade: (upgraders: [upgrader], completionHandler: { context in+ context.pipeline.syncOperations.removeHandler(refusal, promise: nil)+ }),+ )+ try sync.addHandler(refusal)+ var buffer = context.channel.allocator.buffer(capacity: admitted.count)+ buffer.writeBytes(admitted)+ context.fireChannelRead(NIOAny(buffer))+ }+}++// MARK: - Refusal++/// Answers a request on the screen route that is not a WebSocket upgrade.+private final class HostScreenRefusal: ChannelInboundHandler, RemovableChannelHandler, @unchecked Sendable {+ typealias InboundIn = HTTPServerRequestPart+ typealias OutboundOut = HTTPServerResponsePart++ func channelRead(context: ChannelHandlerContext, data: NIOAny) {+ guard case .head = unwrapInboundIn(data) else { return }+ let body = #"{"error":{"code":"upgrade_required","message":"/v1/host/screen is a WebSocket"}}"#+ var headers = HTTPHeaders()+ headers.add(name: "Content-Type", value: "application/json")+ headers.add(name: "Content-Length", value: "\(body.utf8.count)")+ headers.add(name: "Connection", value: "close")+ let head = HTTPResponseHead(version: .http1_1, status: .upgradeRequired, headers: headers)+ var buffer = context.channel.allocator.buffer(capacity: body.utf8.count)+ buffer.writeString(body)+ let channel = context.channel+ context.write(wrapOutboundOut(.head(head)), promise: nil)+ context.write(wrapOutboundOut(.body(.byteBuffer(buffer))), promise: nil)+ context.writeAndFlush(wrapOutboundOut(.end(nil))).whenComplete { _ in+ channel.close(promise: nil)+ }+ }+}++// MARK: - Socket++/// One client. Requests are answered one at a time, in order: each takes a+/// frame on the main actor and encodes it on this client's queue.+private final class HostScreenSocket: ChannelInboundHandler, @unchecked Sendable {+ typealias InboundIn = WebSocketFrame+ typealias OutboundOut = WebSocketFrame++ /// A client keeps one or two requests outstanding; more than this is one+ /// that does not wait for answers.+ private static let maximumQueued = 16++ private let service: VPhoneHostScreenService+ private let work = DispatchQueue(label: "vphone.hostscreen.encode")+ private let encoder = VPhoneHostScreenEncoder()+ private let lock = NSLock()+ private var queued: [VPhoneHostScreenProtocol.Request] = []+ private var busy = false+ private var closed = false+ private var channel: Channel?++ init(service: VPhoneHostScreenService) {+ self.service = service+ }++ func handlerAdded(context: ChannelHandlerContext) {+ channel = context.channel+ }++ func channelRead(context: ChannelHandlerContext, data: NIOAny) {+ let frame = unwrapInboundIn(data)+ switch frame.opcode {+ case .text:+ var buffer = frame.unmaskedData+ let bytes = Data(buffer.readBytes(length: buffer.readableBytes) ?? [])+ do {+ enqueue(try VPhoneHostScreenProtocol.parse(bytes))+ } catch {+ send(VPhoneHostScreenProtocol.errorMessage(id: nil, code: error.code, message: error.description))+ }+ case .ping:+ context.writeAndFlush(wrapOutboundOut(WebSocketFrame(fin: true, opcode: .pong, data: frame.unmaskedData)), promise: nil)+ case .connectionClose:+ let channel = context.channel+ let echo = WebSocketFrame(fin: true, opcode: .connectionClose, data: frame.unmaskedData)+ context.writeAndFlush(wrapOutboundOut(echo)).whenComplete { _ in+ channel.close(promise: nil)+ }+ default:+ break+ }+ }++ func channelInactive(context: ChannelHandlerContext) {+ lock.withLock {+ closed = true+ queued.removeAll()+ }+ work.async { [encoder] in+ encoder.close()+ }+ context.fireChannelInactive()+ }++ func errorCaught(context: ChannelHandlerContext, error _: Error) {+ context.close(promise: nil)+ }++ private func enqueue(_ request: VPhoneHostScreenProtocol.Request) {+ let start = lock.withLock { () -> Bool? in+ guard !closed else { return nil }+ guard queued.count < Self.maximumQueued else { return nil }+ queued.append(request)+ guard !busy else { return false }+ busy = true+ return true+ }+ switch start {+ case nil:+ send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: "too_many_requests",+ message: "at most \(Self.maximumQueued) requests may wait"))+ case true?:+ next()+ case false?:+ break+ }+ }++ private func next() {+ let request = lock.withLock { () -> VPhoneHostScreenProtocol.Request? in+ guard !closed, !queued.isEmpty else {+ busy = false+ return nil+ }+ return queued.removeFirst()+ }+ guard let request else { return }+ let service = service+ Task { @MainActor in+ let started = DispatchTime.now().uptimeNanoseconds+ let captured = await service.capture()+ let captureMicroseconds = Int((DispatchTime.now().uptimeNanoseconds - started) / 1000)+ nonisolated(unsafe) let image = captured?.image+ let turn = captured?.turn ?? 0+ self.work.async {+ self.answer(request, image: image, turn: turn, captureMicroseconds: captureMicroseconds)+ self.next()+ }+ }+ }++ private func answer(_ request: VPhoneHostScreenProtocol.Request, image: CGImage?, turn: Int, captureMicroseconds: Int) {+ guard let image else {+ send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: "display_unavailable",+ message: "the VM display returned no frame"))+ return+ }+ do {+ let result = try encoder.encode(image, request: request, turn: turn)+ var header: [String: Any] = [+ "id": request.id,+ "status": result.status.rawValue,+ "session": result.session,+ "key": result.key,+ "width": result.width,+ "height": result.height,+ "sourceWidth": result.sourceWidth,+ "sourceHeight": result.sourceHeight,+ "scale": service.scale,+ "captureUs": captureMicroseconds,+ "encodeUs": result.encodeMicroseconds,+ ]+ if let record = result.record {+ header["record"] = record.base64EncodedString()+ }+ send(VPhoneHostScreenProtocol.message(header: header, payload: result.payload))+ } catch {+ // The next request starts a fresh session.+ encoder.close()+ send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: error.code, message: error.description))+ }+ }++ private func send(_ message: Data) {+ guard let channel = lock.withLock({ closed ? nil : channel }) else { return }+ channel.eventLoop.execute {+ guard channel.isActive else { return }+ var buffer = channel.allocator.buffer(capacity: message.count)+ buffer.writeBytes(message)+ channel.writeAndFlush(WebSocketFrame(fin: true, opcode: .binary, data: buffer), promise: nil)+ }+ }+}diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneGraphicsDisplayCapture.swift b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneGraphicsDisplayCapture.swiftnew file mode 100644index 00000000..42a1b27d--- /dev/null+++ b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneGraphicsDisplayCapture.swift@@ -0,0 +1,52 @@+import AppKit+import ObjectiveC.runtime+import Virtualization++// MARK: - Graphics Display Capture++/// The current frame of a VM display, from Virtualization's private+/// `_takeScreenshotWithCompletionHandler:`. The VM service answers it from the+/// frame it last presented, so the guest does no work for it, and it works the+/// same with and without a VM window. The buffer is in the panel's+/// orientation, whatever the guest interface shows.+enum VPhoneGraphicsDisplayCapture {+ private typealias CompletionBlock = @convention(block) (AnyObject?) -> Void+ private typealias ScreenshotIMP = @convention(c) (AnyObject, Selector, AnyObject) -> Void++ @MainActor+ static func capture(_ display: VZGraphicsDisplay, completion: @escaping (CGImage?) -> Void) {+ let selector = NSSelectorFromString("_takeScreenshotWithCompletionHandler:")+ guard display.responds(to: selector),+ let cls = object_getClass(display),+ let method = class_getInstanceMethod(cls, selector)+ else {+ completion(nil)+ return+ }+ let function = unsafeBitCast(method_getImplementation(method), to: ScreenshotIMP.self)+ let block: CompletionBlock = { object in+ completion(image(from: object))+ }+ function(display, selector, unsafeBitCast(block, to: AnyObject.self))+ }++ @MainActor+ static func capture(_ display: VZGraphicsDisplay) async -> CGImage? {+ await withCheckedContinuation { continuation in+ capture(display) { image in+ continuation.resume(returning: image)+ }+ }+ }++ private static func image(from object: AnyObject?) -> CGImage? {+ guard let object else { return nil }+ if let image = object as? NSImage {+ return image.cgImage(forProposedRect: nil, context: nil, hints: nil)+ }+ let value = object as CFTypeRef+ guard CFGetTypeID(value) == CGImage.typeID else { return nil }+ // CGImage is a CF type, not a Swift class - unsafeDowncast cannot be used here.+ return (value as! CGImage) // swiftlint:disable:this force_cast+ }+}diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneHostScreenEncoder.swift b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneHostScreenEncoder.swiftnew file mode 100644index 00000000..c9433430--- /dev/null+++ b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneHostScreenEncoder.swift@@ -0,0 +1,359 @@+import Accelerate+import CoreMedia+import CoreVideo+import Foundation+import VideoToolbox+import VPhoneCoreKit++// MARK: - Host Screen Encoder++/// Turns display frames into one video stream: upright, fitted into the+/// requested box and encoded by the hardware encoder. It serves one client and+/// is called from one queue at a time; `encode` returns that frame's result.+///+/// Colour follows the screen helper's own encoder (Display P3 primaries, sRGB+/// transfer, BT.709 matrix, video range), so a consumer configured for the+/// helper's stream decodes this one unchanged.+///+/// Unchecked: all of its state is used from the one queue its client owns.+final class VPhoneHostScreenEncoder: @unchecked Sendable {+ typealias Codec = VPhoneHostScreenProtocol.Codec++ enum Status: String {+ case frame+ case unchanged+ case dropped+ }++ struct Result {+ var status: Status+ var session: Int+ var key = false+ var width: Int+ var height: Int+ var sourceWidth: Int+ var sourceHeight: Int+ var payload = Data()+ /// The decoder configuration record, on a session's first frame.+ var record: Data?+ var encodeMicroseconds = 0+ }++ struct Failure: Error, CustomStringConvertible {+ let code: String+ let description: String+ }++ private struct Layout: Equatable {+ var codec: Codec+ var imageWidth: Int+ var imageHeight: Int+ var turn: Int+ var boxWidth: Int+ var boxHeight: Int+ }++ private var layout: Layout?+ private var sessions = 0+ private var encoder: VTCompressionSession?+ private var transfer: VTPixelTransferSession?+ private var rotator: VTPixelRotationSession?+ private var sourcePool: CVPixelBufferPool?+ private var scaledPool: CVPixelBufferPool?+ private var output = (width: 0, height: 0, sourceWidth: 0, sourceHeight: 0)+ private var bitrate = 0+ private var fps = 0+ /// The pixels of the last frame encoded, to tell a still screen apart.+ private var lastPixels: CFData?+ private var sentRecord = false+ private var frameIndex: Int64 = 0++ deinit {+ close()+ }++ func close() {+ if let encoder {+ VTCompressionSessionInvalidate(encoder)+ }+ encoder = nil+ transfer.map { VTPixelTransferSessionInvalidate($0) }+ transfer = nil+ rotator.map { VTPixelRotationSessionInvalidate($0) }+ rotator = nil+ layout = nil+ }++ /// `turn` is how far the panel buffer turns clockwise to stand upright.+ func encode(_ image: CGImage, request: VPhoneHostScreenProtocol.Request, turn: Int) throws(Failure) -> Result {+ guard let data = image.dataProvider?.data, let base = CFDataGetBytePtr(data) else {+ throw Failure(code: "display_unavailable", description: "the display frame has no pixels")+ }+ let wanted = Layout(+ codec: request.codec, imageWidth: image.width, imageHeight: image.height,+ turn: turn, boxWidth: request.boxWidth, boxHeight: request.boxHeight,+ )+ if wanted != layout || encoder == nil {+ try open(wanted, request: request)+ } else if request.bitrate != bitrate {+ bitrate = request.bitrate+ VTSessionSetProperty(encoder!, key: kVTCompressionPropertyKey_AverageBitRate, value: bitrate as CFNumber)+ }++ var result = Result(+ status: .unchanged, session: sessions, width: output.width, height: output.height,+ sourceWidth: output.sourceWidth, sourceHeight: output.sourceHeight,+ )+ let key = request.key || !sentRecord+ // Every byte is compared: a sampled hash misses a text cursor.+ guard key || !Self.same(data, lastPixels) else { return result }++ let source = try sourceBuffer(image, base: base)+ let picture = try convert(source)+ let started = DispatchTime.now().uptimeNanoseconds+ let encoded = try compress(picture, key: key)+ result.encodeMicroseconds = Int((DispatchTime.now().uptimeNanoseconds - started) / 1000)+ guard let encoded else {+ result.status = .dropped+ return result+ }+ lastPixels = data+ result.status = .frame+ result.key = encoded.key+ result.payload = encoded.payload+ if encoded.key {+ result.record = encoded.record+ sentRecord = sentRecord || encoded.record != nil+ }+ return result+ }++ // MARK: Sessions++ private func check(_ status: OSStatus, _ what: String, code: String = "encoder_failed") throws(Failure) {+ guard status == noErr else { throw Failure(code: code, description: "\(what) failed (\(status))") }+ }++ private func open(_ wanted: Layout, request: VPhoneHostScreenProtocol.Request) throws(Failure) {+ close()+ let sideways = wanted.turn == 90 || wanted.turn == 270+ let sourceWidth = sideways ? wanted.imageHeight : wanted.imageWidth+ let sourceHeight = sideways ? wanted.imageWidth : wanted.imageHeight+ let size = VPhoneHostScreenProtocol.outputSize(+ sourceWidth: sourceWidth, sourceHeight: sourceHeight,+ boxWidth: wanted.boxWidth, boxHeight: wanted.boxHeight,+ )+ output = (size.width, size.height, sourceWidth, sourceHeight)++ var transfer: VTPixelTransferSession?+ try check(VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer), "pixel transfer session")+ guard let transfer else { throw Failure(code: "encoder_failed", description: "no pixel transfer session") }+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_ScalingMode, value: kVTScalingMode_Normal)+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_DestinationColorPrimaries, value: kCVImageBufferColorPrimaries_P3_D65)+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_DestinationTransferFunction, value: kCVImageBufferTransferFunction_sRGB)+ VTSessionSetProperty(transfer, key: kVTPixelTransferPropertyKey_DestinationYCbCrMatrix, value: kCVImageBufferYCbCrMatrix_ITU_R_709_2)+ self.transfer = transfer++ sourcePool = try Self.pool(kCVPixelFormatType_32BGRA, wanted.imageWidth, wanted.imageHeight)+ if wanted.turn != 0 {+ var rotator: VTPixelRotationSession?+ try check(VTPixelRotationSessionCreate(nil, &rotator), "pixel rotation session")+ guard let rotator else { throw Failure(code: "encoder_failed", description: "no pixel rotation session") }+ let angle: CFString = switch wanted.turn {+ case 90: kVTRotation_CW90+ case 180: kVTRotation_180+ default: kVTRotation_CCW90+ }+ VTSessionSetProperty(rotator, key: kVTPixelRotationPropertyKey_Rotation, value: angle)+ self.rotator = rotator+ let scaledWidth = sideways ? size.height : size.width+ let scaledHeight = sideways ? size.width : size.height+ scaledPool = try Self.pool(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, scaledWidth, scaledHeight)+ } else {+ scaledPool = nil+ }++ try openEncoder(wanted.codec, width: size.width, height: size.height, request: request)+ layout = wanted+ sessions += 1+ sentRecord = false+ lastPixels = nil+ frameIndex = 0+ }++ private func openEncoder(_ codec: Codec, width: Int, height: Int, request: VPhoneHostScreenProtocol.Request) throws(Failure) {+ let specification: [CFString: Any] = [+ kVTVideoEncoderSpecification_EnableLowLatencyRateControl: true,+ kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: true,+ ]+ let attributes: [CFString: Any] = [+ kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,+ kCVPixelBufferWidthKey: width,+ kCVPixelBufferHeightKey: height,+ kCVPixelBufferIOSurfacePropertiesKey: [CFString: Any](),+ ]+ var session: VTCompressionSession?+ let created = VTCompressionSessionCreate(+ allocator: nil, width: Int32(width), height: Int32(height),+ codecType: codec == .hevc ? kCMVideoCodecType_HEVC : kCMVideoCodecType_H264,+ encoderSpecification: specification as CFDictionary,+ imageBufferAttributes: attributes as CFDictionary,+ compressedDataAllocator: nil, outputCallback: nil, refcon: nil,+ compressionSessionOut: &session,+ )+ guard created == noErr, let session else {+ throw Failure(code: "codec_unavailable", description: "no hardware \(codec.rawValue) encoder for \(width)x\(height) (\(created))")+ }+ // Required: with reordering a frame would not come out of the call+ // that put it in, and encode() answers each request with its own frame.+ guard VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse) == noErr else {+ VTCompressionSessionInvalidate(session)+ throw Failure(code: "codec_unavailable", description: "the \(codec.rawValue) encoder cannot disable frame reordering")+ }+ bitrate = request.bitrate+ fps = request.fps+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,+ value: codec == .hevc ? kVTProfileLevel_HEVC_Main_AutoLevel : kVTProfileLevel_H264_High_AutoLevel)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ColorPrimaries, value: kCVImageBufferColorPrimaries_P3_D65)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_TransferFunction, value: kCVImageBufferTransferFunction_sRGB)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_YCbCrMatrix, value: kCVImageBufferYCbCrMatrix_ITU_R_709_2)+ // Key frames come when the client asks for one.+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: 36000 as CFNumber)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: fps as CFNumber)+ VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: bitrate as CFNumber)+ guard VTCompressionSessionPrepareToEncodeFrames(session) == noErr else {+ VTCompressionSessionInvalidate(session)+ throw Failure(code: "codec_unavailable", description: "the \(codec.rawValue) encoder did not start")+ }+ encoder = session+ }++ private static func pool(_ format: OSType, _ width: Int, _ height: Int) throws(Failure) -> CVPixelBufferPool {+ let attributes: [CFString: Any] = [+ kCVPixelBufferPixelFormatTypeKey: format,+ kCVPixelBufferWidthKey: width,+ kCVPixelBufferHeightKey: height,+ kCVPixelBufferIOSurfacePropertiesKey: [CFString: Any](),+ ]+ var pool: CVPixelBufferPool?+ let status = CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary, &pool)+ guard status == kCVReturnSuccess, let pool else {+ throw Failure(code: "encoder_failed", description: "no \(width)x\(height) buffer pool (\(status))")+ }+ return pool+ }++ private static func buffer(_ pool: CVPixelBufferPool?) throws(Failure) -> CVPixelBuffer {+ var buffer: CVPixelBuffer?+ guard let pool, CVPixelBufferPoolCreatePixelBuffer(nil, pool, &buffer) == kCVReturnSuccess, let buffer else {+ throw Failure(code: "encoder_failed", description: "no pixel buffer")+ }+ return buffer+ }++ // MARK: Pixels++ /// The frame as a BGRA IOSurface. Pixel transfer refuses the display's+ /// RGBA bytes as they are, so one pass swaps the channels while copying.+ private func sourceBuffer(_ image: CGImage, base: UnsafePointer<UInt8>) throws(Failure) -> CVPixelBuffer {+ let buffer = try Self.buffer(sourcePool)+ CVPixelBufferLockBaseAddress(buffer, [])+ var source = vImage_Buffer(+ data: UnsafeMutableRawPointer(mutating: base),+ height: vImagePixelCount(image.height), width: vImagePixelCount(image.width),+ rowBytes: image.bytesPerRow,+ )+ var destination = vImage_Buffer(+ data: CVPixelBufferGetBaseAddress(buffer),+ height: vImagePixelCount(image.height), width: vImagePixelCount(image.width),+ rowBytes: CVPixelBufferGetBytesPerRow(buffer),+ )+ let order: [UInt8] = switch image.bitmapInfo.intersection(.byteOrderMask) {+ case .byteOrder32Little: [0, 1, 2, 3]+ default: [2, 1, 0, 3]+ }+ let error = vImagePermuteChannels_ARGB8888(&source, &destination, order, vImage_Flags(kvImageNoFlags))+ CVPixelBufferUnlockBaseAddress(buffer, [])+ guard error == kvImageNoError else {+ throw Failure(code: "encoder_failed", description: "the frame could not be copied (\(error))")+ }+ if let space = image.colorSpace {+ CVBufferSetAttachment(buffer, kCVImageBufferCGColorSpaceKey, space, .shouldPropagate)+ }+ return buffer+ }++ private func convert(_ source: CVPixelBuffer) throws(Failure) -> CVPixelBuffer {+ guard let transfer, let encoder else { throw Failure(code: "encoder_failed", description: "no session") }+ let picture = try Self.buffer(VTCompressionSessionGetPixelBufferPool(encoder))+ if let rotator {+ let scaled = try Self.buffer(scaledPool)+ try check(VTPixelTransferSessionTransferImage(transfer, from: source, to: scaled), "pixel transfer")+ try check(VTPixelRotationSessionRotateImage(rotator, scaled, picture), "pixel rotation")+ } else {+ try check(VTPixelTransferSessionTransferImage(transfer, from: source, to: picture), "pixel transfer")+ }+ return picture+ }++ // MARK: Encoding++ private final class Output: @unchecked Sendable {+ let lock = NSLock()+ var status: OSStatus = noErr+ var sample: CMSampleBuffer?+ var done = false+ }++ private func compress(_ picture: CVPixelBuffer, key: Bool) throws(Failure) -> (payload: Data, key: Bool, record: Data?)? {+ guard let encoder else { throw Failure(code: "encoder_failed", description: "no session") }+ let pts = CMTime(value: frameIndex, timescale: CMTimeScale(max(fps, 1)))+ frameIndex += 1+ let properties: CFDictionary? = key ? [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary : nil+ let output = Output()+ let submitted = VTCompressionSessionEncodeFrame(+ encoder, imageBuffer: picture, presentationTimeStamp: pts, duration: .invalid,+ frameProperties: properties, infoFlagsOut: nil,+ ) { status, _, sample in+ output.lock.withLock {+ output.status = status+ output.sample = sample+ output.done = true+ }+ }+ try check(submitted, "encode")+ try check(VTCompressionSessionCompleteFrames(encoder, untilPresentationTimeStamp: pts), "complete frame")+ let (status, sample, done) = output.lock.withLock { (output.status, output.sample, output.done) }+ try check(status, "encode")+ guard done, let sample, let block = CMSampleBufferGetDataBuffer(sample) else {+ return nil+ }+ let length = CMBlockBufferGetDataLength(block)+ var payload = Data(count: length)+ let copied = payload.withUnsafeMutableBytes { raw in+ CMBlockBufferCopyDataBytes(block, atOffset: 0, dataLength: length, destination: raw.baseAddress!)+ }+ try check(copied, "read the encoded frame")+ var isKey = true+ if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]],+ let first = attachments.first, first[kCMSampleAttachmentKey_NotSync] as? Bool == true+ {+ isKey = false+ }+ var record: Data?+ if isKey, let description = CMSampleBufferGetFormatDescription(sample),+ let atoms = CMFormatDescriptionGetExtension(description, extensionKey: kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms) as? [String: Any]+ {+ record = (atoms["hvcC"] ?? atoms["avcC"]) as? Data+ }+ return (payload, isKey, record)+ }++ private static func same(_ data: CFData, _ other: CFData?) -> Bool {+ guard let other, CFDataGetLength(data) == CFDataGetLength(other),+ let a = CFDataGetBytePtr(data), let b = CFDataGetBytePtr(other)+ else { return false }+ return memcmp(a, b, CFDataGetLength(data)) == 0+ }+}diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneScreenRecorder.swift b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneScreenRecorder.swiftindex b0c632ca..402680bc 100644--- a/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneScreenRecorder.swift+++ b/VPhoneExecutable/VPhoneVirtualization/UI/HostDevices/VPhoneScreenRecorder.swift@@ -2,7 +2,6 @@ import AppKitimport AVFoundationimport CoreVideoimport ImageIO-import ObjectiveC.runtimeimport Virtualization// MARK: - Screen Recorder@@ -26,8 +25,6 @@ class VPhoneScreenRecorder {}}- private typealias ScreenshotCompletionBlock = @convention(block) (AnyObject?) -> Void- private typealias ScreenshotIMP = @convention(c) (AnyObject, Selector, AnyObject) -> Voidprivate var writer: AVAssetWriter?private var videoInput: AVAssetWriterInput?@@ -247,47 +244,11 @@ class VPhoneScreenRecorder {from graphicsDisplay: VZGraphicsDisplay,completion: @escaping (CGImage?) -> Void,) {- let selector = NSSelectorFromString("_takeScreenshotWithCompletionHandler:")- guard graphicsDisplay.responds(to: selector),- let cls = object_getClass(graphicsDisplay),- let method = class_getInstanceMethod(cls, selector)- else {- completion(nil)- return- }-- let implementation = method_getImplementation(method)- let function = unsafeBitCast(implementation, to: ScreenshotIMP.self)-- let block: ScreenshotCompletionBlock = { [weak self] imageObject in- completion(self?.convertScreenshotObject(imageObject))- }- let blockObject = unsafeBitCast(block, to: AnyObject.self)- function(graphicsDisplay, selector, blockObject)+ VPhoneGraphicsDisplayCapture.capture(graphicsDisplay, completion: completion)}private func takeGraphicsScreenshot(from graphicsDisplay: VZGraphicsDisplay) async -> CGImage? {- await withCheckedContinuation { continuation in- takeGraphicsScreenshot(from: graphicsDisplay) { cgImage in- continuation.resume(returning: cgImage)- }- }- }-- private func convertScreenshotObject(_ imageObject: AnyObject?) -> CGImage? {- guard let imageObject else { return nil }-- if let nsImage = imageObject as? NSImage {- return nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil)- }-- let cfObject = imageObject as CFTypeRef- if CFGetTypeID(cfObject) == CGImage.typeID {- // CGImage is a CF type, not a Swift class - unsafeDowncast cannot be used here.- return (cfObject as! CGImage) // swiftlint:disable:this force_cast- }-- return nil+ await VPhoneGraphicsDisplayCapture.capture(graphicsDisplay)}private func recordingOutputURL() -> URL {diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/VPhoneVirtualMachineAppDelegate.swift b/VPhoneExecutable/VPhoneVirtualization/UI/VPhoneVirtualMachineAppDelegate.swiftindex eb54b53d..5871dd2b 100644--- a/VPhoneExecutable/VPhoneVirtualization/UI/VPhoneVirtualMachineAppDelegate.swift+++ b/VPhoneExecutable/VPhoneVirtualization/UI/VPhoneVirtualMachineAppDelegate.swift@@ -136,7 +136,17 @@ class VPhoneVirtualMachineAppDelegate: NSObject, NSApplicationDelegate {control.connect(device: device)camServer.connect(device: device)if let listen = command.apiListen {- let proxy = try VPhoneAPIProxy(device: device, listen: listen)+ let hostScreen = VPhoneHostScreenService(+ scale: options.screenScale,+ display: { [weak vm] in vm?.virtualMachine.graphicsDevices.first?.displays.first },+ interfaceDegrees: { [weak control] in+ guard let control, control.isConnected,+ let result = try? await control.call("display.orientation")+ else { return nil }+ return (result["degrees"] as? NSNumber)?.intValue+ },+ )+ let proxy = try VPhoneAPIProxy(device: device, listen: listen, hostScreen: hostScreen)let (url, token) = try await proxy.start()apiProxy = proxyprint("[api] HTTP/WebSocket API: \(url.absoluteString)")diff --git a/VPhoneExecutable/VPhoneVirtualization/VPhoneVirtualization.xcodeproj/project.pbxproj b/VPhoneExecutable/VPhoneVirtualization/VPhoneVirtualization.xcodeproj/project.pbxprojindex 6caebdfb..8c477c33 100644--- a/VPhoneExecutable/VPhoneVirtualization/VPhoneVirtualization.xcodeproj/project.pbxproj+++ b/VPhoneExecutable/VPhoneVirtualization/VPhoneVirtualization.xcodeproj/project.pbxproj@@ -11,6 +11,8 @@DD0000000000000000000102 /* Dynamic in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000202 /* Dynamic */; };DD0000000000000000000103 /* NIOCore in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000203 /* NIOCore */; };DD0000000000000000000104 /* NIOPosix in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000204 /* NIOPosix */; };+ DD0000000000000000000107 /* NIOHTTP1 in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000205 /* NIOHTTP1 */; };+ DD0000000000000000000108 /* NIOWebSocket in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000206 /* NIOWebSocket */; };DD0000000000000000000105 /* libVPhoneVirtualMachineKit.a in Frameworks */ = {isa = PBXBuildFile; fileRef = DD0000000000000000000032 /* libVPhoneVirtualMachineKit.a */; };DD0000000000000000000106 /* ArgumentParser in Frameworks */ = {isa = PBXBuildFile; productRef = DD0000000000000000000201 /* ArgumentParser */; };/* End PBXBuildFile section */@@ -72,6 +74,8 @@DD0000000000000000000102 /* Dynamic in Frameworks */,DD0000000000000000000103 /* NIOCore in Frameworks */,DD0000000000000000000104 /* NIOPosix in Frameworks */,+ DD0000000000000000000107 /* NIOHTTP1 in Frameworks */,+ DD0000000000000000000108 /* NIOWebSocket in Frameworks */,);runOnlyForDeploymentPostprocessing = 0;};@@ -144,6 +148,8 @@DD0000000000000000000202 /* Dynamic */,DD0000000000000000000203 /* NIOCore */,DD0000000000000000000204 /* NIOPosix */,+ DD0000000000000000000205 /* NIOHTTP1 */,+ DD0000000000000000000206 /* NIOWebSocket */,);productName = VPhoneVirtualMachineKit;productReference = DD0000000000000000000032 /* libVPhoneVirtualMachineKit.a */;@@ -575,6 +581,16 @@package = DD0000000000000000000173 /* XCRemoteSwiftPackageReference "swift-nio" */;productName = NIOPosix;};+ DD0000000000000000000205 /* NIOHTTP1 */ = {+ isa = XCSwiftPackageProductDependency;+ package = DD0000000000000000000173 /* XCRemoteSwiftPackageReference "swift-nio" */;+ productName = NIOHTTP1;+ };+ DD0000000000000000000206 /* NIOWebSocket */ = {+ isa = XCSwiftPackageProductDependency;+ package = DD0000000000000000000173 /* XCRemoteSwiftPackageReference "swift-nio" */;+ productName = NIOWebSocket;+ };/* End XCSwiftPackageProductDependency section */};rootObject = DD0000000000000000000001 /* Project object */;diff --git a/VPhoneKit/VPhoneCoreKit/Support/VPhoneAPIRequestGate.swift b/VPhoneKit/VPhoneCoreKit/Support/VPhoneAPIRequestGate.swiftindex 65ff45fc..b3eff075 100644--- a/VPhoneKit/VPhoneCoreKit/Support/VPhoneAPIRequestGate.swift+++ b/VPhoneKit/VPhoneCoreKit/Support/VPhoneAPIRequestGate.swift@@ -96,6 +96,26 @@ public enum VPhoneAPIRequestGate {return .accept(forwarded)}+ // MARK: - Host Routes++ /// Paths the proxy answers itself instead of relaying them to vphoned.+ public enum HostRoute: Equatable, Sendable {+ /// `GET /v1/host/screen`: frames from the VM's own display.+ case screen+ }++ /// The host route an admitted head asks for, or nil to relay it to the+ /// guest. Only the path decides; a query is ignored.+ public static func hostRoute(of admitted: Data) -> HostRoute? {+ guard let end = admitted.range(of: Data("\r\n".utf8)),+ let line = String(data: admitted[admitted.startIndex ..< end.lowerBound], encoding: .ascii)+ else { return nil }+ let parts = line.split(separator: " ", omittingEmptySubsequences: false)+ guard parts.count == 3, parts[0] == "GET" else { return nil }+ let path = parts[1].split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false)[0]+ return path == "/v1/host/screen" ? .screen : nil+ }+// MARK: - Request Headstruct Head {diff --git a/VPhoneKit/VPhoneCoreKit/Support/VPhoneHostScreenProtocol.swift b/VPhoneKit/VPhoneCoreKit/Support/VPhoneHostScreenProtocol.swiftnew file mode 100644index 00000000..a17c7ad4--- /dev/null+++ b/VPhoneKit/VPhoneCoreKit/Support/VPhoneHostScreenProtocol.swift@@ -0,0 +1,168 @@+import Foundation++/// The host screen API, `GET /v1/host/screen` on the `--api-listen` proxy.+///+/// The VM's own display is captured and encoded by `vphone-vm` itself, so the+/// guest does no work for it: no screenshot, no JPEG, no base64. The client+/// pulls one frame per request, which keeps its own pacing and flow control.+///+/// Each request is a WebSocket text message:+///+/// {"id":7,"codec":"hevc","box":[1280,1280],"fps":60,"bitrate":4000000,"key":false}+///+/// and its answer one binary message: a 4-byte little-endian header length,+/// that many bytes of JSON header, then the payload. Requests are answered in+/// the order they arrive. The header always carries `id`, and then either+/// `error` (`{"code","message"}`) with no payload, or `status`:+///+/// - `frame`: the payload is one access unit as VideoToolbox writes it+/// (length-prefixed NAL units). `session` counts encoder sessions and+/// changes when the size, codec or orientation does; the first frame of a+/// session is a key frame. Every key frame carries the decoder configuration+/// record (`avcC` / `hvcC`) as base64 in `record`.+/// - `unchanged`: the display shows what the last frame showed and no key+/// frame was asked for; nothing was encoded.+/// - `dropped`: the encoder's rate control skipped the frame.+///+/// `width` and `height` are the encoded size, `sourceWidth`, `sourceHeight`+/// and `scale` the upright display in pixels and pixels per interface point.+public enum VPhoneHostScreenProtocol {+ public static let path = "/v1/host/screen"++ public enum Codec: String, Sendable {+ case hevc+ case h264+ }++ public struct Request: Equatable, Sendable {+ public var id: UInt64+ public var codec: Codec+ public var boxWidth: Int+ public var boxHeight: Int+ public var fps: Int+ public var bitrate: Int+ public var key: Bool++ public init(id: UInt64, codec: Codec, boxWidth: Int, boxHeight: Int, fps: Int, bitrate: Int, key: Bool) {+ self.id = id+ self.codec = codec+ self.boxWidth = boxWidth+ self.boxHeight = boxHeight+ self.fps = fps+ self.bitrate = bitrate+ self.key = key+ }+ }++ public struct RequestError: Error, Equatable, CustomStringConvertible {+ public let code: String+ public let description: String+ }++ static let maximumBox = 8192+ static let maximumFps = 240+ static let maximumBitrate = 100_000_000++ public static func parse(_ data: Data) throws(RequestError) -> Request {+ guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {+ throw RequestError(code: "bad_request", description: "a request is one JSON object")+ }+ guard let id = (object["id"] as? NSNumber).flatMap(integer), id >= 0 else {+ throw RequestError(code: "bad_request", description: "id must be a non-negative integer")+ }+ guard let name = object["codec"] as? String, let codec = Codec(rawValue: name) else {+ throw RequestError(code: "codec_unavailable", description: "codec must be hevc or h264")+ }+ guard let box = object["box"] as? [NSNumber], box.count == 2,+ let width = integer(box[0]), let height = integer(box[1]),+ (1 ... maximumBox).contains(width), (1 ... maximumBox).contains(height)+ else {+ throw RequestError(code: "bad_request", description: "box must be [width, height] within 1...\(maximumBox)")+ }+ guard let fps = (object["fps"] as? NSNumber).flatMap(integer), (1 ... maximumFps).contains(fps) else {+ throw RequestError(code: "bad_request", description: "fps must be within 1...\(maximumFps)")+ }+ guard let bitrate = (object["bitrate"] as? NSNumber).flatMap(integer),+ (1 ... maximumBitrate).contains(bitrate)+ else {+ throw RequestError(code: "bad_request", description: "bitrate must be within 1...\(maximumBitrate)")+ }+ let key = object["key"] as? Bool ?? false+ return Request(id: UInt64(id), codec: codec, boxWidth: width, boxHeight: height, fps: fps, bitrate: bitrate, key: key)+ }++ /// A JSON number that is a whole number; a boolean is not one.+ private static func integer(_ number: NSNumber) -> Int? {+ guard CFGetTypeID(number) != CFBooleanGetTypeID() else { return nil }+ let value = number.doubleValue+ guard value.rounded() == value, abs(value) < 1e15 else { return nil }+ return Int(value)+ }++ // MARK: - Geometry++ /// The output size for a source in a box: the screen helper's own rule+ /// (`media.OutputSize`), so a stream keeps the size it had when the helper+ /// scaled the guest's screenshots itself. The source is fitted without+ /// enlarging it, the short side is kept at 240 when the source allows, and+ /// both sides are rounded down to even numbers.+ public static func outputSize(sourceWidth: Int, sourceHeight: Int, boxWidth: Int, boxHeight: Int) -> (width: Int, height: Int) {+ let minimumShortSide = 240+ var width = sourceWidth+ var height = sourceHeight+ if width > boxWidth || height > boxHeight {+ width = boxWidth+ height = (sourceHeight * boxWidth + sourceWidth / 2) / sourceWidth+ if height > boxHeight {+ height = boxHeight+ width = (sourceWidth * boxHeight + sourceHeight / 2) / sourceHeight+ }+ }+ let short = min(width, height)+ if short < minimumShortSide, min(sourceWidth, sourceHeight) > short {+ let target = min(minimumShortSide, min(sourceWidth, sourceHeight))+ if sourceWidth <= sourceHeight {+ width = target+ height = (sourceHeight * target + sourceWidth / 2) / sourceWidth+ } else {+ width = (sourceWidth * target + sourceHeight / 2) / sourceHeight+ height = target+ }+ }+ return (max(width & ~1, 2), max(height & ~1, 2))+ }++ /// How far the panel-oriented display buffer turns clockwise to stand+ /// upright under an interface orientation in `display.orientation`'s+ /// degrees: under landscape-left (90) the top of the interface lies along+ /// the buffer's left edge. Unknown values leave the buffer as it is.+ public static func clockwiseTurn(forInterfaceDegrees degrees: Int) -> Int {+ switch degrees {+ case 90: 90+ case 180: 180+ case 270: 270+ default: 0+ }+ }++ // MARK: - Messages++ /// One answer: header length, JSON header, payload.+ public static func message(header: [String: Any], payload: Data = Data()) -> Data {+ let json = (try? JSONSerialization.data(withJSONObject: header, options: [.sortedKeys]))+ ?? Data(#"{"error":{"code":"internal","message":"header is not JSON"}}"#.utf8)+ var length = UInt32(json.count).littleEndian+ var message = Data(bytes: &length, count: 4)+ message.append(json)+ message.append(payload)+ return message+ }++ public static func errorMessage(id: UInt64?, code: String, message text: String) -> Data {+ var header: [String: Any] = ["error": ["code": code, "message": text]]+ if let id {+ header["id"] = id+ }+ return message(header: header)+ }+}diff --git a/VPhoneKit/VPhoneCoreKitTests/Support/APIRequestGateTests.swift b/VPhoneKit/VPhoneCoreKitTests/Support/APIRequestGateTests.swiftindex d766edcc..7785b67c 100644--- a/VPhoneKit/VPhoneCoreKitTests/Support/APIRequestGateTests.swift+++ b/VPhoneKit/VPhoneCoreKitTests/Support/APIRequestGateTests.swift@@ -133,6 +133,44 @@ struct APIRequestGateTests {}}+ // MARK: - Host Routes++ @Test func `the screen path is answered on the host`() {+ let admitted = request("""+ GET /v1/host/screen HTTP/1.1+ Host: localhost+ Upgrade: websocket++ """)+ #expect(Gate.hostRoute(of: admitted) == .screen)+ #expect(Gate.hostRoute(of: Data("GET /v1/host/screen?fps=60 HTTP/1.1\r\n\r\n".utf8)) == .screen)+ }++ @Test func `every other request is relayed to the guest`() {+ for head in [+ "GET /v1/events HTTP/1.1\r\n\r\n",+ "POST /v1/host/screen HTTP/1.1\r\n\r\n",+ "GET /v1/host/screen/extra HTTP/1.1\r\n\r\n",+ "GET /v1/host/screens HTTP/1.1\r\n\r\n",+ "GET /v1/host/screen",+ ] {+ #expect(Gate.hostRoute(of: Data(head.utf8)) == nil, "\(head.debugDescription)")+ }+ }++ @Test func `the route is read from the head the gate admitted`() {+ let data = request("""+ GET /v1/host/screen?token=\(token) HTTP/1.1+ Host: 127.0.0.1:18700++ """)+ guard case let .accept(admitted) = Gate.evaluate(data, token: token, host: "localhost") else {+ Issue.record("the head was not admitted")+ return+ }+ #expect(Gate.hostRoute(of: admitted) == .screen)+ }+// MARK: - Tokens@Test func `token validation and constant time comparison`() {diff --git a/VPhoneKit/VPhoneCoreKitTests/Support/HostScreenProtocolTests.swift b/VPhoneKit/VPhoneCoreKitTests/Support/HostScreenProtocolTests.swiftnew file mode 100644index 00000000..24a324fa--- /dev/null+++ b/VPhoneKit/VPhoneCoreKitTests/Support/HostScreenProtocolTests.swift@@ -0,0 +1,101 @@+import Foundation+import Testing+@testable import VPhoneCoreKit++struct HostScreenProtocolTests {+ typealias HostScreen = VPhoneHostScreenProtocol++ private func parse(_ json: String) throws(HostScreen.RequestError) -> HostScreen.Request {+ try HostScreen.parse(Data(json.utf8))+ }++ private func code(_ json: String) -> String? {+ do {+ _ = try parse(json)+ return nil+ } catch {+ return error.code+ }+ }++ // MARK: - Requests++ @Test func `a full request is read`() throws {+ let request = try parse(#"{"id":7,"codec":"hevc","box":[1280,800],"fps":60,"bitrate":4000000,"key":true}"#)+ #expect(request == HostScreen.Request(id: 7, codec: .hevc, boxWidth: 1280, boxHeight: 800, fps: 60, bitrate: 4_000_000, key: true))+ }++ @Test func `key is optional`() throws {+ let request = try parse(#"{"id":0,"codec":"h264","box":[2,2],"fps":1,"bitrate":1}"#)+ #expect(request.codec == .h264)+ #expect(!request.key)+ }++ @Test func `an unknown codec is its own error`() {+ #expect(code(#"{"id":1,"codec":"jpeg","box":[1,1],"fps":1,"bitrate":1}"#) == "codec_unavailable")+ #expect(code(#"{"id":1,"box":[1,1],"fps":1,"bitrate":1}"#) == "codec_unavailable")+ }++ @Test func `malformed requests are refused`() {+ for json in [+ "",+ "[]",+ #"{"codec":"hevc","box":[1,1],"fps":1,"bitrate":1}"#,+ #"{"id":-1,"codec":"hevc","box":[1,1],"fps":1,"bitrate":1}"#,+ #"{"id":1.5,"codec":"hevc","box":[1,1],"fps":1,"bitrate":1}"#,+ #"{"id":true,"codec":"hevc","box":[1,1],"fps":1,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[1],"fps":1,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[0,10],"fps":1,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[10,8193],"fps":1,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[10,10],"fps":0,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[10,10],"fps":241,"bitrate":1}"#,+ #"{"id":1,"codec":"hevc","box":[10,10],"fps":60}"#,+ #"{"id":1,"codec":"hevc","box":[10,10],"fps":60,"bitrate":100000001}"#,+ ] {+ #expect(code(json) == "bad_request", "\(json)")+ }+ }++ // MARK: - Geometry++ @Test func `output size matches the screen helper`() {+ // Each expected size is what the helper's media.OutputSize returns.+ let size = HostScreen.outputSize+ #expect(size(1488, 2266, 8192, 8192) == (1488, 2266))+ #expect(size(1488, 2266, 1280, 1280) == (840, 1280))+ #expect(size(2266, 1488, 1280, 1280) == (1280, 840))+ #expect(size(2266, 1488, 1280, 400) == (608, 400))+ #expect(size(1179, 2556, 50, 50) == (240, 520))+ #expect(size(100, 200, 50, 50) == (100, 200))+ #expect(size(1001, 1001, 8192, 8192) == (1000, 1000))+ }++ @Test func `the buffer turns with the interface`() {+ #expect(HostScreen.clockwiseTurn(forInterfaceDegrees: 0) == 0)+ #expect(HostScreen.clockwiseTurn(forInterfaceDegrees: 90) == 90)+ #expect(HostScreen.clockwiseTurn(forInterfaceDegrees: 180) == 180)+ #expect(HostScreen.clockwiseTurn(forInterfaceDegrees: 270) == 270)+ #expect(HostScreen.clockwiseTurn(forInterfaceDegrees: 45) == 0)+ }++ // MARK: - Messages++ @Test func `a message is length, header and payload`() throws {+ let data = HostScreen.message(header: ["id": 3, "status": "frame"], payload: Data([0xAA, 0xBB]))+ let length = Int(data.prefix(4).withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) }.littleEndian)+ let header = try #require(try JSONSerialization.jsonObject(with: data.dropFirst(4).prefix(length)) as? [String: Any])+ #expect(header["id"] as? Int == 3)+ #expect(header["status"] as? String == "frame")+ #expect(Array(data.suffix(from: 4 + length)) == [0xAA, 0xBB])+ }++ @Test func `an error carries no payload`() throws {+ let data = HostScreen.errorMessage(id: 9, code: "display_unavailable", message: "no display")+ let length = Int(data.prefix(4).withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) }.littleEndian)+ #expect(data.count == 4 + length)+ let header = try #require(try JSONSerialization.jsonObject(with: data.dropFirst(4)) as? [String: Any])+ let error = try #require(header["error"] as? [String: Any])+ #expect(header["id"] as? Int == 9)+ #expect(error["code"] as? String == "display_unavailable")+ }+}