Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift
11.7 KB
Raw Blame
import Foundation
import NIOCore
import NIOHTTP1
import NIOWebSocket
import Virtualization
import VPhoneCoreKit

// MARK: - Host Screen Service

/// Serves `GET /v1/host/screen` on the API proxy (`VPhoneHostScreenProtocol`):
/// frames of the VM's own display, encoded in this process. The guest takes no
/// screenshot for it; it is asked only for its interface orientation, about
/// once a second while a stream runs.
@MainActor
final class VPhoneHostScreenService {
    private let display: @MainActor () -> VZGraphicsDisplay?
    private let interfaceDegrees: @MainActor () async -> Int?
    /// Image pixels per interface point of the upright display.
    nonisolated let scale: Double

    private var degrees = 0
    /// The first answer, or failure, has come back.
    private var answered = false
    private var checking = false
    private var checkedAt: UInt64 = 0

    init(
        scale: Double,
        display: @escaping @MainActor () -> VZGraphicsDisplay?,
        interfaceDegrees: @escaping @MainActor () async -> Int?,
    ) {
        self.scale = scale
        self.display = display
        self.interfaceDegrees = interfaceDegrees
    }

    /// The current frame and how far it turns clockwise to stand upright.
    func capture() async -> (image: CGImage, turn: Int)? {
        await refreshOrientation()
        guard let display = display(), let image = await VPhoneGraphicsDisplayCapture.capture(display) else {
            return nil
        }
        return (image, VPhoneHostScreenProtocol.clockwiseTurn(forInterfaceDegrees: degrees))
    }

    /// The guest is asked at most once a second, behind the frames. Only the
    /// first frame waits for the answer, briefly, so a landscape guest does not
    /// start with a sideways session; a guest that does not answer keeps the
    /// last value (portrait at first).
    private func refreshOrientation() async {
        let now = DispatchTime.now().uptimeNanoseconds
        if !checking, !answered || now - checkedAt >= 1_000_000_000 {
            checking = true
            checkedAt = now
            Task { @MainActor in
                if let value = await self.interfaceDegrees() {
                    self.degrees = value
                }
                self.answered = true
                self.checking = false
            }
        }
        for _ in 0 ..< 50 where !answered {
            try? await Task.sleep(for: .milliseconds(10))
        }
    }

    // MARK: Pipeline

    /// Takes over a connection the proxy admitted for the screen route: the
    /// admitted head goes through an HTTP server pipeline that upgrades it to
    /// a WebSocket. Call on the channel's event loop, before it reads again.
    nonisolated func attach(_ context: ChannelHandlerContext, admitted: Data) throws {
        let service = self
        let upgrader = NIOWebSocketServerUpgrader(
            maxFrameSize: 1 << 16,
            shouldUpgrade: { channel, request in
                // A client that gave the proxy its token as a subprotocol
                // expects the server to select that protocol, as vphoned does.
                var headers = HTTPHeaders()
                if let offered = request.headers[canonicalForm: "Sec-WebSocket-Protocol"]
                    .first(where: { $0.hasPrefix(VPhoneAPIRequestGate.webSocketProtocolPrefix) })
                {
                    headers.add(name: "Sec-WebSocket-Protocol", value: String(offered))
                }
                return channel.eventLoop.makeSucceededFuture(headers)
            },
            upgradePipelineHandler: { channel, _ in
                channel.eventLoop.makeCompletedFuture {
                    try channel.pipeline.syncOperations.addHandlers([
                        NIOWebSocketFrameAggregator(
                            minNonFinalFragmentSize: 1,
                            maxAccumulatedFrameCount: 8,
                            maxAccumulatedFrameSize: 1 << 16,
                        ),
                        HostScreenSocket(service: service),
                    ])
                }
            },
        )
        let refusal = HostScreenRefusal()
        let sync = context.pipeline.syncOperations
        try sync.configureHTTPServerPipeline(
            withServerUpgrade: (upgraders: [upgrader], completionHandler: { context in
                context.pipeline.syncOperations.removeHandler(refusal, promise: nil)
            }),
        )
        try sync.addHandler(refusal)
        var buffer = context.channel.allocator.buffer(capacity: admitted.count)
        buffer.writeBytes(admitted)
        context.fireChannelRead(NIOAny(buffer))
    }
}

// MARK: - Refusal

/// Answers a request on the screen route that is not a WebSocket upgrade.
private final class HostScreenRefusal: ChannelInboundHandler, RemovableChannelHandler, @unchecked Sendable {
    typealias InboundIn = HTTPServerRequestPart
    typealias OutboundOut = HTTPServerResponsePart

    func channelRead(context: ChannelHandlerContext, data: NIOAny) {
        guard case .head = unwrapInboundIn(data) else { return }
        let body = #"{"error":{"code":"upgrade_required","message":"/v1/host/screen is a WebSocket"}}"#
        var headers = HTTPHeaders()
        headers.add(name: "Content-Type", value: "application/json")
        headers.add(name: "Content-Length", value: "\(body.utf8.count)")
        headers.add(name: "Connection", value: "close")
        let head = HTTPResponseHead(version: .http1_1, status: .upgradeRequired, headers: headers)
        var buffer = context.channel.allocator.buffer(capacity: body.utf8.count)
        buffer.writeString(body)
        let channel = context.channel
        context.write(wrapOutboundOut(.head(head)), promise: nil)
        context.write(wrapOutboundOut(.body(.byteBuffer(buffer))), promise: nil)
        context.writeAndFlush(wrapOutboundOut(.end(nil))).whenComplete { _ in
            channel.close(promise: nil)
        }
    }
}

// MARK: - Socket

/// One client. Requests are answered one at a time, in order: each takes a
/// frame on the main actor and encodes it on this client's queue.
private final class HostScreenSocket: ChannelInboundHandler, @unchecked Sendable {
    typealias InboundIn = WebSocketFrame
    typealias OutboundOut = WebSocketFrame

    /// A client keeps one or two requests outstanding; more than this is one
    /// that does not wait for answers.
    private static let maximumQueued = 16

    private let service: VPhoneHostScreenService
    private let work = DispatchQueue(label: "vphone.hostscreen.encode")
    private let encoder = VPhoneHostScreenEncoder()
    private let lock = NSLock()
    private var queued: [VPhoneHostScreenProtocol.Request] = []
    private var busy = false
    private var closed = false
    private var channel: Channel?

    init(service: VPhoneHostScreenService) {
        self.service = service
    }

    func handlerAdded(context: ChannelHandlerContext) {
        channel = context.channel
    }

    func channelRead(context: ChannelHandlerContext, data: NIOAny) {
        let frame = unwrapInboundIn(data)
        switch frame.opcode {
        case .text:
            var buffer = frame.unmaskedData
            let bytes = Data(buffer.readBytes(length: buffer.readableBytes) ?? [])
            do {
                enqueue(try VPhoneHostScreenProtocol.parse(bytes))
            } catch {
                send(VPhoneHostScreenProtocol.errorMessage(id: nil, code: error.code, message: error.description))
            }
        case .ping:
            context.writeAndFlush(wrapOutboundOut(WebSocketFrame(fin: true, opcode: .pong, data: frame.unmaskedData)), promise: nil)
        case .connectionClose:
            let channel = context.channel
            let echo = WebSocketFrame(fin: true, opcode: .connectionClose, data: frame.unmaskedData)
            context.writeAndFlush(wrapOutboundOut(echo)).whenComplete { _ in
                channel.close(promise: nil)
            }
        default:
            break
        }
    }

    func channelInactive(context: ChannelHandlerContext) {
        lock.withLock {
            closed = true
            queued.removeAll()
        }
        work.async { [encoder] in
            encoder.close()
        }
        context.fireChannelInactive()
    }

    func errorCaught(context: ChannelHandlerContext, error _: Error) {
        context.close(promise: nil)
    }

    private func enqueue(_ request: VPhoneHostScreenProtocol.Request) {
        let start = lock.withLock { () -> Bool? in
            guard !closed else { return nil }
            guard queued.count < Self.maximumQueued else { return nil }
            queued.append(request)
            guard !busy else { return false }
            busy = true
            return true
        }
        switch start {
        case nil:
            send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: "too_many_requests",
                                                       message: "at most \(Self.maximumQueued) requests may wait"))
        case true?:
            next()
        case false?:
            break
        }
    }

    private func next() {
        let request = lock.withLock { () -> VPhoneHostScreenProtocol.Request? in
            guard !closed, !queued.isEmpty else {
                busy = false
                return nil
            }
            return queued.removeFirst()
        }
        guard let request else { return }
        let service = service
        Task { @MainActor in
            let started = DispatchTime.now().uptimeNanoseconds
            let captured = await service.capture()
            let captureMicroseconds = Int((DispatchTime.now().uptimeNanoseconds - started) / 1000)
            nonisolated(unsafe) let image = captured?.image
            let turn = captured?.turn ?? 0
            self.work.async {
                self.answer(request, image: image, turn: turn, captureMicroseconds: captureMicroseconds)
                self.next()
            }
        }
    }

    private func answer(_ request: VPhoneHostScreenProtocol.Request, image: CGImage?, turn: Int, captureMicroseconds: Int) {
        guard let image else {
            send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: "display_unavailable",
                                                       message: "the VM display returned no frame"))
            return
        }
        do {
            let result = try encoder.encode(image, request: request, turn: turn)
            var header: [String: Any] = [
                "id": request.id,
                "status": result.status.rawValue,
                "session": result.session,
                "key": result.key,
                "width": result.width,
                "height": result.height,
                "sourceWidth": result.sourceWidth,
                "sourceHeight": result.sourceHeight,
                "scale": service.scale,
                "captureUs": captureMicroseconds,
                "encodeUs": result.encodeMicroseconds,
            ]
            if let record = result.record {
                header["record"] = record.base64EncodedString()
            }
            send(VPhoneHostScreenProtocol.message(header: header, payload: result.payload))
        } catch {
            // The next request starts a fresh session.
            encoder.close()
            send(VPhoneHostScreenProtocol.errorMessage(id: request.id, code: error.code, message: error.description))
        }
    }

    private func send(_ message: Data) {
        guard let channel = lock.withLock({ closed ? nil : channel }) else { return }
        channel.eventLoop.execute {
            guard channel.isActive else { return }
            var buffer = channel.allocator.buffer(capacity: message.count)
            buffer.writeBytes(message)
            channel.writeAndFlush(WebSocketFrame(fin: true, opcode: .binary, data: buffer), promise: nil)
        }
    }
}