Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift
5.7 KB
Raw Blame
// VPScreenCaptureIdle.c — avconferenced encodes a screen frame only when the
// screen drew.
//
// DeviceHub's stream is avconferenced screen capture: FigVirtualDisplay
// clones each lit display in backboardd, the clone writes frames back, and
// VCScreenCaptureFigCaptureController hands each one to VideoProcessing's
// real-time encoder. After the screen draws, the writeback keeps coming at the
// capture rate for about a second before it stops; a still screen sends
// nothing. The live Clock icon on the Home Screen draws about ten times a
// second, so it never stops: an idle Home Screen streamed about 42 fps per
// display, two streams on an iPhone Duo. Measured on iOS 27.1 (24A94425).
//
// On a guest the encode is the expensive part, and it runs on the host: the
// encoder is the host's AVE through VideoToolbox paravirtualization, and in
// the real-time encoder's user-DPB mode every frame carries reference-frame
// dictionaries the host serializes as binary property lists, then waits for
// the guest's reply. With a DeviceHub viewer open on the idle Home Screen the
// VM process used 91% of a core (macOS 27.0.1, M5 Pro), 14% without a viewer.
//
// So a frame reaches the encoder when the render server's frame counter for
// the main display (CARenderServerGetFrameCounterByIndex) moved since the last
// one, and otherwise once per keep-alive interval. A frame written back with a
// change can arrive after the counter it moved; the keep-alive frame, which
// also lets the encoder refine a still image, bounds that to 250 ms, and the
// writeback runs for about a second after the last draw. On an iPhone Duo the
// main display is the panel the inner display is shown on, so the counter
// covers both streams. Measured on the idle Home Screen: 10 fps per display,
// VM process 50%; scrolling Settings still streams 54–60 fps.

#include <dlfcn.h>
#include <fcntl.h>
#include <objc/runtime.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <CoreFoundation/CoreFoundation.h>

#define VP_QUARTZ_CORE "/System/Library/Frameworks/QuartzCore.framework/QuartzCore"
#define VP_AV_CONFERENCE "/System/Library/PrivateFrameworks/AVConference.framework/AVConference"

// The interval of the frames that pass while the counter stands still.
#define VP_KEEP_ALIVE_NANOSECONDS 250000000ull
// Capture controllers followed at once: one per streamed display.
#define VP_CONTROLLERS 4

// MARK: - Log (same file the rest of the dylib writes)

static void vpIdleLog(const char *format, const char *a, const char *b) {
    int fd = open("/var/mobile/Library/Caches/vphone-devicehubfix.log",
                  O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0666);
    if (fd < 0)
        return;
    fchmod(fd, 0666);
    dprintf(fd, "pid=%d ", getpid());
    dprintf(fd, format, a, b);
    dprintf(fd, "\n");
    close(fd);
}

// MARK: - The capture callback

typedef void (*VPDidReceiveSampleBuffer)(id, SEL, id, CFTypeRef, uint64_t);

static int (*vpFrameCounter)(unsigned int server, unsigned int display);
static VPDidReceiveSampleBuffer vpOriginalDidReceiveSampleBuffer;

static struct {
    const void *controller;
    int counter;
    uint64_t passed;
} vpControllers[VP_CONTROLLERS];
static pthread_mutex_t vpControllersLock = PTHREAD_MUTEX_INITIALIZER;

static bool vpShouldPass(const void *controller) {
    int counter = vpFrameCounter(0, 0);
    uint64_t now = clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
    bool pass = true;
    pthread_mutex_lock(&vpControllersLock);
    int slot = -1;
    for (int index = 0; index < VP_CONTROLLERS && slot < 0; index++)
        if (vpControllers[index].controller == controller)
            slot = index;
    // A new stream takes the slot that passed a frame longest ago (an empty
    // one never did); a stream that ended leaves its slot behind.
    if (slot < 0) {
        slot = 0;
        for (int index = 1; index < VP_CONTROLLERS; index++)
            if (vpControllers[index].passed < vpControllers[slot].passed)
                slot = index;
    }
    if (vpControllers[slot].controller != controller || counter != vpControllers[slot].counter) {
        vpControllers[slot].controller = controller;
        vpControllers[slot].counter = counter;
    } else {
        pass = now - vpControllers[slot].passed >= VP_KEEP_ALIVE_NANOSECONDS;
    }
    if (pass)
        vpControllers[slot].passed = now;
    pthread_mutex_unlock(&vpControllersLock);
    return pass;
}

static void vpDidReceiveSampleBuffer(id self, SEL command, id capture, CFTypeRef sampleBuffer, uint64_t flags) {
    if (sampleBuffer && !vpShouldPass(self))
        return;
    vpOriginalDidReceiveSampleBuffer(self, command, capture, sampleBuffer, flags);
}

__attribute__((constructor)) static void vpInstallScreenCaptureIdle(void) {
    if (strcmp(getprogname(), "avconferenced") != 0)
        return;
    vpFrameCounter = dlsym(dlopen(VP_QUARTZ_CORE, RTLD_LAZY | RTLD_LOCAL), "CARenderServerGetFrameCounterByIndex");
    dlopen(VP_AV_CONFERENCE, RTLD_LAZY | RTLD_LOCAL);
    Class controller = objc_getClass("VCScreenCaptureFigCaptureController");
    Method receive = controller ? class_getInstanceMethod(controller, sel_registerName(
                                      "screenCaptureController:didReceiveSampleBuffer:transformFlags:"))
                                : NULL;
    if (!vpFrameCounter || !receive) {
        vpIdleLog("screen capture idle frames %s, %s", "not found", "every captured frame is encoded");
        return;
    }
    vpOriginalDidReceiveSampleBuffer =
        (VPDidReceiveSampleBuffer)method_setImplementation(receive, (IMP)vpDidReceiveSampleBuffer);
    vpIdleLog("screen capture idle frames %s, %s", "found", "frames pass when the screen drew");
}