Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

Deliver vcam frames to third-party capture clients

Lakr374a2c5687fa
diff --git a/VPhoneGuestComponents/CamFix/libcamfix.m b/VPhoneGuestComponents/CamFix/libcamfix.m
index c2f2d6ed..3b75d3a2 100644
--- a/VPhoneGuestComponents/CamFix/libcamfix.m
+++ b/VPhoneGuestComponents/CamFix/libcamfix.m
@@ -46,6 +46,7 @@
#import <Foundation/Foundation.h>
#import <ImageIO/ImageIO.h>
#import <IOSurface/IOSurfaceRef.h>
+#include <pthread.h>
#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
@@ -56,7 +57,8 @@
#include <mach-o/dyld.h>
#include <mach-o/loader.h>
#include <string.h>
-#include "../VCamCaptured/VCamFrameProtocol.h"
+#include "VCamFrameProtocol.h"
+#include "vcam_dataplane.h"
#define VCAM_UID @"vphone:vcam:0"
@@ -101,29 +103,36 @@ static BOOL cfx_shm_open(void) {
}
// One owner for "map the frame and check it is usable": opens the shm,
-// rejects a zeroed header, and rejects a pixel plane that runs past the
-// mapping. On success returns the header — the pixels start
-// VPHONE_VCAM_SHM_HEADER_SIZE bytes after cfx_shm_base — and writes the pixel-plane
-// length to *outLen. Returns NULL when the frame cannot be read.
-static const vphone_vcam_shm_header_t *cfx_shm_frame(size_t *outLen) {
- if (!cfx_shm_open()) return NULL;
+// rejects a zeroed header, and rejects pixel data that runs past the
+// mapping. Planar 4:2:0 frames carry a chroma plane after luma, and the
+// data plane reads it, so it counts toward the checked length. On success
+// fills *frame with the header fields and a pointer into the mapping.
+// Returns NO when the frame cannot be read.
+static BOOL cfx_shm_frame(vcc_frame_desc_t *frame) {
+ if (!cfx_shm_open()) return NO;
const vphone_vcam_shm_header_t *hdr = (const vphone_vcam_shm_header_t *)cfx_shm_base;
- if (!hdr->width || !hdr->height || !hdr->bytes_per_row) {
+ uint32_t w = hdr->width, h = hdr->height, bpr = hdr->bytes_per_row;
+ uint32_t fmt = hdr->pixel_format;
+ if (!w || !h || !bpr) {
cfxlog(@"shm header zeros");
- return NULL;
+ return NO;
}
- size_t len = (size_t)hdr->bytes_per_row * hdr->height;
+ size_t len = (size_t)bpr * h;
+ if (vcc_is_planar_yuv(fmt)) len += (size_t)(2u * (w / 2)) * (h / 2);
if ((size_t)VPHONE_VCAM_SHM_HEADER_SIZE + len > cfx_shm_size) {
cfxlog(@"shm: pixel range exceeds mapping");
- return NULL;
+ return NO;
}
- *outLen = len;
- return hdr;
-}
-
-static void cfx_release_bytes(void *refcon, const void *base) {
- (void)refcon;
- free((void *)base);
+ memset(frame, 0, sizeof(*frame));
+ frame->width = w;
+ frame->height = h;
+ frame->bytes_per_row = bpr;
+ frame->pixel_format = fmt;
+ frame->timestamp_ns = hdr->timestamp_ns;
+ frame->frame_index = hdr->frame_index;
+ frame->pixels = cfx_shm_base + VPHONE_VCAM_SHM_HEADER_SIZE;
+ frame->pixels_length = len;
+ return YES;
}
static void cfx_cg_release_data(void *info, const void *data, size_t size) {
@@ -131,52 +140,49 @@ static void cfx_cg_release_data(void *info, const void *data, size_t size) {
free((void *)data);
}
+// Snapshot the shm frame and decode it to tightly packed BGRA. Every RGB
+// consumer (CGImage, JPEG, IOSurface) goes through here, so the wire pixel
+// format is honored instead of assumed. Returns a malloc'd buffer of
+// w * 4 * h bytes; the caller frees it.
+static uint8_t *cfx_bgra_from_shm(uint32_t *outW, uint32_t *outH) {
+ vcc_frame_desc_t frame;
+ if (!cfx_shm_frame(&frame)) return NULL;
+ uint8_t *bgra = NULL;
+ uint32_t bpr = 0;
+ if (vcc_bgra_bytes_from_frame(&frame, &bgra, &bpr) != 0) return NULL;
+ *outW = frame.width;
+ *outH = frame.height;
+ return bgra;
+}
+
+// Photo and video delivery are BGRA. The shared data plane builds the
+// CVPixelBuffer, format description and camera attachments and converts
+// the wire format when needed. Timing is stream-local (host epoch-ns PTS
+// values lose precision past 2^53 in downstream float conversions) and
+// advances per buffer, so builds from the preview queue and the photo
+// paths serialize on one lock.
+static pthread_mutex_t cfx_cmsb_lock = PTHREAD_MUTEX_INITIALIZER;
+static vcc_timing_state_t cfx_cmsb_timing;
+static BOOL cfx_cmsb_timing_ready = NO;
+
static CMSampleBufferRef cfx_build_cmsb(void) {
- size_t len = 0;
- const vphone_vcam_shm_header_t *hdr = cfx_shm_frame(&len);
- if (!hdr) return NULL;
- uint32_t w = hdr->width, h = hdr->height, bpr = hdr->bytes_per_row;
- void *pixels = malloc(len);
+ vcc_frame_desc_t frame;
+ if (!cfx_shm_frame(&frame)) return NULL;
+ uint8_t *pixels = malloc(frame.pixels_length);
if (!pixels) return NULL;
- memcpy(pixels, cfx_shm_base + VPHONE_VCAM_SHM_HEADER_SIZE, len);
-
- CVPixelBufferRef pb = NULL;
- CVReturn cvr = CVPixelBufferCreateWithBytes(
- kCFAllocatorDefault,
- w,
- h,
- kCVPixelFormatType_32BGRA,
- pixels,
- bpr,
- cfx_release_bytes,
- NULL,
- NULL,
- &pb);
- if (cvr != kCVReturnSuccess || !pb) { free(pixels); return NULL; }
- CMVideoFormatDescriptionRef desc = NULL;
- OSStatus s = CMVideoFormatDescriptionCreateForImageBuffer(
- kCFAllocatorDefault,
- pb,
- &desc);
- if (s != noErr || !desc) { CVPixelBufferRelease(pb); return NULL; }
- CMSampleTimingInfo timing = {
- .duration = CMTimeMake(1, 30),
- .presentationTimeStamp = CMTimeMake((int64_t)hdr->timestamp_ns, 1000000000),
- .decodeTimeStamp = kCMTimeInvalid,
- };
- CMSampleBufferRef cmsb = NULL;
- s = CMSampleBufferCreateForImageBuffer(
- kCFAllocatorDefault,
- pb,
- true,
- NULL,
- NULL,
- desc,
- &timing,
- &cmsb);
- CFRelease(desc);
- CVPixelBufferRelease(pb);
- return (s == noErr) ? cmsb : NULL;
+ memcpy(pixels, frame.pixels, frame.pixels_length);
+ frame.pixels = pixels;
+
+ pthread_mutex_lock(&cfx_cmsb_lock);
+ if (!cfx_cmsb_timing_ready) {
+ vcc_timing_init(&cfx_cmsb_timing);
+ cfx_cmsb_timing_ready = YES;
+ }
+ CMSampleBufferRef sb = vcc_cmsb_from_frame(&frame, VCC_FMT_BGRA, &cfx_cmsb_timing);
+ pthread_mutex_unlock(&cfx_cmsb_lock);
+ free(pixels);
+ if (!sb) cfxlog(@"build_cmsb: shared data plane returned NULL");
+ return sb;
}
// MARK: - _setActiveFormat: nil-format guard
@@ -363,12 +369,92 @@ static void cfx_install_capturePhoto_hook(void) {
static NSHashTable *cfx_preview_layers = nil;
static dispatch_source_t cfx_preview_timer = NULL;
+
+// MARK: - preview fed from the client's capture graph
+//
+// A real camera has one path: capture graph -> sample buffer -> preview
+// surface. While a vcam preview layer is live, tap the client-side sink node
+// that receives the capture graph's sample buffers and feed the preview from
+// that sample — the same frame, timestamp, pixel format and camera metadata
+// every other consumer sees. The shm reader is the bootstrap fallback for
+// the window before the daemon builds the graph (preview-only sessions).
+
+static pthread_mutex_t cfx_tap_lock = PTHREAD_MUTEX_INITIALIZER;
+static CMSampleBufferRef cfx_tap_latest = NULL; // retained, guarded
+static CFAbsoluteTime cfx_tap_latest_at = 0; // guarded
+#define CFX_TAP_STALE_SECONDS 0.5
+
+static void cfx_tap_store(CMSampleBufferRef sb) {
+ CFRetain(sb);
+ pthread_mutex_lock(&cfx_tap_lock);
+ CMSampleBufferRef old = cfx_tap_latest;
+ cfx_tap_latest = sb;
+ cfx_tap_latest_at = CFAbsoluteTimeGetCurrent();
+ pthread_mutex_unlock(&cfx_tap_lock);
+ if (old) CFRelease(old);
+}
+
+// Returns a retained latest sample, or NULL if none is fresh.
+static CMSampleBufferRef cfx_tap_fetch_fresh(void) {
+ CMSampleBufferRef out = NULL;
+ pthread_mutex_lock(&cfx_tap_lock);
+ if (cfx_tap_latest &&
+ CFAbsoluteTimeGetCurrent() - cfx_tap_latest_at < CFX_TAP_STALE_SECONDS) {
+ out = (CMSampleBufferRef)CFRetain(cfx_tap_latest);
+ }
+ pthread_mutex_unlock(&cfx_tap_lock);
+ return out;
+}
+
+// When the capture graph last delivered a sample to this process. Video
+// delivery below is the fallback for sessions whose graph never runs; once
+// the graph delivers, the data outputs are fed natively and the fallback
+// must stop, or delegates would see two interleaved streams.
+static _Atomic double cfx_graph_last_at = 0;
+
+static BOOL cfx_graph_is_delivering(void) {
+ return CFAbsoluteTimeGetCurrent() - cfx_graph_last_at < CFX_TAP_STALE_SECONDS;
+}
+
+static IMP cfx_orig_rqsn_render = NULL;
+
+static void cfx_rqsn_render_tap(id self, SEL _cmd, CMSampleBufferRef sb, id input) {
+ if (sb) cfx_graph_last_at = CFAbsoluteTimeGetCurrent();
+ // Hold buffers only while a vcam preview is being pumped — never retain
+ // graph buffers in clients without preview layers.
+ if (sb && cfx_preview_layers.count > 0) cfx_tap_store(sb);
+ typedef void (*Fn)(id, SEL, CMSampleBufferRef, id);
+ ((Fn)cfx_orig_rqsn_render)(self, _cmd, sb, input);
+}
+
+static void cfx_install_remote_queue_tap(void) {
+ // Client-side tail of the capture graph. In this VM the synthetic source
+ // is the only camera, so samples arriving here while a vcam preview is
+ // tracked are ours by construction.
+ Class cls = NSClassFromString(@"BWRemoteQueueSinkNode");
+ if (!cls) { cfxlog(@"[tap] BWRemoteQueueSinkNode missing"); return; }
+ SEL sel = @selector(renderSampleBuffer:forInput:);
+ Method m = class_getInstanceMethod(cls, sel);
+ if (!m) { cfxlog(@"[tap] renderSampleBuffer:forInput: missing"); return; }
+ cfx_orig_rqsn_render = method_setImplementation(m, (IMP)cfx_rqsn_render_tap);
+ cfxlog(@"[tap] installed BWRemoteQueueSinkNode tap (orig=%p)", cfx_orig_rqsn_render);
+}
+
static IMP cfx_orig_pv_initWithSession = NULL;
static IMP cfx_orig_pv_initWithSessionMakeConnection = NULL;
static void cfx_pump_preview_once(void) {
if (!cfx_preview_layers || cfx_preview_layers.count == 0) return;
- CGImageRef img = cfx_build_cgimage_from_shm();
+ CGImageRef img = NULL;
+ // Preferred: the sample the capture graph just delivered.
+ CMSampleBufferRef sb = cfx_tap_fetch_fresh();
+ if (sb) {
+ CVPixelBufferRef pb = CMSampleBufferGetImageBuffer(sb);
+ if (pb) img = vcc_cgimage_from_pixel_buffer(pb);
+ CFRelease(sb);
+ }
+ // Bootstrap fallback: no graph yet, straight from the shm frame.
+ if (!img) img = cfx_build_cgimage_from_shm();
if (!img) return;
dispatch_async(dispatch_get_main_queue(), ^{
for (CALayer *layer in cfx_preview_layers) {
@@ -379,6 +465,95 @@ static void cfx_pump_preview_once(void) {
});
}
+// MARK: - client-side video delivery for vcam sessions
+//
+// When a session's capture graph never runs, its AVCaptureVideoDataOutput
+// delegates starve. Deliver the shared-layer sample directly to the
+// delegate on the output's own callback queue — the same pattern as the
+// photo delivery path. In this VM the synthetic source is the only camera,
+// so any vcam session's data output is ours. Stops while the graph
+// delivers (see cfx_graph_is_delivering).
+
+static NSHashTable *cfx_vcam_sessions = nil; // weak, guarded
+static pthread_mutex_t cfx_vcam_sessions_lock = PTHREAD_MUTEX_INITIALIZER;
+static uint64_t cfx_video_deliver_count = 0;
+static _Atomic uint64_t cfx_video_deliver_ok = 0;
+static _Atomic int cfx_video_deliver_logged = 0;
+
+static void cfx_preview_start_timer(void);
+
+// Also arms the 30 Hz pump: a session with a data output and no preview
+// layer would otherwise never start it.
+static void cfx_track_vcam_session(id session) {
+ if (!session) return;
+ BOOL added = NO;
+ pthread_mutex_lock(&cfx_vcam_sessions_lock);
+ if (!cfx_vcam_sessions) cfx_vcam_sessions = [NSHashTable weakObjectsHashTable];
+ if (![cfx_vcam_sessions containsObject:session]) {
+ [cfx_vcam_sessions addObject:session];
+ added = YES;
+ }
+ pthread_mutex_unlock(&cfx_vcam_sessions_lock);
+ if (!added) return;
+ cfxlog(@"[video delivery] tracking session %p", session);
+ dispatch_async(dispatch_get_main_queue(), ^{ cfx_preview_start_timer(); });
+}
+
+static void cfx_deliver_video_frames_once(void) {
+ pthread_mutex_lock(&cfx_vcam_sessions_lock);
+ NSArray *sessions = cfx_vcam_sessions.allObjects;
+ pthread_mutex_unlock(&cfx_vcam_sessions_lock);
+ if (sessions.count == 0 || cfx_graph_is_delivering()) return;
+ Class dataOutCls = NSClassFromString(@"AVCaptureVideoDataOutput");
+ if (!dataOutCls) return;
+
+ for (id sess in sessions) {
+ NSArray *outputs = nil;
+ @try { outputs = [sess valueForKey:@"outputs"]; } @catch (NSException *e) { continue; }
+ for (id out in outputs) {
+ if (![out isKindOfClass:dataOutCls]) continue;
+ id delegate = nil;
+ dispatch_queue_t q = nil;
+ @try { delegate = [out valueForKey:@"sampleBufferDelegate"]; } @catch (NSException *e) {}
+ @try { q = [out valueForKey:@"sampleBufferCallbackQueue"]; } @catch (NSException *e) {}
+ if (!delegate) continue;
+ NSArray *conns = nil;
+ @try { conns = [out valueForKey:@"connections"]; } @catch (NSException *e) {}
+ for (id conn in conns) {
+ BOOL enabled = NO;
+ @try { enabled = [[conn valueForKey:@"enabled"] boolValue]; } @catch (NSException *e) {}
+ if (!enabled) continue;
+ CMSampleBufferRef sb = cfx_build_cmsb();
+ if (!sb) continue;
+ cfx_video_deliver_count++;
+ dispatch_async(q ?: dispatch_get_main_queue(), ^{
+ @autoreleasepool {
+ @try {
+ ((void (*)(id, SEL, id, CMSampleBufferRef, id))objc_msgSend)(
+ delegate,
+ @selector(captureOutput:didOutputSampleBuffer:fromConnection:),
+ out,
+ sb,
+ conn);
+ cfx_video_deliver_ok++;
+ } @catch (NSException *e) {
+ if (cfx_video_deliver_logged++ < 8) {
+ cfxlog(@"[video delivery] delegate exception: %@", e);
+ }
+ }
+ CFRelease(sb);
+ }
+ });
+ }
+ }
+ }
+ if (cfx_video_deliver_count && (cfx_video_deliver_count % 90) == 1) {
+ cfxlog(@"[video delivery] attempted=%llu ok=%llu",
+ (unsigned long long)cfx_video_deliver_count,
+ (unsigned long long)cfx_video_deliver_ok);
+ }
+}
+
static void cfx_preview_start_timer(void) {
if (cfx_preview_timer) return;
dispatch_queue_t q = dispatch_queue_create("com.vphone.camfix.preview", DISPATCH_QUEUE_SERIAL);
@@ -389,7 +564,10 @@ static void cfx_preview_start_timer(void) {
33333333ull,
2000000ull);
dispatch_source_set_event_handler(cfx_preview_timer, ^{
- @autoreleasepool { cfx_pump_preview_once(); }
+ @autoreleasepool {
+ cfx_pump_preview_once();
+ cfx_deliver_video_frames_once();
+ }
});
dispatch_resume(cfx_preview_timer);
cfxlog(@"preview pump armed (30 Hz)");
@@ -713,35 +891,7 @@ static BOOL cfx_output_is_for_vcam(id self) {
// MARK: - JPEG + IOSurface builders
static NSData *cfx_build_jpeg_from_shm(void) {
- size_t len = 0;
- const vphone_vcam_shm_header_t *hdr = cfx_shm_frame(&len);
- if (!hdr) return nil;
- uint32_t w = hdr->width, h = hdr->height, bpr = hdr->bytes_per_row;
-
- void *copy = malloc(len);
- if (!copy) return nil;
- memcpy(copy, cfx_shm_base + VPHONE_VCAM_SHM_HEADER_SIZE, len);
-
- CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
- CGDataProviderRef dp = CGDataProviderCreateWithData(
- NULL,
- copy,
- len,
- cfx_cg_release_data);
- CGImageRef img = CGImageCreate(
- w,
- h,
- 8,
- 32,
- bpr,
- cs,
- (CGBitmapInfo)(kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst),
- dp,
- NULL,
- false,
- kCGRenderingIntentDefault);
- CGDataProviderRelease(dp);
- CGColorSpaceRelease(cs);
+ CGImageRef img = cfx_build_cgimage_from_shm();
if (!img) return nil;
NSMutableData *data = [NSMutableData data];
@@ -760,17 +910,14 @@ static NSData *cfx_build_jpeg_from_shm(void) {
}
static CGImageRef cfx_build_cgimage_from_shm(void) CF_RETURNS_RETAINED {
- size_t len = 0;
- const vphone_vcam_shm_header_t *hdr = cfx_shm_frame(&len);
- if (!hdr) return NULL;
- uint32_t w = hdr->width, h = hdr->height, bpr = hdr->bytes_per_row;
- void *copy = malloc(len);
- if (!copy) return NULL;
- memcpy(copy, cfx_shm_base + VPHONE_VCAM_SHM_HEADER_SIZE, len);
+ uint32_t w = 0, h = 0;
+ uint8_t *bgra = cfx_bgra_from_shm(&w, &h);
+ if (!bgra) return NULL;
+ size_t len = (size_t)w * 4 * h;
CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
CGDataProviderRef dp = CGDataProviderCreateWithData(
NULL,
- copy,
+ bgra,
len,
cfx_cg_release_data);
CGImageRef img = CGImageCreate(
@@ -778,7 +925,7 @@ static CGImageRef cfx_build_cgimage_from_shm(void) CF_RETURNS_RETAINED {
h,
8,
32,
- bpr,
+ w * 4,
cs,
(CGBitmapInfo)(kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst),
dp,
@@ -791,24 +938,25 @@ static CGImageRef cfx_build_cgimage_from_shm(void) CF_RETURNS_RETAINED {
}
static IOSurfaceRef cfx_build_iosurface_from_shm(uint32_t *outW, uint32_t *outH) CF_RETURNS_RETAINED {
- size_t len = 0;
- const vphone_vcam_shm_header_t *hdr = cfx_shm_frame(&len);
- if (!hdr) return NULL;
- uint32_t w = hdr->width, h = hdr->height, bpr = hdr->bytes_per_row;
+ uint32_t w = 0, h = 0;
+ uint8_t *bgra = cfx_bgra_from_shm(&w, &h);
+ if (!bgra) return NULL;
+ size_t len = (size_t)w * 4 * h;
NSDictionary *props = @{
(NSString *)kIOSurfaceWidth: @(w),
(NSString *)kIOSurfaceHeight: @(h),
(NSString *)kIOSurfacePixelFormat: @(kCVPixelFormatType_32BGRA),
(NSString *)kIOSurfaceBytesPerElement: @(4),
- (NSString *)kIOSurfaceBytesPerRow: @(bpr),
+ (NSString *)kIOSurfaceBytesPerRow: @(w * 4),
(NSString *)kIOSurfaceAllocSize: @(len),
};
IOSurfaceRef surf = IOSurfaceCreate((CFDictionaryRef)props);
- if (!surf) return NULL;
+ if (!surf) { free(bgra); return NULL; }
IOSurfaceLock(surf, 0, NULL);
void *base = IOSurfaceGetBaseAddress(surf);
- if (base) memcpy(base, cfx_shm_base + VPHONE_VCAM_SHM_HEADER_SIZE, len);
+ if (base) memcpy(base, bgra, len);
IOSurfaceUnlock(surf, 0, NULL);
+ free(bgra);
if (outW) *outW = w;
if (outH) *outH = h;
return surf;
@@ -1280,7 +1428,52 @@ static void cfx_install_moment_capture_hooks(void) {
static IMP cfx_orig_setRunning = NULL;
static IMP cfx_orig_setInterrupted = NULL;
+// Diagnoses why a vcam session never reaches the daemon's graph builder:
+// logs what the client thinks its session looks like at start time.
+static void cfx_log_session_start(AVCaptureSession *session) {
+ @try {
+ NSMutableArray *lines = [NSMutableArray array];
+ [lines addObject:[NSString stringWithFormat:@"inputs=%lu",
+ (unsigned long)session.inputs.count]];
+ for (AVCaptureInput *inp in session.inputs) {
+ NSString *uid = nil;
+ if ([inp isKindOfClass:[AVCaptureDeviceInput class]]) {
+ uid = ((AVCaptureDeviceInput *)inp).device.uniqueID;
+ }
+ [lines addObject:[NSString stringWithFormat:@"input %@ cls=%@ ports=%lu",
+ uid ?: @"?",
+ NSStringFromClass([inp class]),
+ (unsigned long)inp.ports.count]];
+ }
+ for (AVCaptureOutput *outp in session.outputs) {
+ BOOL anyEnabled = NO;
+ NSUInteger videoConns = 0;
+ for (AVCaptureConnection *c in outp.connections) {
+ if (c.enabled || c.active) anyEnabled = YES;
+ for (AVCaptureInputPort *port in c.inputPorts) {
+ if ([port.mediaType isEqualToString:AVMediaTypeVideo]) {
+ videoConns++;
+ break;
+ }
+ }
+ }
+ [lines addObject:[NSString stringWithFormat:@"output %@ conns=%lu video=%lu anyEnabled/Active=%d",
+ NSStringFromClass([outp class]),
+ (unsigned long)outp.connections.count,
+ (unsigned long)videoConns,
+ anyEnabled]];
+ }
+ cfxlog(@"[session start] %p %@", session, [lines componentsJoinedByString:@" | "]);
+ } @catch (NSException *e) {
+ cfxlog(@"[session start] dump exception: %@", e);
+ }
+}
+
static void cfx_session_setRunning_hook(id self, SEL _cmd, BOOL running) {
+ if (running && cfx_session_is_for_vcam(self)) {
+ cfx_log_session_start(self);
+ cfx_track_vcam_session(self);
+ }
if (!running && cfx_session_is_for_vcam(self)) {
cfxlog(@"[session _setRunning:NO] suppressed for vcam session %p", self);
return;
@@ -1302,6 +1495,54 @@ static void cfx_session_setInterrupted_hook(id self, SEL _cmd,
((OrigFn)cfx_orig_setInterrupted)(self, _cmd, interrupted, reason, interruptor);
}
+// MARK: - device input / port diagnostics
+//
+// A vcam session that never reaches startRunning usually dies at addInput:
+// because the client-side device object has no ports (streams missing from
+// the remote source copy). Log the port count at input creation and the
+// app's own canAddInput: checks.
+
+static IMP cfx_orig_dvinput_init = NULL;
+static IMP cfx_orig_canAddInput = NULL;
+
+__attribute__((ns_returns_retained))
+static id cfx_dvinput_init_hook(id self, SEL _cmd, AVCaptureDevice *device, NSError **err) {
+ typedef id (*Fn)(id, SEL, AVCaptureDevice *, NSError **);
+ id ret = ((Fn)cfx_orig_dvinput_init)(self, _cmd, device, err);
+ if ([device.uniqueID isEqualToString:VCAM_UID]) {
+ cfxlog(@"[DeviceInput init] device=%@ ports=%lu err=%@",
+ device.uniqueID,
+ (unsigned long)((AVCaptureDeviceInput *)ret).ports.count,
+ err && *err ? *err : nil);
+ }
+ return ret;
+}
+
+static BOOL cfx_canAddInput_hook(id self, SEL _cmd, AVCaptureInput *input) {
+ typedef BOOL (*Fn)(id, SEL, AVCaptureInput *);
+ BOOL ret = ((Fn)cfx_orig_canAddInput)(self, _cmd, input);
+ if ([input isKindOfClass:[AVCaptureDeviceInput class]] &&
+ [((AVCaptureDeviceInput *)input).device.uniqueID isEqualToString:VCAM_UID]) {
+ cfxlog(@"[canAddInput:%@] -> %d", VCAM_UID, ret);
+ }
+ return ret;
+}
+
+static void cfx_install_input_diagnostics(void) {
+ Class cls = NSClassFromString(@"AVCaptureDeviceInput");
+ Method m = cls ? class_getInstanceMethod(cls, @selector(initWithDevice:error:)) : NULL;
+ if (m) {
+ cfx_orig_dvinput_init = method_setImplementation(m, (IMP)cfx_dvinput_init_hook);
+ cfxlog(@"installed DeviceInput init diag");
+ }
+ Class sessCls = NSClassFromString(@"AVCaptureSession");
+ Method m2 = sessCls ? class_getInstanceMethod(sessCls, @selector(canAddInput:)) : NULL;
+ if (m2) {
+ cfx_orig_canAddInput = method_setImplementation(m2, (IMP)cfx_canAddInput_hook);
+ cfxlog(@"installed canAddInput diag");
+ }
+}
+
static void cfx_install_session_guards(void) {
Class cls = NSClassFromString(@"AVCaptureSession");
if (!cls) { cfxlog(@"AVCaptureSession missing"); return; }
@@ -1338,6 +1579,7 @@ static BOOL cfx_session_isRunning_hook(id self, SEL _cmd) {
typedef BOOL (*Fn)(id, SEL);
BOOL real = ((Fn)cfx_orig_isRunning)(self, _cmd);
if (cfx_session_is_for_vcam(self)) {
+ cfx_track_vcam_session(self);
if (cfx_isRunning_logged < 3) {
cfxlog(@"[isRunning] real=%d -> forcing YES (session=%p)", real, self);
cfx_isRunning_logged++;
@@ -1393,10 +1635,12 @@ static void cfx_install_all_hooks(void) {
cfx_install_capturePhoto_hook();
cfx_install_moment_capture_hooks();
cfx_install_session_guards();
+ cfx_install_input_diagnostics();
cfx_install_session_state_lies();
cfx_install_preview_layer_hooks();
cfx_install_photo_representation_hooks();
cfx_install_capturerequest_stubs();
+ cfx_install_remote_queue_tap();
cfx_start_scan_timer();
});
}
diff --git a/VPhoneGuestComponents/Makefile b/VPhoneGuestComponents/Makefile
index 3977deb8..b564a9b4 100644
--- a/VPhoneGuestComponents/Makefile
+++ b/VPhoneGuestComponents/Makefile
@@ -8,14 +8,34 @@ CLANG := xcrun --sdk iphoneos clang
GUEST_C_FLAGS := -arch arm64e -miphoneos-version-min=26.0 -Os
COMMON_FLAGS := $(GUEST_C_FLAGS) -fobjc-arc
+# Sources and headers are discovered under a component's folders at any
+# depth, so adding, splitting or nesting files needs no Makefile change.
+# The folders themselves are prerequisites too: their timestamps catch
+# files removed from the discovered inputs.
+find_sources = $(sort $(shell find $(1) -type f \( -name '*.m' -o -name '*.mm' -o -name '*.c' \)))
+find_headers = $(sort $(shell find $(1) -type f \( -name '*.h' -o -name '*.hpp' -o -name '*.inc' \)))
+find_folders = $(sort $(shell find $(1) -type d))
+
+# Both camera dylibs compile the shared data plane and read the frame
+# protocol header, which lives with vcamcaptured. Headers resolve through
+# -I, never through relative paths.
+VCAM_SHARED_DIRS := VCamDataPlane
+VCAM_HEADER_DIRS := VCamDataPlane VCamCaptured
+
CAMFIX := $(STAGE)/camfix/libcamfix.dylib
-CAMFIX_SOURCES := $(wildcard CamFix/*.m)
-CAMFIX_INCLUDES := $(wildcard CamFix/*.h CamFix/*.inc VCamCaptured/*.h)
+CAMFIX_DIRS := CamFix $(VCAM_SHARED_DIRS)
+CAMFIX_SOURCES := $(call find_sources,$(CAMFIX_DIRS))
+CAMFIX_INPUTS := $(call find_headers,CamFix $(VCAM_HEADER_DIRS)) \
+ $(call find_folders,CamFix $(VCAM_HEADER_DIRS))
+CAMFIX_INCLUDE_FLAGS := $(addprefix -I,$(call find_folders,CamFix) $(VCAM_HEADER_DIRS))
LOCATIONFIX := $(STAGE)/locationfix/libvlocation.dylib
-LOCATIONFIX_SOURCES := $(wildcard LocationFix/*.m)
+LOCATIONFIX_SOURCES := $(call find_sources,LocationFix)
VCAMCAPTURED := $(STAGE)/vcamcaptured/libvcamcaptured.dylib
-VCAMCAPTURED_SOURCES := $(wildcard VCamCaptured/*.m)
-VCAMCAPTURED_INCLUDES := $(wildcard VCamCaptured/*.h VCamCaptured/*.inc)
+VCAMCAPTURED_DIRS := VCamCaptured $(VCAM_SHARED_DIRS)
+VCAMCAPTURED_SOURCES := $(call find_sources,$(VCAMCAPTURED_DIRS))
+VCAMCAPTURED_INPUTS := $(call find_headers,$(VCAM_HEADER_DIRS)) \
+ $(call find_folders,$(VCAM_HEADER_DIRS))
+VCAMCAPTURED_INCLUDE_FLAGS := $(addprefix -I,$(call find_folders,VCamCaptured) $(VCAM_SHARED_DIRS))
LAUNCHHOOK := $(STAGE)/launchhook/launchdhook-vphone.dylib
LAUNCHHOOK_SOURCES := $(wildcard LaunchHook/*.c) Shared/RootHideLoaderLinks.c
LAUNCHHOOK_INCLUDES := $(wildcard LaunchHook/*.h LaunchHook/*.inc Shared/*.h)
@@ -85,8 +105,8 @@ test-vcam-dataplane:
$(STAGE)/camfix $(STAGE)/locationfix $(STAGE)/vcamcaptured $(STAGE)/launchhook $(STAGE)/systemhook $(STAGE)/gpu:
@mkdir -p $@
-$(CAMFIX): $(CAMFIX_SOURCES) $(CAMFIX_INCLUDES) CamFix VCamCaptured | $(STAGE)/camfix
- $(CLANG) $(COMMON_FLAGS) -dynamiclib \
+$(CAMFIX): $(CAMFIX_SOURCES) $(CAMFIX_INPUTS) | $(STAGE)/camfix
+ $(CLANG) $(COMMON_FLAGS) $(CAMFIX_INCLUDE_FLAGS) -dynamiclib \
-install_name @rpath/libcamfix.dylib \
-o $@ $(CAMFIX_SOURCES) \
-framework Foundation -framework AVFoundation -framework ImageIO \
@@ -101,11 +121,12 @@ $(LOCATIONFIX): $(LOCATIONFIX_SOURCES) LocationFix | $(STAGE)/locationfix
-framework Foundation -framework CoreLocation
@codesign --force --sign - $@
-$(VCAMCAPTURED): $(VCAMCAPTURED_SOURCES) $(VCAMCAPTURED_INCLUDES) VCamCaptured | $(STAGE)/vcamcaptured
- $(CLANG) $(COMMON_FLAGS) -dynamiclib \
+$(VCAMCAPTURED): $(VCAMCAPTURED_SOURCES) $(VCAMCAPTURED_INPUTS) | $(STAGE)/vcamcaptured
+ $(CLANG) $(COMMON_FLAGS) $(VCAMCAPTURED_INCLUDE_FLAGS) -dynamiclib \
-install_name @rpath/libvcamcaptured.dylib \
-o $@ $(VCAMCAPTURED_SOURCES) \
- -framework Foundation -framework CoreMedia -framework CoreVideo
+ -framework Foundation -framework CoreMedia -framework CoreVideo \
+ -framework CoreGraphics
@codesign --force --sign - $@
$(LAUNCHHOOK): $(LAUNCHHOOK_SOURCES) $(LAUNCHHOOK_INCLUDES) LaunchHook Shared | $(STAGE)/launchhook
diff --git a/VPhoneGuestComponents/VCamCaptured/VCamCaptureObservation.inc b/VPhoneGuestComponents/VCamCaptured/VCamCaptureObservation.inc
index 02af02f4..d5ee5895 100644
--- a/VPhoneGuestComponents/VCamCaptured/VCamCaptureObservation.inc
+++ b/VPhoneGuestComponents/VCamCaptured/VCamCaptureObservation.inc
@@ -156,6 +156,30 @@ static void vcc_dump_sink_node_methods(void) {
// we know the pipeline runs and we just need to substitute the
// sample-buffer contents.
+// Video sink nodes (BWImageQueueSinkNode / BWRemoteQueueSinkNode) created
+// for any client session. Strong refs: they keep each node valid and its
+// client transport pinned for the lifetime of the daemon — one leak per
+// session start, the same tradeoff the synth streams already make. The
+// viewfinder queue drives them (vcc_drive_sinks_once) while the init hooks
+// append from graph-build threads, so every access holds the array's lock.
+static NSMutableArray *vcc_driven_sinks = nil;
+static pthread_mutex_t vcc_driven_sinks_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static void vcc_track_driven_sink(id sink) {
+ if (!sink) return;
+ pthread_mutex_lock(&vcc_driven_sinks_lock);
+ if (!vcc_driven_sinks) vcc_driven_sinks = [NSMutableArray array];
+ if (![vcc_driven_sinks containsObject:sink]) [vcc_driven_sinks addObject:sink];
+ pthread_mutex_unlock(&vcc_driven_sinks_lock);
+}
+
+static NSArray *vcc_driven_sinks_snapshot(void) {
+ pthread_mutex_lock(&vcc_driven_sinks_lock);
+ NSArray *snapshot = vcc_driven_sinks ? [vcc_driven_sinks copy] : @[];
+ pthread_mutex_unlock(&vcc_driven_sinks_lock);
+ return snapshot;
+}
+
static unsigned long vcc_render_call_count = 0;
static unsigned long vcc_iqsn_init_count = 0;
static unsigned long vcc_rqsn_init_count = 0;
@@ -200,6 +224,7 @@ static id vcc_iqsn_init_hook(
vcc_iqsn_init_count++;
vcc_log(@" [IQSN init] -> %p sinkID=%@ count=%lu",
ret, sinkID, vcc_iqsn_init_count);
+ vcc_track_driven_sink(ret);
return ret;
}
@@ -216,6 +241,7 @@ static id vcc_rqsn_init_hook(
vcc_rqsn_init_count++;
vcc_log(@" [RQSN init] -> %p mediaType=0x%x sinkID=%@ count=%lu",
ret, mediaType, sinkID, vcc_rqsn_init_count);
+ vcc_track_driven_sink(ret);
return ret;
}
diff --git a/VPhoneGuestComponents/VCamCaptured/VCamFrameReceiver.inc b/VPhoneGuestComponents/VCamCaptured/VCamFrameReceiver.inc
index b29ea7e6..c966661d 100644
--- a/VPhoneGuestComponents/VCamCaptured/VCamFrameReceiver.inc
+++ b/VPhoneGuestComponents/VCamCaptured/VCamFrameReceiver.inc
@@ -55,11 +55,19 @@ static int vcc_shm_read_latest(void) {
uint32_t h = hdr->height;
uint32_t bpr = hdr->bytes_per_row;
uint32_t fmt = hdr->pixel_format;
+ uint64_t ts = hdr->timestamp_ns;
uint64_t idx = hdr->frame_index;
uint32_t pix_len = hdr->pixels_length;
if (pix_len == 0 || pix_len > VPHONE_VCAM_SHM_MAX_PIXELS) return 0;
if (w == 0 || h == 0 || bpr == 0 || pix_len < (size_t)bpr * h) return 0;
+ // Planar 4:2:0 wire data also carries the chroma plane after luma; refuse
+ // truncated frames instead of reading garbage chroma.
+ if (vcc_is_planar_yuv(fmt)) {
+ size_t luma = (size_t)bpr * h;
+ size_t chroma = (size_t)(2u * (w / 2)) * (h / 2);
+ if (pix_len < luma + chroma) return 0;
+ }
if (vcc_staging_capacity < pix_len) {
uint8_t *larger = (uint8_t *)realloc(vcc_staging_pixels, pix_len);
@@ -85,6 +93,9 @@ static int vcc_shm_read_latest(void) {
vcc_latest_frame.width = w;
vcc_latest_frame.height = h;
vcc_latest_frame.bytes_per_row = bpr;
+ vcc_latest_frame.pixel_format = fmt;
+ vcc_latest_frame.timestamp_ns = ts;
+ vcc_latest_frame.frame_index = idx;
vcc_staging_pixels = old_pixels;
vcc_staging_capacity = old_capacity;
pthread_mutex_unlock(&vcc_latest_frame.lock);
diff --git a/VPhoneGuestComponents/VCamCaptured/VCamSourceInstallation.inc b/VPhoneGuestComponents/VCamCaptured/VCamSourceInstallation.inc
index 4656030d..e5f18561 100644
--- a/VPhoneGuestComponents/VCamCaptured/VCamSourceInstallation.inc
+++ b/VPhoneGuestComponents/VCamCaptured/VCamSourceInstallation.inc
@@ -5,6 +5,9 @@
// declaration: the two must stay identical or the device lookup stops matching.
static NSString *const kVccSynthDeviceID = @"vphone:vcam:device:0";
+// Opaque unique ID of the one realtime video stream this source publishes.
+static NSString *const kVccSynthStreamID = @"vphone:vcam:stream:video:0";
+
static id vcc_build_backing(void) {
Class backingClass = NSClassFromString(@"FigCaptureSourceBacking");
if (!backingClass) {
@@ -100,10 +103,52 @@ static id vcc_build_backing(void) {
// see one; clients that pick the canonical ISP pixel format (AVF's
// _preferredFormatForPreset: matcher for AVCaptureSessionPresetHigh)
// pick the 420v one. Same preset list + frame-rate range on both.
+ // Phase 3: AVCaptureDeviceFormat consistency. Key names below are
+ // grounded in Apple's shipped CMCapture/CMCaptureCore (extracted from
+ // the dyld shared cache — same Fig framework family the guest runs);
+ // semantics come from the public AVCaptureDeviceFormat API spec in
+ // the SDK headers. Every value tells the same story as the delivered
+ // samples (see vcam_dataplane.c):
+ //
+ // AVF property <- published key value
+ // ---------------------------------------------------------------
+ // videoFieldOfView <- VideoFieldOfView 63.0
+ // (same constant derives the per-frame camera intrinsic matrix
+ // focal length: fx = (w/2)/tan(FOV/2) — one knob, no drift)
+ // videoSupportedFrameRateRanges <- MinFrameRate/MaxFrameRate 1..60
+ // videoMaxZoomFactor <- VideoStabilizationTypeOverrideForStandard=3
+ // (maxZoomFactor's dimension-table fast path; yields 16.0 for
+ // 1280-wide — keeps setVideoZoomFactor: in range for Camera.app)
+ // videoHDRSupported <- HDRSupported NO (8-bit SDR)
+ // isCameraIntrinsicMatrixDeliverySupported
+ // <- CameraCalibrationDataDeliverySupported YES
+ // IntrinsicMatrixReferenceWidth/Height
+ // (we attach kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix to
+ // every delivered sample, so YES is truthful)
+ // dimensions / formatDescription <- Width/Height/PixelFormatType 1280x720 '420v'|'BGRA'
+ //
+ // Deliberately NOT published (encoding unknown without runtime ground
+ // truth; a wrong shape could break the FigCaptureSourceVideoFormat
+ // init): HighResStillImageDimensions, supportedColorSpaces
+ // ("ColorSpace"), stabilization mode lists, photo dimensions.
NSDictionary *commonKeys = @{
@"DefaultActiveFormat" : @NO, // overridden on the active one
figKeyN("kFigSupportedFormat_VideoMinFrameRate") : @(1),
figKeyN("kFigSupportedFormat_VideoMaxFrameRate") : @(60),
+ // Horizontal field of view in degrees. Same constant that derives the
+ // per-frame camera intrinsic matrix focal length, so AVCaptureDevice
+ // format.fieldOfView and the CMSampleBuffer intrinsics tell one story.
+ // Calibration knob — measure on a physical device before trusting.
+ @"VideoFieldOfView" : @(VCC_VCAM_HFOV_DEG),
+ // 8-bit SDR delivery: advertising HDR would be a lie AVF clients
+ // would act on (tone mapping, EDR pipelines).
+ @"HDRSupported" : @NO,
+ // We attach kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix to
+ // every delivered sample with the principal point at frame center —
+ // so intrinsic delivery is supported, referenced to the full frame.
+ @"CameraCalibrationDataDeliverySupported" : @YES,
+ @"IntrinsicMatrixReferenceWidth" : @1280,
+ @"IntrinsicMatrixReferenceHeight" : @720,
// -[FigCaptureSourceVideoFormat maxZoomFactor] takes a fast path that
// returns 1.0 for raw bayer formats; for BGRA (our case) it falls to
// a fancy path that reads stabilizationTypeOverrideForCinematic / -ForStandard
@@ -177,6 +222,13 @@ static id vcc_build_backing(void) {
@selector(alloc));
if (!alloced) return nil;
uint32_t mediaTypeVideo = 0x76696465; // 'vide'
+ // Advertise one realtime (unsynchronized) video stream. Without a stream
+ // uniqueID the client-side AVCaptureFigVideoDevice has no ports: sessions
+ // never ask the vendor for streams, no capture graph is built for
+ // third-party clients, and their AVCaptureVideoDataOutputs starve.
+ // Camera.app never noticed because its viewfinder path is driven
+ // independently.
+ NSArray *unsyncStreams = @[ kVccSynthStreamID ];
id backing = ((id (*)(id, SEL, uint32_t, id, id, id, id, id, id))objc_msgSend)(
alloced,
initSel,
@@ -186,11 +238,12 @@ static id vcc_build_backing(void) {
formats,
@[],
@[],
- @[]);
- vcc_log(@" backing = %p (attrs.count=%lu formats.count=%lu)",
+ unsyncStreams);
+ vcc_log(@" backing = %p (attrs.count=%lu formats.count=%lu streams=%@)",
backing,
(unsigned long)attrs.count,
- (unsigned long)formats.count);
+ (unsigned long)formats.count,
+ unsyncStreams);
return backing;
}
diff --git a/VPhoneGuestComponents/VCamCaptured/VCamStillSink.inc b/VPhoneGuestComponents/VCamCaptured/VCamStillSink.inc
index a436cdef..ab9dba14 100644
--- a/VPhoneGuestComponents/VCamCaptured/VCamStillSink.inc
+++ b/VPhoneGuestComponents/VCamCaptured/VCamStillSink.inc
@@ -137,7 +137,9 @@ static void vcc_construct_still_sink(void) {
// proven the synthetic-handler approach works at the API level.
static void vcc_drive_still_sink_once(void) {
if (!vcc_synth_still_sink) return;
- CMSampleBufferRef sbuf = vcc_build_cmsb_from_shm();
+ // Photo sinks receive BGRA (the still pipeline JPEG-encodes directly);
+ // delivered format matches the BGRA FigCaptureSourceVideoFormat.
+ CMSampleBufferRef sbuf = vcc_build_cmsb_from_shm_fmt(VCC_FMT_BGRA);
if (!sbuf) {
vcc_log(@" still-drive: no shm frame yet");
return;
@@ -205,6 +207,22 @@ static id vcc_parsed_cfg_init_hook(id self,
&& [cameraCfgs count] > 0) {
vcc_log(@" [ParsedCfg] cameraCfgs[0] class=%@",
NSStringFromClass([[cameraCfgs firstObject] class]));
+ // Always dump the camera source config's identity — the graph builder
+ // routes on these values; a nil/empty captureDeviceID here means it
+ // will never reach the device vendor.
+ id cameraCfg = [cameraCfgs firstObject];
+ id sourceID = nil, deviceType = nil, captureDeviceID = nil, source = nil;
+ @try { sourceID = [cameraCfg valueForKey:@"sourceID"]; } @catch (NSException *e) {}
+ @try { deviceType = [cameraCfg valueForKey:@"sourceDeviceType"]; } @catch (NSException *e) {}
+ @try { captureDeviceID = [cameraCfg valueForKey:@"captureDeviceID"]; } @catch (NSException *e) {}
+ @try { source = [cameraCfg valueForKey:@"source"]; } @catch (NSException *e) {}
+ vcc_log(@" [ParsedCfg] cameraCfg sourceID=%@ deviceType=%@ captureDeviceID=%@ source=%@",
+ sourceID ?: @"(nil)", deviceType ?: @"(nil)",
+ captureDeviceID ?: @"(nil)",
+ source ? NSStringFromClass([source class]) : @"(nil)");
+ NSString *desc = [cameraCfg description];
+ if (desc.length > 800) desc = [desc substringToIndex:800];
+ vcc_log(@" [ParsedCfg] cameraCfg desc=%@", desc);
}
if (stillCfgs && [stillCfgs respondsToSelector:@selector(count)]
&& [stillCfgs count] > 0) {
diff --git a/VPhoneGuestComponents/VCamCaptured/VCamViewfinderHooks.inc b/VPhoneGuestComponents/VCamCaptured/VCamViewfinderHooks.inc
index 0892cba5..17b269b9 100644
--- a/VPhoneGuestComponents/VCamCaptured/VCamViewfinderHooks.inc
+++ b/VPhoneGuestComponents/VCamCaptured/VCamViewfinderHooks.inc
@@ -15,28 +15,62 @@ typedef struct vcc_latest_frame_s {
uint32_t width;
uint32_t height;
uint32_t bytes_per_row;
+ uint32_t pixel_format;
+ uint64_t timestamp_ns;
+ uint64_t frame_index;
uint8_t *pixels;
size_t pixels_capacity;
size_t pixels_length;
} vcc_latest_frame_t;
extern vcc_latest_frame_t vcc_latest_frame;
-// CVPixelBuffer release callback for the malloc'd pixel buffer. Has to be
-// a real C function (block isn't compatible with the callback signature).
-static void vcc_cv_release_bytes(void *refcon, const void *baseAddress) {
- (void)refcon;
- free((void *)baseAddress);
-}
-
static IMP vcc_vfs_init_orig = NULL;
static IMP vcc_vfs_open_orig = NULL;
static IMP vcc_vfs_close_orig = NULL;
static NSMutableArray *vcc_vf_streams = nil; // strong refs
static dispatch_source_t vcc_vf_timer = NULL;
static dispatch_queue_t vcc_vf_q = NULL;
-static uint64_t vcc_vf_pts_ns = 0;
static uint64_t vcc_vf_enqueue_count = 0;
static uint64_t vcc_vf_enqueue_success = 0;
+static uint64_t vcc_sink_drive_count = 0;
+static uint64_t vcc_sink_drive_ok = 0;
+
+static CMSampleBufferRef vcc_build_cmsb_from_shm_fmt(uint32_t fmt_out);
+
+// Deliver the latest shm frame to every captured video sink — the
+// client-graph tail. Driving them with our samples is what delivers frames
+// to third-party AVCaptureVideoDataOutput clients, the same way the
+// still-sink drive delivers photos. Runs on the same 30 Hz queue as the
+// viewfinder drive. Graph-built sinks expect the active advertised format's
+// samples — BGRA is what clients see selected (camfix substitutes the first
+// format for third-party sessions).
+static void vcc_drive_sinks_once(void) {
+ NSArray *sinks = vcc_driven_sinks_snapshot();
+ if (sinks.count == 0) return;
+ CMSampleBufferRef cmsb = vcc_build_cmsb_from_shm_fmt(VCC_FMT_BGRA);
+ if (!cmsb) return;
+ SEL sel = @selector(renderSampleBuffer:forInput:);
+ for (id sink in sinks) {
+ if (![sink respondsToSelector:sel]) continue;
+ @try {
+ ((void (*)(id, SEL, CMSampleBufferRef, id))objc_msgSend)(
+ sink, sel, cmsb, nil);
+ vcc_sink_drive_ok++;
+ } @catch (NSException *e) {
+ if ((vcc_sink_drive_count & 63) == 1) {
+ vcc_log(@" [SINK drive] exception on %p: %@", sink, e);
+ }
+ }
+ }
+ CFRelease(cmsb);
+ vcc_sink_drive_count++;
+ if ((vcc_sink_drive_count & 59) == 1) {
+ vcc_log(@" [SINK drive] frames=%llu sinks=%lu ok=%llu",
+ (unsigned long long)vcc_sink_drive_count,
+ (unsigned long)sinks.count,
+ (unsigned long long)vcc_sink_drive_ok);
+ }
+}
typedef id (*VccVfsInitFn)(id self, SEL _cmd);
typedef void (*VccVfsOpenFn)(id self, SEL _cmd, id dest);
@@ -75,61 +109,69 @@ static void vcc_vfs_close_hook(id self, SEL _cmd) {
orig(self, _cmd);
}
-// Build a CMSampleBuffer wrapping the latest shm frame. Caller must
-// CFRelease the result.
-static CMSampleBufferRef vcc_build_cmsb_from_shm(void) {
+// Build a CMSampleBuffer wrapping the latest shm frame, in the requested
+// delivery format. The heavy lifting (pixel-format-honest CVPixelBuffer,
+// extension-bearing format description, camera attachments, host-frame
+// timing) lives in the shared data plane, which Tests/VCamDataPlaneTests.c
+// proves on the host. Caller must CFRelease.
+//
+// Serializes builds because the shared timing state advances per buffer and
+// the viewfinder drive queue and the still-image drive run on different
+// threads.
+static pthread_mutex_t vcc_delivery_lock = PTHREAD_MUTEX_INITIALIZER;
+static vcc_timing_state_t vcc_delivery_timing;
+static pthread_once_t vcc_delivery_timing_once = PTHREAD_ONCE_INIT;
+
+static void vcc_delivery_timing_init(void) {
+ vcc_timing_init(&vcc_delivery_timing);
+}
+
+static CMSampleBufferRef vcc_build_cmsb_from_shm_fmt(uint32_t fmt_out) {
+ pthread_mutex_lock(&vcc_delivery_lock);
+ pthread_once(&vcc_delivery_timing_once, vcc_delivery_timing_init);
+
+ vcc_frame_desc_t frame;
+ memset(&frame, 0, sizeof(frame));
+ uint8_t *pixels = NULL;
pthread_mutex_lock(&vcc_latest_frame.lock);
- uint32_t w = vcc_latest_frame.width;
- uint32_t h = vcc_latest_frame.height;
- uint32_t bpr = vcc_latest_frame.bytes_per_row;
size_t len = vcc_latest_frame.pixels_length;
- if (!len || !w || !h || !bpr) {
- pthread_mutex_unlock(&vcc_latest_frame.lock);
- return NULL;
+ if (len && vcc_latest_frame.width && vcc_latest_frame.height &&
+ vcc_latest_frame.bytes_per_row) {
+ pixels = malloc(len);
+ if (pixels) memcpy(pixels, vcc_latest_frame.pixels, len);
}
- void *pixels = malloc(len);
- if (!pixels) {
- pthread_mutex_unlock(&vcc_latest_frame.lock);
- return NULL;
+ if (pixels) {
+ frame.width = vcc_latest_frame.width;
+ frame.height = vcc_latest_frame.height;
+ frame.bytes_per_row = vcc_latest_frame.bytes_per_row;
+ frame.pixel_format = vcc_latest_frame.pixel_format;
+ frame.timestamp_ns = vcc_latest_frame.timestamp_ns;
+ frame.frame_index = vcc_latest_frame.frame_index;
+ frame.pixels = pixels;
+ frame.pixels_length = len;
}
- memcpy(pixels, vcc_latest_frame.pixels, len);
pthread_mutex_unlock(&vcc_latest_frame.lock);
- CVPixelBufferRef pb = NULL;
- CVReturn cvr = CVPixelBufferCreateWithBytes(
- kCFAllocatorDefault, w, h, kCVPixelFormatType_32BGRA,
- pixels, bpr, vcc_cv_release_bytes, NULL, NULL, &pb);
- if (cvr != kCVReturnSuccess || !pb) {
- free(pixels);
- return NULL;
- }
-
- CMVideoFormatDescriptionRef desc = NULL;
- OSStatus s = CMVideoFormatDescriptionCreateForImageBuffer(
- kCFAllocatorDefault, pb, &desc);
- if (s != noErr || !desc) {
- CVPixelBufferRelease(pb);
+ if (!pixels) {
+ pthread_mutex_unlock(&vcc_delivery_lock);
return NULL;
}
- // Monotonic PTS at 30 fps. The viewfinder's internal jitter buffer
- // discards frames whose PTS doesn't advance, so we increment per call.
- vcc_vf_pts_ns += 33333333ull; // 1/30s in nanoseconds
- CMSampleTimingInfo timing = {
- .duration = CMTimeMake(1, 30),
- .presentationTimeStamp = CMTimeMake((int64_t)vcc_vf_pts_ns, 1000000000),
- .decodeTimeStamp = kCMTimeInvalid,
- };
-
- CMSampleBufferRef cmsb = NULL;
- s = CMSampleBufferCreateForImageBuffer(
- kCFAllocatorDefault, pb, true, NULL, NULL, desc, &timing, &cmsb);
- CFRelease(desc);
- CVPixelBufferRelease(pb);
- if (s != noErr || !cmsb) return NULL;
+ CMSampleBufferRef cmsb = vcc_cmsb_from_frame(&frame, fmt_out,
+ &vcc_delivery_timing);
+ free(pixels);
+ pthread_mutex_unlock(&vcc_delivery_lock);
return cmsb;
}
+// Viewfinder/video delivery: the active advertised format. The synthetic
+// source publishes 420v as DefaultActiveFormat, so that's what AVF clients
+// pick — the delivered sample must be the same thing, not a BGRA buffer
+// wearing a 420v costume.
+static CMSampleBufferRef vcc_build_cmsb_from_shm(void) {
+ return vcc_build_cmsb_from_shm_fmt(VCC_FMT_420V);
+}
+
static void vcc_vf_drive_once(void) {
if (!vcc_vf_streams || vcc_vf_streams.count == 0) return;
CMSampleBufferRef cmsb = vcc_build_cmsb_from_shm();
@@ -181,10 +223,14 @@ static void vcc_install_viewfinder_hooks(void) {
dispatch_time(DISPATCH_TIME_NOW, 0),
33333333ull, 2000000ull);
dispatch_source_set_event_handler(vcc_vf_timer, ^{
- @autoreleasepool { vcc_vf_drive_once(); }
+ @autoreleasepool {
+ vcc_vf_drive_once();
+ vcc_drive_sinks_once();
+ }
});
dispatch_resume(vcc_vf_timer);
- vcc_log(@" viewfinder drive timer armed (30 Hz)");
+ vcc_log(@" viewfinder drive timer armed (30 Hz, delivering '420v' "
+ @"+ camera metadata)");
}
static void vcc_install_sink_observation(void) {
diff --git a/VPhoneGuestComponents/VCamCaptured/libvcamcaptured.m b/VPhoneGuestComponents/VCamCaptured/libvcamcaptured.m
index 9671878c..489d4ab0 100644
--- a/VPhoneGuestComponents/VCamCaptured/libvcamcaptured.m
+++ b/VPhoneGuestComponents/VCamCaptured/libvcamcaptured.m
@@ -62,6 +62,7 @@
#include <sys/types.h>
#include <unistd.h>
#include "VCamFrameProtocol.h"
+#include "vcam_dataplane.h"
// MARK: - shared frame layout (matches vphoned_vcam.h)
//