Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
Camera: the capture graph is built, and its preview gets the camera's frames
vcvkkc9481d26705f
diff --git a/userland/src/camera_device.c b/userland/src/camera_device.cindex 51ab58a..28388c8 100644--- a/userland/src/camera_device.c+++ b/userland/src/camera_device.c@@ -22,10 +22,9 @@* Everything asked of the device and the stream that is not answered here is said in mediaserverd's* log ("redwood: camera: ..."), once per property.*- * NOT FINISHED: no frames reach applications yet. AVFoundation lists the back camera with the- * plist's formats, and Camera.app opens, but its session takes the stream and gives it back at once- * (the capture graph is not built: captureSession_buildGraphWithConfiguration takes the device- * back), over and over; the stream is never started.+ * The graph builds, the session runs, and the pictures of the PrimaryScaler output reach Camera.app's+ * preview. NOT FINISHED: a photograph is not taken yet ("StillImageCaptureNow" is not answered), and+ * the session is rebuilt every few seconds, so the preview comes in bursts rather than steadily.*/#include <CoreFoundation/CoreFoundation.h>#include <CoreMedia/CoreMedia.h>@@ -188,29 +187,6 @@ put(CFMutableDictionaryRef d, const char *key, CFTypeRef value)CFRelease(value);}-static void-put_k(CFMutableDictionaryRef d, CFStringRef key, CFTypeRef value)-{- CFDictionarySetValue(d, key, value);- CFRelease(value);-}--static int64_t-get_i(CFDictionaryRef d, const char *key, int64_t otherwise)-{- CFStringRef k = CFStringCreateWithCString(NULL, key, kCFStringEncodingUTF8);- CFTypeRef v = d ? CFDictionaryGetValue(d, k) : NULL;- int64_t out = otherwise;-- CFRelease(k);- if (v != NULL && CFGetTypeID(v) == CFNumberGetTypeID()) {- CFNumberGetValue(v, kCFNumberSInt64Type, &out);- } else if (v != NULL && CFGetTypeID(v) == CFBooleanGetTypeID()) {- out = CFBooleanGetValue(v);- }- return out;-}-static CFDictionaryRefnew_dict(void){@@ -357,10 +333,8 @@ struct output {CFDictionaryRef config; /* from VideoOutputConfigurations */bool enabled;void * handler; /* a block: ^(CVPixelBufferRef, CMTime) */- CVPixelBufferPoolRef pool;- int pool_width, pool_height;- uint32_t pool_format;- uint64_t delivered;+ bool said_shape;+ uint64_t delivered, refused;};#define OUTPUTS 3@@ -369,12 +343,14 @@ static const char *const output_names[OUTPUTS] = { "PrimaryScaler", "SecondaryScstruct stream {CFArrayRef formats;CFDictionaryRef supported;+ CFDictionaryRef colour_info; /* from VideoOutputsColorInfo */int format_index;double min_fps, max_fps;struct output out[OUTPUTS];pthread_mutex_t lock;pthread_t worker;volatile bool running;+ volatile bool graph_running; /* the capture graph has started this stream: frames may be given out */bool worker_started;int torch;uint64_t frames;@@ -449,6 +425,19 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato}pthread_mutex_unlock(&s->lock);value = d;+ } else if (key_is(key, "VideoOutputsColorInfo")) {+ /*+ * ReadWrite, and the capture graph READS it before it starts the session:+ * -[BWMultiStreamCameraSourceNode _verifyColorSpacePropertiesForAllOutputs] does a+ * getProperty:error: for it, and an error there fails the session's transition to running+ * and tears the whole graph down again. What it holds matters only for an output whose+ * colour space is not sRGB (_verifyColorSpacePropertiesForOutputID: returns at once when+ * the output's colour space is 0), and sRGB is the only one this stream offers, so what+ * the graph set is given back as it is.+ */+ pthread_mutex_lock(&s->lock);+ value = s->colour_info != NULL ? CFRetain(s->colour_info) : new_dict();+ pthread_mutex_unlock(&s->lock);} else if (key_is(key, "SupportedPropertiesDictionary")) {value = CFRetain(s->supported);} else if (key_is(key, "FocalLength")) {@@ -566,6 +555,15 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)pthread_mutex_unlock(&s->lock);return 0;}+ if (key_is(key, "VideoOutputsColorInfo")) {+ pthread_mutex_lock(&s->lock);+ if (s->colour_info != NULL) {+ CFRelease(s->colour_info);+ }+ s->colour_info = value != NULL && CFGetTypeID(value) == CFDictionaryGetTypeID() ? CFRetain(value) : NULL;+ pthread_mutex_unlock(&s->lock);+ return 0;+ }if (key_is(key, "TorchLevel")) {torch_set(s, value);return 0;@@ -625,32 +623,39 @@ render_piece(void *context, size_t i)}static CVPixelBufferRef-output_buffer(struct output *out, int width, int height, uint32_t pf)+output_buffer(struct output *out){+ CVPixelBufferPoolRef pool = NULL;CVPixelBufferRef pb = NULL;+ CFTypeRef value = out->config != NULL ? CFDictionaryGetValue(out->config, CFSTR("BufferPool")) : NULL;- if (out->pool == NULL || out->pool_width != width || out->pool_height != height || out->pool_format != pf) {- CFMutableDictionaryRef a = (CFMutableDictionaryRef)new_dict();-- if (out->pool != NULL) {- CVPixelBufferPoolRelease(out->pool);- out->pool = NULL;- }- put_k(a, kCVPixelBufferWidthKey, number_i(width));- put_k(a, kCVPixelBufferHeightKey, number_i(height));- put_k(a, kCVPixelBufferPixelFormatTypeKey, number_i(pf));- put_k(a, kCVPixelBufferIOSurfacePropertiesKey, new_dict());- if (CVPixelBufferPoolCreate(NULL, NULL, a, &out->pool) != kCVReturnSuccess) {- out->pool = NULL;+ /*+ * The graph makes the pool itself and hands it over in the output's configuration+ * (kFigCaptureStreamVideoOutputConfigurationKey_BufferPool), sized and laid out as the node+ * behind the output wants it (an IOSurface each, extra rows at the bottom where a scaler wants+ * them, as few buffers as the node holds at once) -- there is nothing for this side to choose,+ * and a buffer of our own making is not one the graph would take. The configuration carries no+ * width or height: the pool's buffers have them.+ */+ if (value != NULL && CFGetTypeID(value) == CVPixelBufferPoolGetTypeID()) {+ pool = (CVPixelBufferPoolRef)value;+ }+ if (pool == NULL) {+ if (!out->said_shape) {+ out->said_shape = true;+ say("%s: no pool of the graph's in the configuration: no pictures", out->name);}- CFRelease(a);- out->pool_width = width;- out->pool_height = height;- out->pool_format = pf;- say("%s: %dx%d %c%c%c%c buffers%s", out->name, width, height, pf >> 24, pf >> 16 & 255, pf >> 8 & 255, pf & 255,- out->pool ? "" : " CANNOT BE MADE");+ return NULL;}- if (out->pool == NULL || CVPixelBufferPoolCreatePixelBuffer(NULL, out->pool, &pb) != kCVReturnSuccess) {+ if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess) {+ /*+ * The preview pool holds one buffer (MinimumBufferCount 1): while the graph still has it,+ * there is none to be had, and this frame is not given out. Said at most every 300th frame.+ */+ if (out->refused++ % 300 == 0) {+ say("%s: the graph's pool has no buffer to spare (%llu frames so)", out->name,+ (unsigned long long)out->refused);+ }return NULL;}return pb;@@ -661,21 +666,28 @@ typedef void (^pixel_buffer_handler)(CVPixelBufferRef, CMTime);static voiddeliver(struct stream *s, struct output *out, const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw){- int width = (int)get_i(out->config, "Width", 0), height = (int)get_i(out->config, "Height", 0);- uint32_t pf = (uint32_t)get_i(out->config, "PixelFormat", PF_420F);- CVPixelBufferRef pb;+ CVPixelBufferRef pb = output_buffer(out);+ uint32_t pf, width, height;(void)s;- if (width <= 0 || height <= 0 || width > 8192 || height > 8192) {+ if (pb == NULL) {return;}- pf = pf == PF_420V || pf == PF_BGRA ? pf : PF_420F;- if ((pb = output_buffer(out, width, height, pf)) == NULL) {+ pf = CVPixelBufferGetPixelFormatType(pb);+ width = (uint32_t)CVPixelBufferGetWidth(pb);+ height = (uint32_t)CVPixelBufferGetHeight(pb);+ if (width == 0 || height == 0 || (pf != PF_420F && pf != PF_420V && pf != PF_BGRA)) {+ if (out->delivered == 0 && out->said_shape == false) {+ out->said_shape = true;+ say("%s: %ux%u %c%c%c%c buffers are not for this ISP", out->name, width, height, pf >> 24, pf >> 16 & 255,+ pf >> 8 & 255, pf & 255);+ }+ CVPixelBufferRelease(pb);return;}CVPixelBufferLockBaseAddress(pb, 0);- struct render_job job = { isp, f, raw, pb, pf, (uint32_t)width, (uint32_t)height, 8 };+ struct render_job job = { isp, f, raw, pb, pf, width, height, 8 };dispatch_apply_f(job.pieces, dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), &job, render_piece);CVPixelBufferUnlockBaseAddress(pb, 0);@@ -689,7 +701,8 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc((pixel_buffer_handler)out->handler)(pb, pts);CVPixelBufferRelease(pb);if (out->delivered++ == 0) {- say("%s: the first picture given (%dx%d)", out->name, width, height);+ say("%s: the first picture given (%ux%u %c%c%c%c)", out->name, width, height, pf >> 24, pf >> 16 & 255,+ pf >> 8 & 255, pf & 255);}}@@ -750,11 +763,25 @@ worker(void *context)(void)write(device, &c, sizeof(c));}s->frames++;+ /*+ * Only once the graph has started the stream. Its PixelBufferHandler blocks hand the buffer+ * to multiStreamCameraSourceNode_sendBufferToBufferServicingThread, which reaches into the+ * source node's innards: called before the graph is started (or after it has been torn down)+ * it walks a pointer that is not there yet and mediaserverd dies of SIGSEGV. The frames+ * before that are read and thrown away, which is also what settles the exposure.+ */pthread_mutex_lock(&s->lock);- for (int i = 0; i < OUTPUTS; i++) {+ for (int i = 0; s->graph_running && i < OUTPUTS; i++) {struct output *out = &s->out[i];- if (out->handler != NULL && out->config != NULL && (out->enabled || i == 0)) {+ /*+ * The scalers' outputs are the moving picture and are given every frame; the still image+ * output is a photograph, which the graph asks for one at a time ("StillImageCaptureNow")+ * and which is the whole sensor, 4032x3024 -- not something to make thirty times a second.+ * (NOT FINISHED: nothing answers StillImageCaptureNow yet, so no photograph is taken.)+ */+ if (out->handler != NULL && out->config != NULL && (out->enabled || i == 0) &&+ strcmp(out->name, "StillImage") != 0) {deliver(s, out, &isp, f, raw);}}@@ -770,17 +797,29 @@ worker(void *context)return NULL;}+/*+ * Reading frames from the kernel and giving them to the graph are two things: the first begins when+ * the capture graph reserves the stream (`graph` false, from the device's RequestStreams: the sensor+ * takes a moment to settle and the exposure longer), the second only when the graph starts the+ * stream itself (`graph` true, the FigCaptureStream start method).+ */static OSStatus-stream_start(CMBaseObjectRef o)+stream_begin(CMBaseObjectRef o, bool graph){struct stream *s = stream_of(o);+ if (graph) {+ pthread_mutex_lock(&s->lock);+ s->graph_running = true;+ pthread_mutex_unlock(&s->lock);+ say("stream: the graph starts the stream (format %d)", s->format_index);+ }if (s->running) {return 0;}s->running = true;s->frames = 0;- say("stream: start (format %d)", s->format_index);+ say("stream: frames begin (format %d)", s->format_index);if (pthread_create(&s->worker, NULL, worker, o) != 0) {s->running = false;return ERR_UNSUPPORTED;@@ -789,12 +828,21 @@ stream_start(CMBaseObjectRef o)return 0;}+static OSStatus+stream_start(CMBaseObjectRef o)+{+ return stream_begin(o, true);+}+static OSStatusstream_stop(CMBaseObjectRef o){struct stream *s = stream_of(o);say("stream: stop");+ pthread_mutex_lock(&s->lock);+ s->graph_running = false;+ pthread_mutex_unlock(&s->lock);s->running = false;if (s->worker_started) {pthread_join(s->worker, NULL);@@ -837,9 +885,6 @@ stream_finalize(CMBaseObjectRef o)if (s->out[i].handler) {CFRelease(s->out[i].handler);}- if (s->out[i].pool) {- CVPixelBufferPoolRelease(s->out[i].pool);- }}if (s->formats) {CFRelease(s->formats);@@ -847,6 +892,9 @@ stream_finalize(CMBaseObjectRef o)if (s->supported) {CFRelease(s->supported);}+ if (s->colour_info) {+ CFRelease(s->colour_info);+ }if (s->torch >= 0) {close(s->torch);}@@ -960,7 +1008,7 @@ device_request_streams(CMBaseObjectRef o, CFArrayRef streams, CFDictionaryRef opCMBaseObjectRef st = (CMBaseObjectRef)CFArrayGetValueAtIndex(streams, i);if (st != NULL) {- (void)stream_start(st);+ (void)stream_begin(st, false);}}}diff --git a/userland/src/inject_media.c b/userland/src/inject_media.cindex e329c3a..04b4dce 100644--- a/userland/src/inject_media.c+++ b/userland/src/inject_media.c@@ -24,9 +24,12 @@** The camera (only where the kernel has one: cam=5, /dev/redwood-camera): the vendor is made for* everybody, with the create function of our capture device (camera_device.c) in place of the- * one H10ISP.mediacapture would give for the iPhone's ISP. NOT FINISHED: no frames reach- * applications yet (see camera_device.c). Its stream takes the torch level too, so FigFlashlight- * would need nothing of its own then (not tried yet).+ * one H10ISP.mediacapture would give for the iPhone's ISP; the capture graph is built around it and+ * its frames reach Camera.app's preview (see camera_device.c). Its stream takes the torch level too,+ * so FigFlashlight would need nothing of its own then (not tried yet).+ *+ * What the capture stack asks of hardware this phone has none of is refused here: there is no ISP+ * processing session, so there can be no video stabilization (see video_stabilization_method below).** CMCapture is not loaded yet when this library is: the methods are replaced when its image* arrives.@@ -204,6 +207,41 @@ flashlight_stream_set(__unused id self, __unused SEL selector, id key, id value)return 0;}+/*+ * Video stabilization, of which this hardware has none: the capture graph's VIS node+ * (-[FigCaptureVISPipeline _newVISNodeWithUpstreamOutput:...]) is made with the ISP's processing+ * session, and this device has no such session, so the node comes back nil, the VIS pipeline with+ * it, the movie file sink's tail pipeline with that, and captureSession_buildGraphWithConfiguration+ * tears the whole graph down -- the preview too, for all that the preview has nothing to do with+ * stabilizing a recording. The connection's stabilization method is where the whole stack reads the+ * wish from (+[FigVideoCaptureConnectionConfiguration videoStabilizationMethods:includeIris:] ->+ * cspc_getVideoStabilizationType -> the connections' FigCaptureVideoStabilizationType), and 0 there+ * means none: the graph is then built without a VIS pipeline anywhere+ * (+[FigCaptureVISPipeline visPipelineIsRequiredForFigCaptureVideoStabilizationType:] says no to+ * every type but 2 and 3), and recordings are as steady as the hand that holds the phone.+ */+static int+video_stabilization_method(id self, SEL selector)+{+ (void)self;+ (void)selector;+ return 0;+}++/*+ * The type the stack acts on. The connection configuration arrives from the application's+ * AVFoundation with the type in it, which is read straight out of the object and not computed from+ * the method above (-[FigCaptureSessionParsedConfiguration _computeVideoStabilizationTypes] only+ * fills in what the client left out), so the wish has to be refused here as well.+ */+static int+video_stabilization_type(id self, SEL selector)+{+ (void)self;+ (void)selector;+ return 0; /* kFigCaptureVideoStabilizationType_None */+}+struct method_spec {const char *selector, *types;IMP imp;@@ -251,12 +289,17 @@ static struct replacement {const char *class_name, *selector;uintptr_t expected;IMP replacement, *original;+ bool instance; /* an instance method, not a class method */bool done;} replacements[] = {{ "BWCoreImageFilterRenderer", "prewarmPortraitFilterVersion:semanticStyleFilters:", 0x1a2115c0cUL,- (IMP)prewarm_portrait_filter, NULL, false },+ (IMP)prewarm_portrait_filter, NULL, false, false },{ "BWFigCaptureDeviceVendor", "sharedCaptureDeviceVendor", 0x1a1ef0d04UL,- (IMP)shared_vendor, &original_shared_vendor, false },+ (IMP)shared_vendor, &original_shared_vendor, false, false },+ { "FigVideoCaptureConnectionConfiguration", "videoStabilizationMethod", 0x1a1e99ef8UL,+ (IMP)video_stabilization_method, NULL, true, false },+ { "FigVideoCaptureConnectionConfiguration", "videoStabilizationType", 0x1a20d8f48UL,+ (IMP)video_stabilization_type, NULL, true, false },};static void@@ -272,10 +315,12 @@ image_added(__unused const struct mach_header *header, __unused intptr_t image_scontinue;}r->done = true;- method = class_getClassMethod(cls, sel_registerName(r->selector));+ method = r->instance ? class_getInstanceMethod(cls, sel_registerName(r->selector)) :+ class_getClassMethod(cls, sel_registerName(r->selector));original = method ? method_getImplementation(method) : NULL;if (original == NULL || (uintptr_t)original != r->expected + slide) {- say("redwood: +[%s %s]: found %#lx, expected %#lx; left alone\n", r->class_name, r->selector,+ say("redwood: %c[%s %s]: found %#lx, expected %#lx; left alone\n", r->instance ? '-' : '+', r->class_name,+ r->selector,original ? (uintptr_t)original - slide : 0, r->expected);continue;}@@ -283,7 +328,7 @@ image_added(__unused const struct mach_header *header, __unused intptr_t image_s*r->original = original;}method_setImplementation(method, r->replacement);- say("redwood: mediaserverd: +[%s %s] replaced\n", r->class_name, r->selector);+ say("redwood: mediaserverd: %c[%s %s] replaced\n", r->instance ? '-' : '+', r->class_name, r->selector);}}