Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
Camera: a photograph of every pixel, and a preview that does not tear
vcvkkac1eadbe0c70
diff --git a/userland/rc.jobs.camera b/userland/rc.jobs.cameraindex f64d891..c307750 100644--- a/userland/rc.jobs.camera+++ b/userland/rc.jobs.camera@@ -1,10 +1,10 @@# Commands run by the launchd job com.redwood.demo (see com.redwood.demo.plist).-# The wide camera for iOS (RC_JOBS=rc.jobs.camera, with cam=5; with cam_pattern=1 its frames are colour bars):+# The cameras for iOS (RC_JOBS=rc.jobs.camera, with cam=5; with cam_pattern=1 its frames are colour bars):# first what AVFoundation lists (camera_avtest: the device is libredwood-media.dylib's in mediaserverd,# camera_device.c, whose frames come from the kernel's /dev/redwood-camera), then Camera.app opened from the home# screen as a finger opens it (with cam=5 the loader's /arm-io/isp node puts its icon on the first page, top right),-# its preview left running for 40 seconds (xl screenshot meanwhile). NOT FINISHED: the preview stays black, no-# frames reach it yet (see camera_device.c). The kernel's "camera:" lines say when the frames came and why they+# its preview left running for 40 seconds (xl screenshot meanwhile), and then the camera is the hand's: the shutter,+# the front camera, zoom, a tap to focus. The kernel's "camera:" lines say when the frames came and why they# stopped; mediaserverd's "redwood: camera:" lines what the capture stack asked of the device./redwood/bin/sound_test springboard/redwood/bin/camera_avtestdiff --git a/userland/src/camera_device.c b/userland/src/camera_device.cindex c1d37ed..65bcb29 100644--- a/userland/src/camera_device.c+++ b/userland/src/camera_device.c@@ -38,6 +38,7 @@#include <dlfcn.h>#include <errno.h>#include <fcntl.h>+#include <mach/mach_time.h>#include <math.h>#include <pthread.h>#include <stdarg.h>@@ -396,6 +397,8 @@ format(int sensor_width, int sensor_height, int max_fps, bool binned, double fov/* ---- the stream ------------------------------------------------------------------------------------- */+#define HELD 3 /* the moving pictures kept from the pool after they are given (see deliver) */+struct output {const char * name;CFDictionaryRef config; /* from VideoOutputConfigurations */@@ -403,7 +406,8 @@ struct output {void * handler; /* a block: ^(CVPixelBufferRef, CMTime) */bool said_shape;uint64_t delivered, refused;- CVPixelBufferRef held; /* the picture given last, kept from the pool (see deliver) */+ CVPixelBufferRef held[HELD]; /* the pictures given last, kept from the pool (see deliver) */+ unsigned held_next;bool said_pool;};@@ -811,6 +815,7 @@ struct render_job {const uint8_t *raw;CVPixelBufferRef pb;uint32_t format, width, height, pieces;+ bool still; /* the photograph: every pixel of the sensor (isp_render_still_nv12) */};static void@@ -824,6 +829,10 @@ render_piece(void *context, size_t i)if (j->format == PF_BGRA) {isp_render_bgra(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddress(j->pb), CVPixelBufferGetBytesPerRow(j->pb), j->width,j->height, first, end);+ } else if (j->still) {+ isp_render_still_nv12(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddressOfPlane(j->pb, 0),+ CVPixelBufferGetBytesPerRowOfPlane(j->pb, 0), CVPixelBufferGetBaseAddressOfPlane(j->pb, 1),+ CVPixelBufferGetBytesPerRowOfPlane(j->pb, 1), j->width, j->height, first, end, j->format == PF_420V);} else {isp_render_nv12(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddressOfPlane(j->pb, 0), CVPixelBufferGetBytesPerRowOfPlane(j->pb, 0),CVPixelBufferGetBaseAddressOfPlane(j->pb, 1), CVPixelBufferGetBytesPerRowOfPlane(j->pb, 1), j->width, j->height, first, end,@@ -856,12 +865,17 @@ output_buffer(struct output *out)}return NULL;}- if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && out->held != NULL) {- let_go(out); /* a pool that will not grow by one: the picture kept is the one to have */- (void)CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb);+ for (unsigned i = 0; CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && i < HELD; i++) {+ /* a pool that will not grow that far: the oldest picture kept is the one to have */+ unsigned oldest = (out->held_next + i) % HELD;++ if (out->held[oldest] != NULL) {+ CVPixelBufferRelease(out->held[oldest]);+ out->held[oldest] = NULL;+ }if (!out->said_pool) {out->said_pool = true;- say("%s: the graph's pool has no second buffer: the picture on the screen is drawn over", out->name);+ say("%s: the graph's pool will not grow by %u: fewer pictures kept", out->name, HELD);}}if (pb == NULL) {@@ -883,9 +897,11 @@ typedef void (^pixel_buffer_handler)(CVPixelBufferRef, CMTime);static voidlet_go(struct output *out){- if (out->held != NULL) {- CVPixelBufferRelease(out->held);- out->held = NULL;+ for (unsigned i = 0; i < HELD; i++) {+ if (out->held[i] != NULL) {+ CVPixelBufferRelease(out->held[i]);+ out->held[i] = NULL;+ }}}@@ -965,9 +981,19 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc}CVPixelBufferLockBaseAddress(pb, 0);- struct render_job job = { isp, f, raw, pb, pf, width, height, 8 };+ struct render_job job = { isp, f, raw, pb, pf, width, height, 8, strcmp(out->name, "StillImage") == 0 && pf != PF_BGRA };+ uint64_t began = mach_absolute_time();dispatch_apply_f(job.pieces, dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), &job, render_piece);+ if (job.still) {+ static mach_timebase_info_data_t tb;++ if (tb.denom == 0) {+ mach_timebase_info(&tb);+ }+ say("%s: %ux%u made of every pixel in %.0f ms", out->name, width, height,+ (double)(mach_absolute_time() - began) * tb.numer / tb.denom / 1e6);+ }CVPixelBufferUnlockBaseAddress(pb, 0);CVBufferSetAttachment(pb, kCVImageBufferYCbCrMatrixKey, kCVImageBufferYCbCrMatrix_ITU_R_601_4, kCVAttachmentMode_ShouldPropagate);CVBufferSetAttachment(pb, kCVImageBufferColorPrimariesKey, kCVImageBufferColorPrimaries_ITU_R_709_2, kCVAttachmentMode_ShouldPropagate);@@ -985,14 +1011,18 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc((pixel_buffer_handler)out->handler)(pb, pts);/*- * The moving picture's buffer is kept until the next one is given. The preview's pool would- * otherwise hand the same buffer straight back -- the graph lets go of it as soon as it has passed- * it on -- while the window server is still copying it to the screen, and the next frame drawn- * into it showed as bands of two frames, torn across. Kept, it makes the pool take another.+ * The moving picture's buffers are kept for HELD frames after they are given. The preview's pool+ * would otherwise hand a buffer straight back -- the graph lets go of it as soon as it has passed+ * it on -- while it is still on its way to the screen (the preview sink, Camera.app's image queue,+ * the window server copying it), and the next frame drawn into it showed as bands of two frames,+ * torn across. Kept, they make the pool take others. (One kept was not enough.)*/if (strcmp(out->name, "StillImage") != 0) {- let_go(out);- out->held = pb;+ if (out->held[out->held_next] != NULL) {+ CVPixelBufferRelease(out->held[out->held_next]);+ }+ out->held[out->held_next] = pb;+ out->held_next = (out->held_next + 1) % HELD;} else {CVPixelBufferRelease(pb);}diff --git a/userland/src/camera_isp.c b/userland/src/camera_isp.cindex 626472f..d570427 100644--- a/userland/src/camera_isp.c+++ b/userland/src/camera_isp.c@@ -711,6 +711,104 @@ isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_frame *f}}+/* one line (y) of the sensor demosaiced, as 8-bit R, G, B (the unpacked lines around it in this thread's l) */+static void+demosaic_row(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint32_t row, const int colour[4],+ const struct pipe *p, int *const l[3], uint8_t *rgb)+{+ for (int i = 0; i < 3; i++) {+ unpack_line(f, raw, (int)row - 1 + i, l[i]);+ }+ full_line(isp, f, (const int *const *)l, row, colour, p, rgb);+}++void+isp_render_still_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y, size_t y_stride,+ uint8_t *uv, size_t uv_stride, uint32_t width, uint32_t height, uint32_t first_row, uint32_t end_row, bool video_range)+{+ uint32_t x0, y0, cw, ch, fw = f->width;+ int colour[4];+ struct pipe p;+ static __thread int *lines[3];+ static __thread uint8_t *rows[2], *out[2];+ static __thread uint32_t have_fw, have_width;+ int32_t cached[2] = { -1, -1 };++ if (fw > have_fw) {+ for (int i = 0; i < 3; i++) {+ free(lines[i]);+ lines[i] = malloc(sizeof(int) * fw);+ }+ for (int i = 0; i < 2; i++) {+ free(rows[i]);+ rows[i] = malloc(3 * (size_t)fw);+ }+ have_fw = fw;+ }+ if (width > have_width) {+ for (int i = 0; i < 2; i++) {+ free(out[i]);+ out[i] = malloc(3 * (size_t)width);+ }+ have_width = width;+ }+ if (lines[2] == NULL || rows[1] == NULL || out[1] == NULL) {+ return;+ }+ crop(isp, f, width, height, &x0, &y0, &cw, &ch);+ cfa(f->bayer, colour);+ pipe_init(isp, &p);+ float sx_step = 2.0f * cw / width, sy_step = 2.0f * ch / height;++ for (uint32_t row = first_row & ~1u; row < end_row && row < height; row += 2) {+ for (int k = 0; k < 2; k++) {+ uint32_t r = row + k < height ? row + k : row, at = isp->turned ? height - 1 - r : r;+ float sy = 2.0f * y0 + (at + 0.5f) * sy_step - 0.5f, ty;+ uint32_t sy0 = sy < 0 ? 0 : (uint32_t)sy, sy1, slot[2];++ sy0 = sy0 > f->height - 1 ? f->height - 1 : sy0;+ sy1 = sy0 + 1 > f->height - 1 ? f->height - 1 : sy0 + 1;+ ty = sy < (float)sy0 ? 0 : sy - sy0;+ /* the two lines of the sensor it is between, demosaiced (kept from the line before where they are the same) */+ for (int n = 0; n < 2; n++) {+ uint32_t want = n ? sy1 : sy0;++ if (cached[0] == (int32_t)want || cached[1] == (int32_t)want) {+ slot[n] = cached[0] == (int32_t)want ? 0 : 1;+ continue;+ }+ /* (into the slot the other line does not need: sy0's for sy1, and sy1's kept for sy0) */+ slot[n] = n == 1 ? slot[0] ^ 1 : cached[0] == (int32_t)sy1 ? 1 : 0;+ demosaic_row(isp, f, raw, want, colour, &p, lines, rows[slot[n]]);+ cached[slot[n]] = (int32_t)want;+ }+ const uint8_t *a = rows[slot[0]], *b = rows[slot[1]];+ uint8_t *o = out[k];++ for (uint32_t x = 0; x < width; x++) {+ float sx = 2.0f * x0 + (x + 0.5f) * sx_step - 0.5f, tx;+ uint32_t sx0 = sx < 0 ? 0 : (uint32_t)sx, sx1;++ sx0 = sx0 > fw - 1 ? fw - 1 : sx0;+ sx1 = sx0 + 1 > fw - 1 ? fw - 1 : sx0 + 1;+ tx = sx < (float)sx0 ? 0 : sx - sx0;+ uint8_t *px = o + 3 * (isp->turned ? width - 1 - x : x);++ for (int c = 0; c < 3; c++) {+ float top = a[3 * sx0 + c] + tx * (a[3 * sx1 + c] - a[3 * sx0 + c]);+ float bottom = b[3 * sx0 + c] + tx * (b[3 * sx1 + c] - b[3 * sx0 + c]);++ px[c] = (uint8_t)(top + ty * (bottom - top) + 0.5f);+ }+ }+ }+ bool second = row + 1 < height;++ nv12_pair(out[0], out[1], width, y + (size_t)row * y_stride, second ? y + (size_t)(row + 1) * y_stride : NULL,+ uv + (size_t)(row / 2) * uv_stride, video_range);+ }+}+/* ---- lens shading from a flat field --------------------------------------------------------------------- */booldiff --git a/userland/src/camera_isp.h b/userland/src/camera_isp.hindex 51a8dc2..13e5c89 100644--- a/userland/src/camera_isp.h+++ b/userland/src/camera_isp.h@@ -111,6 +111,14 @@ void isp_render_bgra(const struct isp *isp, const struct redwood_camera_frame *f* Every pixel at the sensor's own size (f->width x f->height): the two colours a site does not have* from the same colour's sites around it (bilinear); rows from first_row to end_row (even), NV12.*/+/*+ * A photograph, width x height: every pixel of the sensor's part the picture shows (the zoom's, turned as the+ * moving pictures are) demosaiced as isp_render_full_nv12() does, and scaled to the picture between the+ * two nearest lines and pixels; rows from first_row to end_row (even), NV12.+ */+void isp_render_still_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y,+ size_t y_stride, uint8_t *uv, size_t uv_stride, uint32_t width, uint32_t height, uint32_t first_row, uint32_t end_row,+ bool video_range);void isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y, size_t y_stride,uint8_t *uv, size_t uv_stride, uint32_t first_row, uint32_t end_row, bool video_range);/*diff --git a/userland/src/inject_media.c b/userland/src/inject_media.cindex e0bd78a..973f429 100644--- a/userland/src/inject_media.c+++ b/userland/src/inject_media.c@@ -35,9 +35,12 @@* arrives.*/#include <CoreFoundation/CoreFoundation.h>+#include <CoreMedia/CoreMedia.h>+#include <CoreVideo/CoreVideo.h>#include <dlfcn.h>#include <fcntl.h>#include <mach-o/dyld.h>+#include <mach/mach_time.h>#include <objc/message.h>#include <objc/runtime.h>#include <pthread.h>@@ -334,6 +337,45 @@ gnr_render(id self, SEL selector, void *sample_buffer, id input)}}+static bool+is_node(id node, const char *name)+{+ return node != nil && strcmp(object_getClassName(node), name) == 0;+}++/*+ * How long a photograph's encoding took, from the picture handed to the photo encoder node to the one+ * it hands on (HEVC in software, see register_hevc_encoder below: the slowest step of a photograph).+ */+static void+time_photo_encoder(id output, CMSampleBufferRef sample_buffer)+{+ CVImageBufferRef image = CMSampleBufferGetImageBuffer(sample_buffer);+ static uint64_t began;+ static mach_timebase_info_data_t timebase;+ id connection, input;++ if (image == NULL || CVPixelBufferGetWidth(image) < 2000) {+ return; /* not a photograph */+ }+ if (is_node(((id (*)(id, SEL))objc_msgSend)(output, sel_registerName("node")), "BWPhotoEncoderNode")) {+ if (began != 0) {+ if (timebase.denom == 0) {+ mach_timebase_info(&timebase);+ }+ say("redwood: mediaserverd: a photograph of %zux%zu encoded in %.0f ms\n", CVPixelBufferGetWidth(image),+ CVPixelBufferGetHeight(image), (double)(mach_absolute_time() - began) * timebase.numer / timebase.denom / 1e6);+ }+ began = 0;+ return;+ }+ connection = ((id (*)(id, SEL))objc_msgSend)(output, sel_registerName("connection"));+ input = connection != nil ? ((id (*)(id, SEL))objc_msgSend)(connection, sel_registerName("input")) : nil;+ if (input != nil && is_node(((id (*)(id, SEL))objc_msgSend)(input, sel_registerName("node")), "BWPhotoEncoderNode")) {+ began = mach_absolute_time();+ }+}+/** A node's output given no sample buffer at all, which no node takes: the session's audio source* (-[BWAudioSourceNode _generateSamples]), whose audio unit never started here (see audio_source_start@@ -345,7 +387,7 @@ gnr_render(id self, SEL selector, void *sample_buffer, id input)static IMP original_emit_sample_buffer;static void-emit_sample_buffer(id self, SEL selector, void *sample_buffer)+emit_sample_buffer(id self, SEL selector, CMSampleBufferRef sample_buffer){static bool said;@@ -356,7 +398,8 @@ emit_sample_buffer(id self, SEL selector, void *sample_buffer)}return;}- ((void (*)(id, SEL, void *))original_emit_sample_buffer)(self, selector, sample_buffer);+ time_photo_encoder(self, sample_buffer);+ ((void (*)(id, SEL, CMSampleBufferRef))original_emit_sample_buffer)(self, selector, sample_buffer);}/*diff --git a/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plist b/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plistnew file mode 100644index 0000000..b434647--- /dev/null+++ b/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plist@@ -0,0 +1,12 @@+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">+<plist version="1.0">+<dict>+ <!-- VCPHEVC.videocodec, the software HEVC codec (the photographs' HEIC, src/inject_media.c), takes every key+ of this domain for one of its options, by name, a string each; on its own it runs on one thread -->+ <key>threading</key>+ <string>auto</string>+ <key>max-threads</key>+ <string>7</string>+</dict>+</plist>