Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

Camera: a photograph of every pixel, and a preview that does not tear

vcvkkac1eadbe0c70
diff --git a/userland/rc.jobs.camera b/userland/rc.jobs.camera
index f64d891..c307750 100644
--- a/userland/rc.jobs.camera
+++ b/userland/rc.jobs.camera
@@ -1,10 +1,10 @@
# Commands run by the launchd job com.redwood.demo (see com.redwood.demo.plist).
-# The wide camera for iOS (RC_JOBS=rc.jobs.camera, with cam=5; with cam_pattern=1 its frames are colour bars):
+# The cameras for iOS (RC_JOBS=rc.jobs.camera, with cam=5; with cam_pattern=1 its frames are colour bars):
# first what AVFoundation lists (camera_avtest: the device is libredwood-media.dylib's in mediaserverd,
# camera_device.c, whose frames come from the kernel's /dev/redwood-camera), then Camera.app opened from the home
# screen as a finger opens it (with cam=5 the loader's /arm-io/isp node puts its icon on the first page, top right),
-# its preview left running for 40 seconds (xl screenshot meanwhile). NOT FINISHED: the preview stays black, no
-# frames reach it yet (see camera_device.c). The kernel's "camera:" lines say when the frames came and why they
+# its preview left running for 40 seconds (xl screenshot meanwhile), and then the camera is the hand's: the shutter,
+# the front camera, zoom, a tap to focus. The kernel's "camera:" lines say when the frames came and why they
# stopped; mediaserverd's "redwood: camera:" lines what the capture stack asked of the device.
/redwood/bin/sound_test springboard
/redwood/bin/camera_avtest
diff --git a/userland/src/camera_device.c b/userland/src/camera_device.c
index c1d37ed..65bcb29 100644
--- a/userland/src/camera_device.c
+++ b/userland/src/camera_device.c
@@ -38,6 +38,7 @@
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
+#include <mach/mach_time.h>
#include <math.h>
#include <pthread.h>
#include <stdarg.h>
@@ -396,6 +397,8 @@ format(int sensor_width, int sensor_height, int max_fps, bool binned, double fov
/* ---- the stream ------------------------------------------------------------------------------------- */
+#define HELD 3 /* the moving pictures kept from the pool after they are given (see deliver) */
+
struct output {
const char * name;
CFDictionaryRef config; /* from VideoOutputConfigurations */
@@ -403,7 +406,8 @@ struct output {
void * handler; /* a block: ^(CVPixelBufferRef, CMTime) */
bool said_shape;
uint64_t delivered, refused;
- CVPixelBufferRef held; /* the picture given last, kept from the pool (see deliver) */
+ CVPixelBufferRef held[HELD]; /* the pictures given last, kept from the pool (see deliver) */
+ unsigned held_next;
bool said_pool;
};
@@ -811,6 +815,7 @@ struct render_job {
const uint8_t *raw;
CVPixelBufferRef pb;
uint32_t format, width, height, pieces;
+ bool still; /* the photograph: every pixel of the sensor (isp_render_still_nv12) */
};
static void
@@ -824,6 +829,10 @@ render_piece(void *context, size_t i)
if (j->format == PF_BGRA) {
isp_render_bgra(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddress(j->pb), CVPixelBufferGetBytesPerRow(j->pb), j->width,
j->height, first, end);
+ } else if (j->still) {
+ isp_render_still_nv12(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddressOfPlane(j->pb, 0),
+ CVPixelBufferGetBytesPerRowOfPlane(j->pb, 0), CVPixelBufferGetBaseAddressOfPlane(j->pb, 1),
+ CVPixelBufferGetBytesPerRowOfPlane(j->pb, 1), j->width, j->height, first, end, j->format == PF_420V);
} else {
isp_render_nv12(j->isp, j->f, j->raw, CVPixelBufferGetBaseAddressOfPlane(j->pb, 0), CVPixelBufferGetBytesPerRowOfPlane(j->pb, 0),
CVPixelBufferGetBaseAddressOfPlane(j->pb, 1), CVPixelBufferGetBytesPerRowOfPlane(j->pb, 1), j->width, j->height, first, end,
@@ -856,12 +865,17 @@ output_buffer(struct output *out)
}
return NULL;
}
- if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && out->held != NULL) {
- let_go(out); /* a pool that will not grow by one: the picture kept is the one to have */
- (void)CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb);
+ for (unsigned i = 0; CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && i < HELD; i++) {
+ /* a pool that will not grow that far: the oldest picture kept is the one to have */
+ unsigned oldest = (out->held_next + i) % HELD;
+
+ if (out->held[oldest] != NULL) {
+ CVPixelBufferRelease(out->held[oldest]);
+ out->held[oldest] = NULL;
+ }
if (!out->said_pool) {
out->said_pool = true;
- say("%s: the graph's pool has no second buffer: the picture on the screen is drawn over", out->name);
+ say("%s: the graph's pool will not grow by %u: fewer pictures kept", out->name, HELD);
}
}
if (pb == NULL) {
@@ -883,9 +897,11 @@ typedef void (^pixel_buffer_handler)(CVPixelBufferRef, CMTime);
static void
let_go(struct output *out)
{
- if (out->held != NULL) {
- CVPixelBufferRelease(out->held);
- out->held = NULL;
+ for (unsigned i = 0; i < HELD; i++) {
+ if (out->held[i] != NULL) {
+ CVPixelBufferRelease(out->held[i]);
+ out->held[i] = NULL;
+ }
}
}
@@ -965,9 +981,19 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc
}
CVPixelBufferLockBaseAddress(pb, 0);
- struct render_job job = { isp, f, raw, pb, pf, width, height, 8 };
+ struct render_job job = { isp, f, raw, pb, pf, width, height, 8, strcmp(out->name, "StillImage") == 0 && pf != PF_BGRA };
+ uint64_t began = mach_absolute_time();
dispatch_apply_f(job.pieces, dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), &job, render_piece);
+ if (job.still) {
+ static mach_timebase_info_data_t tb;
+
+ if (tb.denom == 0) {
+ mach_timebase_info(&tb);
+ }
+ say("%s: %ux%u made of every pixel in %.0f ms", out->name, width, height,
+ (double)(mach_absolute_time() - began) * tb.numer / tb.denom / 1e6);
+ }
CVPixelBufferUnlockBaseAddress(pb, 0);
CVBufferSetAttachment(pb, kCVImageBufferYCbCrMatrixKey, kCVImageBufferYCbCrMatrix_ITU_R_601_4, kCVAttachmentMode_ShouldPropagate);
CVBufferSetAttachment(pb, kCVImageBufferColorPrimariesKey, kCVImageBufferColorPrimaries_ITU_R_709_2, kCVAttachmentMode_ShouldPropagate);
@@ -985,14 +1011,18 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc
((pixel_buffer_handler)out->handler)(pb, pts);
/*
- * The moving picture's buffer is kept until the next one is given. The preview's pool would
- * otherwise hand the same buffer straight back -- the graph lets go of it as soon as it has passed
- * it on -- while the window server is still copying it to the screen, and the next frame drawn
- * into it showed as bands of two frames, torn across. Kept, it makes the pool take another.
+ * The moving picture's buffers are kept for HELD frames after they are given. The preview's pool
+ * would otherwise hand a buffer straight back -- the graph lets go of it as soon as it has passed
+ * it on -- while it is still on its way to the screen (the preview sink, Camera.app's image queue,
+ * the window server copying it), and the next frame drawn into it showed as bands of two frames,
+ * torn across. Kept, they make the pool take others. (One kept was not enough.)
*/
if (strcmp(out->name, "StillImage") != 0) {
- let_go(out);
- out->held = pb;
+ if (out->held[out->held_next] != NULL) {
+ CVPixelBufferRelease(out->held[out->held_next]);
+ }
+ out->held[out->held_next] = pb;
+ out->held_next = (out->held_next + 1) % HELD;
} else {
CVPixelBufferRelease(pb);
}
diff --git a/userland/src/camera_isp.c b/userland/src/camera_isp.c
index 626472f..d570427 100644
--- a/userland/src/camera_isp.c
+++ b/userland/src/camera_isp.c
@@ -711,6 +711,104 @@ isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_frame *f
}
}
+/* one line (y) of the sensor demosaiced, as 8-bit R, G, B (the unpacked lines around it in this thread's l) */
+static void
+demosaic_row(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint32_t row, const int colour[4],
+ const struct pipe *p, int *const l[3], uint8_t *rgb)
+{
+ for (int i = 0; i < 3; i++) {
+ unpack_line(f, raw, (int)row - 1 + i, l[i]);
+ }
+ full_line(isp, f, (const int *const *)l, row, colour, p, rgb);
+}
+
+void
+isp_render_still_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y, size_t y_stride,
+ uint8_t *uv, size_t uv_stride, uint32_t width, uint32_t height, uint32_t first_row, uint32_t end_row, bool video_range)
+{
+ uint32_t x0, y0, cw, ch, fw = f->width;
+ int colour[4];
+ struct pipe p;
+ static __thread int *lines[3];
+ static __thread uint8_t *rows[2], *out[2];
+ static __thread uint32_t have_fw, have_width;
+ int32_t cached[2] = { -1, -1 };
+
+ if (fw > have_fw) {
+ for (int i = 0; i < 3; i++) {
+ free(lines[i]);
+ lines[i] = malloc(sizeof(int) * fw);
+ }
+ for (int i = 0; i < 2; i++) {
+ free(rows[i]);
+ rows[i] = malloc(3 * (size_t)fw);
+ }
+ have_fw = fw;
+ }
+ if (width > have_width) {
+ for (int i = 0; i < 2; i++) {
+ free(out[i]);
+ out[i] = malloc(3 * (size_t)width);
+ }
+ have_width = width;
+ }
+ if (lines[2] == NULL || rows[1] == NULL || out[1] == NULL) {
+ return;
+ }
+ crop(isp, f, width, height, &x0, &y0, &cw, &ch);
+ cfa(f->bayer, colour);
+ pipe_init(isp, &p);
+ float sx_step = 2.0f * cw / width, sy_step = 2.0f * ch / height;
+
+ for (uint32_t row = first_row & ~1u; row < end_row && row < height; row += 2) {
+ for (int k = 0; k < 2; k++) {
+ uint32_t r = row + k < height ? row + k : row, at = isp->turned ? height - 1 - r : r;
+ float sy = 2.0f * y0 + (at + 0.5f) * sy_step - 0.5f, ty;
+ uint32_t sy0 = sy < 0 ? 0 : (uint32_t)sy, sy1, slot[2];
+
+ sy0 = sy0 > f->height - 1 ? f->height - 1 : sy0;
+ sy1 = sy0 + 1 > f->height - 1 ? f->height - 1 : sy0 + 1;
+ ty = sy < (float)sy0 ? 0 : sy - sy0;
+ /* the two lines of the sensor it is between, demosaiced (kept from the line before where they are the same) */
+ for (int n = 0; n < 2; n++) {
+ uint32_t want = n ? sy1 : sy0;
+
+ if (cached[0] == (int32_t)want || cached[1] == (int32_t)want) {
+ slot[n] = cached[0] == (int32_t)want ? 0 : 1;
+ continue;
+ }
+ /* (into the slot the other line does not need: sy0's for sy1, and sy1's kept for sy0) */
+ slot[n] = n == 1 ? slot[0] ^ 1 : cached[0] == (int32_t)sy1 ? 1 : 0;
+ demosaic_row(isp, f, raw, want, colour, &p, lines, rows[slot[n]]);
+ cached[slot[n]] = (int32_t)want;
+ }
+ const uint8_t *a = rows[slot[0]], *b = rows[slot[1]];
+ uint8_t *o = out[k];
+
+ for (uint32_t x = 0; x < width; x++) {
+ float sx = 2.0f * x0 + (x + 0.5f) * sx_step - 0.5f, tx;
+ uint32_t sx0 = sx < 0 ? 0 : (uint32_t)sx, sx1;
+
+ sx0 = sx0 > fw - 1 ? fw - 1 : sx0;
+ sx1 = sx0 + 1 > fw - 1 ? fw - 1 : sx0 + 1;
+ tx = sx < (float)sx0 ? 0 : sx - sx0;
+ uint8_t *px = o + 3 * (isp->turned ? width - 1 - x : x);
+
+ for (int c = 0; c < 3; c++) {
+ float top = a[3 * sx0 + c] + tx * (a[3 * sx1 + c] - a[3 * sx0 + c]);
+ float bottom = b[3 * sx0 + c] + tx * (b[3 * sx1 + c] - b[3 * sx0 + c]);
+
+ px[c] = (uint8_t)(top + ty * (bottom - top) + 0.5f);
+ }
+ }
+ }
+ bool second = row + 1 < height;
+
+ nv12_pair(out[0], out[1], width, y + (size_t)row * y_stride, second ? y + (size_t)(row + 1) * y_stride : NULL,
+ uv + (size_t)(row / 2) * uv_stride, video_range);
+ }
+}
+
/* ---- lens shading from a flat field --------------------------------------------------------------------- */
bool
diff --git a/userland/src/camera_isp.h b/userland/src/camera_isp.h
index 51a8dc2..13e5c89 100644
--- a/userland/src/camera_isp.h
+++ b/userland/src/camera_isp.h
@@ -111,6 +111,14 @@ void isp_render_bgra(const struct isp *isp, const struct redwood_camera_frame *f
* Every pixel at the sensor's own size (f->width x f->height): the two colours a site does not have
* from the same colour's sites around it (bilinear); rows from first_row to end_row (even), NV12.
*/
+/*
+ * A photograph, width x height: every pixel of the sensor's part the picture shows (the zoom's, turned as the
+ * moving pictures are) demosaiced as isp_render_full_nv12() does, and scaled to the picture between the
+ * two nearest lines and pixels; rows from first_row to end_row (even), NV12.
+ */
+void isp_render_still_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y,
+ size_t y_stride, uint8_t *uv, size_t uv_stride, uint32_t width, uint32_t height, uint32_t first_row, uint32_t end_row,
+ bool video_range);
void isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, uint8_t *y, size_t y_stride,
uint8_t *uv, size_t uv_stride, uint32_t first_row, uint32_t end_row, bool video_range);
/*
diff --git a/userland/src/inject_media.c b/userland/src/inject_media.c
index e0bd78a..973f429 100644
--- a/userland/src/inject_media.c
+++ b/userland/src/inject_media.c
@@ -35,9 +35,12 @@
* arrives.
*/
#include <CoreFoundation/CoreFoundation.h>
+#include <CoreMedia/CoreMedia.h>
+#include <CoreVideo/CoreVideo.h>
#include <dlfcn.h>
#include <fcntl.h>
#include <mach-o/dyld.h>
+#include <mach/mach_time.h>
#include <objc/message.h>
#include <objc/runtime.h>
#include <pthread.h>
@@ -334,6 +337,45 @@ gnr_render(id self, SEL selector, void *sample_buffer, id input)
}
}
+static bool
+is_node(id node, const char *name)
+{
+ return node != nil && strcmp(object_getClassName(node), name) == 0;
+}
+
+/*
+ * How long a photograph's encoding took, from the picture handed to the photo encoder node to the one
+ * it hands on (HEVC in software, see register_hevc_encoder below: the slowest step of a photograph).
+ */
+static void
+time_photo_encoder(id output, CMSampleBufferRef sample_buffer)
+{
+ CVImageBufferRef image = CMSampleBufferGetImageBuffer(sample_buffer);
+ static uint64_t began;
+ static mach_timebase_info_data_t timebase;
+ id connection, input;
+
+ if (image == NULL || CVPixelBufferGetWidth(image) < 2000) {
+ return; /* not a photograph */
+ }
+ if (is_node(((id (*)(id, SEL))objc_msgSend)(output, sel_registerName("node")), "BWPhotoEncoderNode")) {
+ if (began != 0) {
+ if (timebase.denom == 0) {
+ mach_timebase_info(&timebase);
+ }
+ say("redwood: mediaserverd: a photograph of %zux%zu encoded in %.0f ms\n", CVPixelBufferGetWidth(image),
+ CVPixelBufferGetHeight(image), (double)(mach_absolute_time() - began) * timebase.numer / timebase.denom / 1e6);
+ }
+ began = 0;
+ return;
+ }
+ connection = ((id (*)(id, SEL))objc_msgSend)(output, sel_registerName("connection"));
+ input = connection != nil ? ((id (*)(id, SEL))objc_msgSend)(connection, sel_registerName("input")) : nil;
+ if (input != nil && is_node(((id (*)(id, SEL))objc_msgSend)(input, sel_registerName("node")), "BWPhotoEncoderNode")) {
+ began = mach_absolute_time();
+ }
+}
+
/*
* A node's output given no sample buffer at all, which no node takes: the session's audio source
* (-[BWAudioSourceNode _generateSamples]), whose audio unit never started here (see audio_source_start
@@ -345,7 +387,7 @@ gnr_render(id self, SEL selector, void *sample_buffer, id input)
static IMP original_emit_sample_buffer;
static void
-emit_sample_buffer(id self, SEL selector, void *sample_buffer)
+emit_sample_buffer(id self, SEL selector, CMSampleBufferRef sample_buffer)
{
static bool said;
@@ -356,7 +398,8 @@ emit_sample_buffer(id self, SEL selector, void *sample_buffer)
}
return;
}
- ((void (*)(id, SEL, void *))original_emit_sample_buffer)(self, selector, sample_buffer);
+ time_photo_encoder(self, sample_buffer);
+ ((void (*)(id, SEL, CMSampleBufferRef))original_emit_sample_buffer)(self, selector, sample_buffer);
}
/*
diff --git a/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plist b/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plist
new file mode 100644
index 0000000..b434647
--- /dev/null
+++ b/userland/var/mobile/Library/Preferences/com.apple.VideoProcessing.plist
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <!-- VCPHEVC.videocodec, the software HEVC codec (the photographs' HEIC, src/inject_media.c), takes every key
+ of this domain for one of its options, by name, a string each; on its own it runs on one thread -->
+ <key>threading</key>
+ <string>auto</string>
+ <key>max-threads</key>
+ <string>7</string>
+</dict>
+</plist>