Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

Camera for iOS: photographs, colours by the cameras' own tuning, focus, zoom, the front camera

vcvkk98bc44afef0e
diff --git a/tools/camtuning.py b/tools/camtuning.py
new file mode 100755
index 0000000..800744f
--- /dev/null
+++ b/tools/camtuning.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Make a camera's colour tuning, as libredwood-media's processor reads it (userland/src/camera_isp.c,
+isp_load_tuning), out of the camera's tuning file of Android's camera stack
+(/vendor/lib64/camera/com.qti.tuned.<name>.bin of the phone: the vendor's, kept in fw-camera/ and never in
+the repository; so is what this makes of it).
+
+usage: camtuning.py TUNED.bin OUT.isp (camtuning.py TUNED.bin - lists what is there only)
+
+The tuning file is a "QTI Chromatix Header" / "Parameter Parser V3.0.1" file, laid out as the sensors'
+module files are (see camsensor.py): a table of symbols { id, name[32], kind, ?, parent, offset, size }
+and the data they point into. Two of its modules are taken, each as Chromatix keeps it -- a tree whose
+first level is the flash (led_idx_data: 0 without one, the only one taken), the next the scene's light
+(aec_data: from a lux index to another, Qualcomm's measure of how dark the scene is: higher is darker),
+the last the light's colour temperature (cct_data: from a CCT to another, in kelvin):
+
+ cc13 (the colour matrix, IPE's): per light and temperature a 3x3 matrix from the sensor's colours,
+ white balanced, to linear sRGB, by rows R, G, B (each row sums to 1)
+ lsc34 (lens shading): per light and temperature a gain for each of R, Gr, Gb and B at each point of
+ a 17 x 13 grid laid over the sensor (corners on its corners, rows from the top)
+
+and, of its white balance (Qualcomm's AWB), the reference lights (refPtV1): ten of them for every camera
+of this phone, each as the sensor sees a grey under it, red over green and blue over green -- from the
+bluest daylight to a candle, in the order 10000 K, D75, D65, D50 (whose gains are the AWB's first ones,
+initDataV1), cool white and TL84 fluorescent tubes, U30, A (a filament lamp), horizon, candle. Each is
+given the colour temperature that the tables above are picked by for it: its own, but 3800 K for U30, a
+3000 K light without the infrared a filament gives -- as a white LED is -- whose tables are TL84's (the
+wide camera under a warm white LED: its colour shading measured on a white sheet is TL84's table's to
+within 3 %, the 2550-2900 K one's is 12 % off in red).
+
+OUT.isp: "RWCISP02"; the number of the matrices' light ranges and per range { lux index from, to (two
+floats), the number of temperature ranges, per range { CCT from, to, the matrix (9 floats) } }; the
+same for the lens shading, with per temperature range { CCT from, to, R, Gr, Gb, B grids (17 x 13 floats
+each) }; the number of reference lights (0 where there are not the ten) and each { R/G, B/G, CCT }.
+Everything little-endian, counts as 32-bit words.
+"""
+import struct
+import sys
+
+GRID = 17 * 13
+
+
+def symbols(data):
+ count = struct.unpack_from('<I', data, 0xa4)[0]
+ entries = [struct.unpack_from('<3I', data, 0xa8 + 12 * i) for i in range(count)]
+ table, pool = entries[0][1], entries[1][1]
+ syms, at = {}, table
+ while at + 0x38 <= pool:
+ sid = struct.unpack_from('<I', data, at)[0]
+ name = data[at + 4:at + 0x24].split(b'\0')[0].decode('latin1')
+ _, _, _, ref, size = struct.unpack_from('<5i', data, at + 0x24)
+ syms[sid] = (name, data[pool + ref:pool + ref + size])
+ at += 0x38
+ return syms
+
+
+def module(syms, name, floats):
+ """[(lux from, lux to, [(cct from, cct to, values)])] of the module's first tree, without the flash"""
+ led = next(v for n, v in syms.values() if n == 'mod_%s_led_idx_data' % name)
+ _, regions, sid = next(e for e in struct.iter_unpack('<3I', led) if e[0] == 0)
+ out = []
+ for i in range(regions):
+ lux_from, lux_to, _, _, n, cct_sid = struct.unpack_from('<4f2I', syms[sid][1], 24 * i)
+ cct = syms[cct_sid][1]
+ size = len(cct) // n
+ ranges = []
+ for j in range(n):
+ v = struct.unpack_from('<%df' % (2 + floats), cct, size * j)
+ ranges.append((v[0], v[1], v[2:]))
+ out.append((lux_from, lux_to, ranges))
+ return out
+
+
+# the colour temperature each of the ten reference lights picks the tables by (see above)
+REFERENCE_CCTS = (10000, 7500, 6500, 5000, 4150, 4000, 3800, 2850, 2300, 1900)
+
+
+def references(syms):
+ """[(R/G, B/G, CCT)] of the AWB's reference lights, or [] where they are not the ten known"""
+ v = next((v for n, v in syms.values() if n == 'refPtV1'), None)
+ if v is None:
+ return []
+ f = struct.unpack('<%df' % (len(v) // 4), v)
+ points = [(f[i], f[i + 1]) for i in range(6, len(f) - 1, 2) if f[i] > 0 and f[i + 1] > 0]
+ if len(points) != len(REFERENCE_CCTS):
+ return []
+ return [(rg, bg, cct) for (rg, bg), cct in zip(points, REFERENCE_CCTS)]
+
+
+def main():
+ if len(sys.argv) < 3:
+ sys.exit(__doc__)
+ data = open(sys.argv[1], 'rb').read()
+ if not data.startswith(b'QTI Chromatix Header') or b'Parameter Parser V3.0.1' not in data[:0x60]:
+ sys.exit('camtuning.py: not a tuning file of the kind known here')
+ syms = symbols(data)
+ ccm = module(syms, 'cc13', 9)
+ lsc = module(syms, 'lsc34', 4 * GRID)
+ refs = references(syms)
+
+ for what, tree in (('colour matrix', ccm), ('lens shading', lsc)):
+ for lux_from, lux_to, ranges in tree:
+ for cct_from, cct_to, v in ranges:
+ if what == 'colour matrix':
+ shown = ' '.join('%.3f' % x for x in v)
+ else:
+ corners = [v[c * GRID] for c in range(4)]
+ shown = 'corner gains R %.2f Gr %.2f Gb %.2f B %.2f' % tuple(corners)
+ print('%-13s lux index %4.0f-%-4.0f CCT %5.0f-%-5.0f %s' % (what, lux_from, lux_to, cct_from, cct_to, shown))
+ for rg, bg, cct in refs:
+ print('reference light R/G %.3f B/G %.3f, tables of %.0f K' % (rg, bg, cct))
+ if sys.argv[2] == '-':
+ return
+
+ out = bytearray(b'RWCISP02')
+ for tree in (ccm, lsc):
+ out += struct.pack('<I', len(tree))
+ for lux_from, lux_to, ranges in tree:
+ out += struct.pack('<2fI', lux_from, lux_to, len(ranges))
+ for cct_from, cct_to, v in ranges:
+ out += struct.pack('<2f%df' % len(v), cct_from, cct_to, *v)
+ out += struct.pack('<I', len(refs))
+ for ref in refs:
+ out += struct.pack('<3f', *ref)
+ open(sys.argv[2], 'wb').write(out)
+
+
+main()
diff --git a/userland/build.sh b/userland/build.sh
index 9089c92..2566c2a 100755
--- a/userland/build.sh
+++ b/userland/build.sh
@@ -273,6 +273,14 @@ for module in ../fw-camera/com.qti.sensormodule.redwood_*.bin; do
mkdir -p root/fw
../tools/camsensor.py "$module" "root/fw/camera-$sensor.tab" >/dev/null
done
+# ... and their colour tuning (libredwood-media's processor, src/camera_isp.c), made by tools/camtuning.py out of the
+# cameras' tuning files beside them, com.qti.tuned.redwood_<maker>_<sensor>_<camera>.bin: camera-<sensor>.isp
+for tuned in ../fw-camera/com.qti.tuned.redwood_*.bin; do
+ [ -f "$tuned" ] || continue
+ sensor=$(basename "$tuned" .bin | sed 's/^com\.qti\.tuned\.redwood_[a-z]*_\([a-z0-9]*\)_.*/\1/')
+ mkdir -p root/fw
+ ../tools/camtuning.py "$tuned" "root/fw/camera-$sensor.isp" >/dev/null
+done
# ... and with ADSP=1 the image of the audio DSP, from the same place into ../fw-audio (bsd/dev/redwood_adsp.c):
# adsp.mdt and adsp.b02 to adsp.b36, 20 MB. Only when asked for: beside GL=1 the payload would be more than the 64 MB
# the loader takes.
diff --git a/userland/src/camera_device.c b/userland/src/camera_device.c
index 8d118d8..c1d37ed 100644
--- a/userland/src/camera_device.c
+++ b/userland/src/camera_device.c
@@ -1,39 +1,34 @@
/*
- * The wide camera as iOS's capture stack wants a camera: a FigCaptureDevice with one FigCaptureStream
- * (the back camera), made for mediaserverd's BWFigCaptureDeviceVendor in place of the one Apple's
- * H10ISP.mediacapture makes of the iPhone's ISP (part of libredwood-media.dylib; inject_media.c
- * gives the vendor our device's create function when the kernel has a camera: cam=5).
+ * The phone's cameras as iOS's capture stack wants them: a FigCaptureDevice with two FigCaptureStreams,
+ * the wide camera as the back camera and the front one as the front camera, made for mediaserverd's
+ * BWFigCaptureDeviceVendor in place of the device Apple's H10ISP.mediacapture makes of the iPhone's ISP
+ * (part of libredwood-media.dylib; inject_media.c gives the vendor our device's create function when
+ * the kernel has a camera: cam=5).
*
* Both are CoreMedia "derived objects" (CMDerivedObjectCreate): a table of functions for the base
* class (properties) and one for the device or stream class, as H10ISP's are laid out in iOS 16.5:
* device RequestStreams, RelinquishStreams (and a processing session and Pearl calibration, not here)
* stream Start, Stop, AddAttachmentsToSampleBuffer
- * The stream's frames go to the capture graph through the blocks it is given in its property
+ * A stream's frames go to the capture graph through the blocks it is given in its property
* "VideoOutputHandlers" ({ output: { PixelBufferHandler: ^(CVPixelBufferRef, CMTime) } }), for the
- * outputs "VideoOutputConfigurations" describes (size, pixel format) and "VideoOutputsEnabled"
+ * outputs "VideoOutputConfigurations" describes (the graph's own buffer pools) and "VideoOutputsEnabled"
* switches on. A frame: read from the kernel's /dev/redwood-camera (RAW10), made into a picture by
- * camera_isp.c (exposure, white balance, gamma, scaling), one pixel buffer for each output.
+ * camera_isp.c (exposure, white balance kept to the sensor's reference lights, the colour matrix and
+ * lens shading of the camera's own tuning, gamma, zoom), one pixel buffer for each output, with the
+ * metadata the graph keeps its frame statistics and its zoom by. The kernel gives one sensor's frames
+ * at a time: a stream that starts stops the other's frames and has the kernel change sensors.
*
- * The formats the stream offers are named as the iPhone X's back camera's are in CMCapture's
- * D22/AVCaptureSession.plist (FigCaptureStreamFormats "Photo" and "Video-60", by the sensor sizes
- * and frame rates the plist matches them by), so that its AVCaptureDevice formats and session presets
- * resolve: the pictures are scaled from the sensor's 4000x3000 to whatever size is asked for.
+ * The formats the streams offer are named as the iPhone X's cameras' are in CMCapture's
+ * D22/AVCaptureSession.plist (FigCaptureStreamFormats, by the sensor sizes and frame rates the plist
+ * matches them by), so that their AVCaptureDevice formats and session presets resolve: the pictures
+ * are scaled from the sensor's frames to whatever size is asked for.
*
- * Everything asked of the device and the stream that is not answered here is said in mediaserverd's
- * log ("redwood: camera: ..."), once per property.
- *
- * The graph builds, the session runs, and the pictures of the PrimaryScaler output are Camera.app's
- * preview, steadily.
+ * The shutter: StillImageCaptureNow asks for one frame of the whole sensor on the StillImage output
+ * (the time machine's TimeMachineSuspendNow/ResumeNow are taken and ignored). Focus, where there is
+ * a lens motor (the wide camera's): by contrast, continuous, and on the window a tap gives (af_frame()).
*
- * NOT FINISHED: no photograph is taken. The shutter reaches mediaserverd (-[BWStillImageCoordinatorNode
- * initiateStillImageCaptureNowWithSettings:] and then -[BWFigVideoCaptureDevice
- * _captureStillImageNowWithSettings:]), and there it quietly comes to nothing: the device's
- * _convertStillImageCaptureSettings:toStillImageCaptureNowOptions:captureStreamPropertyValues: gives
- * back no stream settings for this port, so _captureStillImageNowOnStreamWithOptions: writes nothing
- * on the stream, reports success, and no "StillImageCaptureNow" ever arrives here. What this side
- * would do with it is written below and waits for that: one frame of the sensor, the whole of it, to
- * the StillImage output. (Those methods of CMCapture are objc_direct: they cannot be replaced to be
- * looked into, which is as far as this went.)
+ * Everything asked of the device and the streams that is not answered here is said in mediaserverd's
+ * log ("redwood: camera: ..."), once per property.
*/
#include <CoreFoundation/CoreFoundation.h>
#include <CoreMedia/CoreMedia.h>
@@ -43,6 +38,7 @@
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
+#include <math.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdio.h>
@@ -117,6 +113,40 @@ static void *(*derived_storage)(CMBaseObjectRef);
static CMBaseClassID (*device_class_id)(void);
static CMBaseClassID (*stream_class_id)(void);
static CFStringRef port_back; /* kFigCapturePortType_BackFacingCamera */
+static CFStringRef port_front; /* kFigCapturePortType_FrontFacingCamera */
+
+/*
+ * The frames' metadata: what H10ISP's stream attaches to every frame (a dictionary under
+ * kFigCaptureSampleBufferAttachmentKey_MetadataDictionary, through the stream's
+ * AddAttachmentsToSampleBuffer when the graph defers it), and what the capture device keeps its frame
+ * statistics of. -[BWFrameStatisticsByPortType updateWithFrameMetadata:updateFocusDistance:] counts
+ * a frame only for a buffer that has it, and -[BWFigVideoCaptureDevice
+ * _stillImageCaptureInProgressSubstateForFrameStatisticsByPortType:...] puts a standard photograph
+ * off until more than four frames have been counted (then only while AFStatus is 2, a coarse focus
+ * scan): without it the shutter's capture was put off on every frame (the retry in
+ * _updateStateUsingVideoSampleBuffer:fromCaptureStream:), for ever, and StillImageCaptureNow never
+ * came. The dictionary goes on the pixel buffer too, and mutable, as the ISP's does: FigImageControl_Focus
+ * (the device's camera controls, on every frame once the sample buffer has metadata) reads it from
+ * the pixel buffer with no check that it is there -- CFDictionaryGetValue(NULL, ...): mediaserverd
+ * died of SIGSEGV -- and adds its own keys to it. The keys are CoreMedia's; the values come from the
+ * frame's header and our ISP's levels, in Apple's units where they are known (the exposure in seconds,
+ * gains with 256 for 1x); lux and SNR are rough.
+ */
+enum {
+ MD_PORT, MD_PTS, MD_EXPOSURE, MD_AGC, MD_ISP_DGAIN, MD_SENSOR_DGAIN, MD_ISO_BASE, MD_FRAME_RATE, MD_LUX, MD_SNR,
+ MD_AF_STATUS, MD_FOCUSING, MD_AE_STABLE, MD_AE_LOCKED, MD_AWB_STABLE, MD_AWB_LOCKED, MD_AF_LOCKED, MD_ZOOM, MD_KEYS
+};
+static const char *const md_names[MD_KEYS] = {
+ "kFigCaptureStreamMetadata_PortType", "kFigCaptureSampleBufferAttachmentKey_OriginalPresentationTimeStamp",
+ "kFigCaptureStreamMetadata_ExposureTime", "kFigCaptureStreamMetadata_AGC", "kFigCaptureStreamMetadata_ispDGain",
+ "kFigCaptureStreamMetadata_sensorDGain", "kFigCaptureStreamMetadata_ISOBase", "kFigCaptureStreamMetadata_CurrentFrameRate",
+ "kFigCaptureStreamMetadata_LuxLevel", "kFigCaptureStreamMetadata_NormalizedSNR", "kFigCaptureStreamMetadata_AFStatus",
+ "kFigCaptureStreamMetadata_FocusingMethod", "kFigCaptureStreamMetadata_AEStable", "kFigCaptureStreamMetadata_AELocked",
+ "kFigCaptureStreamMetadata_AWBStable", "kFigCaptureStreamMetadata_AWBLocked", "kFigCaptureStreamMetadata_AFLocked",
+ "kFigCaptureStreamMetadata_ZoomFactor",
+};
+static CFStringRef md_key[MD_KEYS];
+static CFStringRef md_attachment; /* kFigCaptureSampleBufferAttachmentKey_MetadataDictionary */
static bool
find_functions(void)
@@ -129,7 +159,24 @@ find_functions(void)
stream_class_id = dlsym(RTLD_DEFAULT, "FigCaptureStreamGetClassID");
port = dlsym(RTLD_DEFAULT, "kFigCapturePortType_BackFacingCamera");
port_back = port != NULL ? *port : NULL;
- if (derived_create == NULL || derived_storage == NULL || device_class_id == NULL || stream_class_id == NULL || port_back == NULL) {
+ port = dlsym(RTLD_DEFAULT, "kFigCapturePortType_FrontFacingCamera");
+ port_front = port != NULL ? *port : NULL;
+ port = dlsym(RTLD_DEFAULT, "kFigCaptureSampleBufferAttachmentKey_MetadataDictionary");
+ md_attachment = port != NULL ? *port : NULL;
+ for (int i = 0; i < MD_KEYS; i++) {
+ port = dlsym(RTLD_DEFAULT, md_names[i]);
+ md_key[i] = port != NULL ? *port : NULL;
+ if (md_key[i] == NULL) {
+ say("%s is not there: the frames' metadata go without it", md_names[i]);
+ }
+ }
+ if (md_attachment == NULL || md_key[MD_PORT] == NULL) {
+ say("kFigCaptureSampleBufferAttachmentKey_MetadataDictionary or the port's key is not there: frames without "
+ "metadata, and the shutter will wait for ever");
+ md_attachment = NULL;
+ }
+ if (derived_create == NULL || derived_storage == NULL || device_class_id == NULL || stream_class_id == NULL || port_back == NULL ||
+ port_front == NULL) {
say("CoreMedia's or CMCapture's functions are not there: no camera");
return false;
}
@@ -142,23 +189,10 @@ static CFMutableSetRef said_names;
static pthread_mutex_t said_lock = PTHREAD_MUTEX_INITIALIZER;
static void
-say_once(const char *what, CFStringRef key, CFTypeRef value)
+say_value(const char *what, CFStringRef key, CFTypeRef value)
{
char name[160] = "?", text[300] = "";
- CFStringRef tag = CFStringCreateWithFormat(NULL, NULL, CFSTR("%s %@"), what, key);
-
- pthread_mutex_lock(&said_lock);
- if (said_names == NULL) {
- said_names = CFSetCreateMutable(NULL, 0, &kCFTypeSetCallBacks);
- }
- bool first = !CFSetContainsValue(said_names, tag);
- CFSetAddValue(said_names, tag);
- pthread_mutex_unlock(&said_lock);
- CFRelease(tag);
- if (!first) {
- return;
- }
CFStringGetCString(key, name, sizeof(name), kCFStringEncodingUTF8);
if (value != NULL) {
CFStringRef d = CFCopyDescription(value);
@@ -172,6 +206,25 @@ say_once(const char *what, CFStringRef key, CFTypeRef value)
say("%s %s%s%s", what, name, value ? " = " : "", text);
}
+static void
+say_once(const char *what, CFStringRef key, CFTypeRef value)
+{
+ CFStringRef tag = CFStringCreateWithFormat(NULL, NULL, CFSTR("%s %@"), what, key);
+
+ pthread_mutex_lock(&said_lock);
+ if (said_names == NULL) {
+ said_names = CFSetCreateMutable(NULL, 0, &kCFTypeSetCallBacks);
+ }
+ bool first = !CFSetContainsValue(said_names, tag);
+
+ CFSetAddValue(said_names, tag);
+ pthread_mutex_unlock(&said_lock);
+ CFRelease(tag);
+ if (first) {
+ say_value(what, key, value);
+ }
+}
+
/* ---- numbers and dictionaries ---------------------------------------------------------------------- */
static CFNumberRef
@@ -242,6 +295,11 @@ static const struct prop stream_props[] = {
{ "RegionOfInterestForCameraControls", "Collection", "ReadWrite" }, { "AEMeteringMode", "Number", "ReadWrite" },
{ "MaxFrameCoalescingDuration", "Number", "ReadWrite" }, { "LimitAERange", "Boolean", "ReadWrite" },
{ "TimeMachineSuspendNow", "Boolean", "WriteOnly" }, { "TimeMachineResumeNow", "Boolean", "WriteOnly" },
+ /* focus (the wide camera's lens motor; see af_frame()) */
+ { "AutoFocusNow", "Collection", "WriteOnly" }, { "ContinuousAutoFocusNow", "Collection", "WriteOnly" },
+ { "FocusNow", "Collection", "WriteOnly" }, { "PeakTrackingFocusNow", "Collection", "WriteOnly" },
+ { "LockAFNow", "Boolean", "WriteOnly" }, { "ManualFocusNow", "Collection", "WriteOnly" },
+ { "AFWindowParams", "Collection", "ReadWrite" }, { "AutoFocusMode", "Number", "ReadWrite" },
}, device_props[] = {
{ "StreamArray", "Collection", "ReadOnly" }, { "Clock", "Clock", "ReadOnly" }, { "ClientIsActive", "Boolean", "ReadWrite" },
{ "SupportedSynchronizedStreamsGroups", "Collection", "ReadOnly" }, { "MemoryPoolID", "Number", "ReadWrite" },
@@ -289,7 +347,7 @@ output_info(int width, int height)
}
static CFDictionaryRef
-format(int sensor_width, int sensor_height, int max_fps, bool binned)
+format(int sensor_width, int sensor_height, int max_fps, bool binned, double fov)
{
CFMutableDictionaryRef f = (CFMutableDictionaryRef)new_dict(), outputs = (CFMutableDictionaryRef)new_dict();
CMVideoFormatDescriptionRef fd = NULL;
@@ -303,7 +361,7 @@ format(int sensor_width, int sensor_height, int max_fps, bool binned)
put(f, "VideoIsBinned", CFRetain(binned ? kCFBooleanTrue : kCFBooleanFalse));
put(f, "VideoBinningFactorHorizontal", number_i(binned ? 2 : 1));
put(f, "VideoBinningFactorVertical", number_i(binned ? 2 : 1));
- put(f, "VideoFieldOfView", number_f(65.0));
+ put(f, "VideoFieldOfView", number_f(fov));
put(f, "VideoMinIntegrationTime", number_i(100));
put(f, "VideoMaxIntegrationTime", number_i(1000000 / 15));
put(f, "VideoRawBitDepth", number_i(10));
@@ -345,18 +403,51 @@ struct output {
void * handler; /* a block: ^(CVPixelBufferRef, CMTime) */
bool said_shape;
uint64_t delivered, refused;
+ CVPixelBufferRef held; /* the picture given last, kept from the pool (see deliver) */
+ bool said_pool;
};
+static void let_go(struct output *out);
+
#define OUTPUTS 3
static const char *const output_names[OUTPUTS] = { "PrimaryScaler", "SecondaryScaler", "StillImage" };
+#define MAX_ZOOM 16 /* (the most the iPhone X's wide camera zooms digitally) */
+
+/*
+ * The device's cameras, a stream each, as the iPhone X's are in CMCapture's D22/AVCaptureSession.plist
+ * (its devices' uniqueIDs, and the stream formats its formats and presets are matched to by sensor size
+ * and frame rate): the wide camera as the back camera, the front one as the front camera. The kernel
+ * gives one sensor's frames at a time; a stream's frames start with its own sensor's (see worker()).
+ */
+static const struct camera {
+ uint32_t sensor; /* the kernel's (the frames' sensor, REDWOOD_CAMERA_SENSOR) */
+ const char *name, *unique_id;
+ bool turned; /* its sensor the other way up from the iPhone's: the pictures turned half round */
+ double focal_length, f_number, pixel_size, fov;
+ int formats[2][3]; /* sensor width, height, frames a second of "Photo" and of a video format */
+} cameras[2] = {
+ { 1, "Back Camera", "com.apple.avfoundation.avcapturedevice.built-in_video:0", false, 4.0, 1.8, 1.22, 65.0,
+ { { 4032, 3024, 30 }, { 4224, 2376, 60 } } }, /* "Photo", "Video-60" */
+ { 2, "Front Camera", "com.apple.avfoundation.avcapturedevice.built-in_video:1", true, 2.87, 2.2, 1.0, 70.0,
+ { { 3088, 2320, 30 }, { 3840, 2160, 30 } } }, /* "Photo", "Video-30" */
+};
+
struct stream {
CFArrayRef formats;
CFDictionaryRef supported;
CFDictionaryRef colour_info; /* from VideoOutputsColorInfo */
+ CFDictionaryRef metadata; /* the last frame's (frame_metadata()), for a buffer that has none of its own */
+ pthread_mutex_t metadata_lock; /* (not `lock': AddAttachmentsToSampleBuffer may come while a frame is given out) */
unsigned stills_wanted; /* photographs asked for by StillImageCaptureNow, not yet given */
int format_index;
double min_fps, max_fps;
+ float zoom; /* ZoomFactor: the picture is the middle 1/zoom of the sensor each way */
+ const struct camera *camera;
+ CFStringRef port;
+ bool af_now; /* a focus asked for (AutoFocusNow and the like): a search at the next frame */
+ bool af_locked; /* LockAFNow: no search but one asked for */
+ float af_window[4]; /* AFWindowParams: where to focus, as fractions of the frame (x0, y0, x1, y1) */
struct output out[OUTPUTS];
pthread_mutex_t lock;
pthread_t worker;
@@ -392,11 +483,11 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato
(void)allocator;
if (key_is(key, "PortType")) {
- value = CFRetain(port_back);
+ value = CFRetain(s->port);
} else if (key_is(key, "UniqueID") || key_is(key, "ModelID")) {
- value = CFRetain(CFSTR("com.apple.avfoundation.avcapturedevice.built-in_video:0"));
+ value = CFStringCreateWithCString(NULL, s->camera->unique_id, kCFStringEncodingUTF8);
} else if (key_is(key, "NonLocalizedName") || key_is(key, "LocalizedName")) {
- value = CFRetain(CFSTR("Back Camera"));
+ value = CFStringCreateWithCString(NULL, s->camera->name, kCFStringEncodingUTF8);
} else if (key_is(key, "Manufacturer")) {
value = CFRetain(CFSTR("Apple Inc."));
} else if (key_is(key, "SupportedFormatsArray")) {
@@ -452,22 +543,22 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato
} else if (key_is(key, "SupportedPropertiesDictionary")) {
value = CFRetain(s->supported);
} else if (key_is(key, "FocalLength")) {
- value = number_f(4.0); /* (the iPhone X's wide camera's, in millimetres) */
+ value = number_f(s->camera->focal_length); /* (the iPhone X's cameras', in millimetres) */
} else if (key_is(key, "LensFNumber")) {
- value = number_f(1.8);
+ value = number_f(s->camera->f_number);
} else if (key_is(key, "PixelSize")) {
/*
* The capture-graph builder REQUIRES this of the stream: -[BWFigVideoCaptureStream
* initWithCaptureStream:...] does a getProperty: (not IfSupported:) for it, and without it the
* init fails, so FigVideoCaptureSourcesActivateAndCreateDevices errors and the graph is never
- * built. The sensor's pixel pitch in micrometres (the iPhone X wide camera's); any value in
+ * built. The sensor's pixel pitch in micrometres (the iPhone X cameras'); any value in
* (0, 100) is accepted.
*/
- value = number_f(1.22);
+ value = number_f(s->camera->pixel_size);
} else if (key_is(key, "MaxZoomFactor")) {
- value = number_f(16);
+ value = number_f(MAX_ZOOM);
} else if (key_is(key, "ZoomFactor")) {
- value = number_f(1);
+ value = number_f(s->zoom);
} else if (key_is(key, "SupportedColorSpaces")) {
CFNumberRef srgb = number_i(0);
@@ -488,6 +579,57 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato
static void torch_set(struct stream *s, CFTypeRef value);
+static double
+dict_number(CFDictionaryRef d, const char *name, double otherwise)
+{
+ CFStringRef k = CFStringCreateWithCString(NULL, name, kCFStringEncodingUTF8);
+ CFTypeRef v = CFDictionaryGetValue(d, k);
+ double n = otherwise;
+
+ CFRelease(k);
+ if (v != NULL && CFGetTypeID(v) == CFNumberGetTypeID()) {
+ CFNumberGetValue(v, kCFNumberDoubleType, &n);
+ }
+ return n;
+}
+
+/* AFWindowParams: the first window, as fractions of the frame; none (or not one) is the middle */
+static void
+af_window(struct stream *s, CFTypeRef value)
+{
+ CFDictionaryRef w = NULL;
+ double x0, y0, width, height;
+
+ if (value != NULL && CFGetTypeID(value) == CFArrayGetTypeID() && CFArrayGetCount(value) > 0) {
+ w = CFArrayGetValueAtIndex(value, 0);
+ }
+ if (w == NULL || CFGetTypeID(w) != CFDictionaryGetTypeID() || (width = dict_number(w, "width", 0)) <= 0 ||
+ (height = dict_number(w, "height", 0)) <= 0) {
+ s->af_window[0] = s->af_window[1] = 0.3f;
+ s->af_window[2] = s->af_window[3] = 0.7f;
+ return;
+ }
+ x0 = dict_number(w, "hStart", 0) / 1000;
+ y0 = dict_number(w, "vStart", 0) / 1000;
+ width /= 1000;
+ height /= 1000;
+ /* (at least a tenth of the frame each way, about the middle of what was asked) */
+ if (width < 0.1) {
+ x0 -= (0.1 - width) / 2;
+ width = 0.1;
+ }
+ if (height < 0.1) {
+ y0 -= (0.1 - height) / 2;
+ height = 0.1;
+ }
+ x0 = x0 < 0 ? 0 : x0 + width > 1 ? 1 - width : x0;
+ y0 = y0 < 0 ? 0 : y0 + height > 1 ? 1 - height : y0;
+ s->af_window[0] = (float)x0;
+ s->af_window[1] = (float)y0;
+ s->af_window[2] = (float)(x0 + width);
+ s->af_window[3] = (float)(y0 + height);
+}
+
static OSStatus
stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)
{
@@ -506,6 +648,24 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)
say("stream: format %lld", (long long)i);
return 0;
}
+ if (key_is(key, "ZoomFactor")) {
+ /*
+ * The zoom: the camera's pictures -- moving and still alike -- made of the middle of the sensor
+ * (src/camera_isp.c, crop()), as the iPhone's ISP crops for it. Digital: no more detail than
+ * that part of the sensor has.
+ */
+ double zoom = 1;
+
+ if (value != NULL && CFGetTypeID(value) == CFNumberGetTypeID()) {
+ CFNumberGetValue(value, kCFNumberDoubleType, &zoom);
+ }
+ zoom = zoom < 1 ? 1 : zoom > MAX_ZOOM ? MAX_ZOOM : zoom;
+ if (fabs(zoom - s->zoom) >= 0.25 || (zoom == 1) != (s->zoom == 1)) {
+ say("stream: zoom %.2fx", zoom);
+ }
+ s->zoom = (float)zoom;
+ return 0;
+ }
if (key_is(key, "MinimumFrameRate") || key_is(key, "MaximumFrameRate")) {
double fps = 0;
@@ -541,6 +701,7 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)
if (out->config != NULL) {
CFRelease(out->config);
}
+ let_go(out);
out->config = CFGetTypeID(values[i]) == CFDictionaryGetTypeID() ? CFRetain(values[i]) : NULL;
} else if (enabled) {
int64_t on = 1;
@@ -592,6 +753,30 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)
torch_set(s, value);
return 0;
}
+ if (key_is(key, "AutoFocusNow") || key_is(key, "ContinuousAutoFocusNow") || key_is(key, "FocusNow") ||
+ key_is(key, "PeakTrackingFocusNow") || key_is(key, "LockAFNow") || key_is(key, "ManualFocusNow") ||
+ key_is(key, "AFWindowParams") || key_is(key, "AutoFocusMode")) {
+ /*
+ * Camera.app's tap: AutoFocusMode "AutoFocusModeWeightedSum", AFWindowParams an array of
+ * { hStart, vStart, width, height, weight } in thousandths of the frame (the first window is
+ * the one taken here; an empty value is the middle), then AutoFocusNow (true). LockAFNow
+ * keeps the lens where it is until focus is asked for again.
+ */
+ if (!key_is(key, "AFWindowParams") || (value != NULL && CFGetTypeID(value) == CFArrayGetTypeID() &&
+ CFArrayGetCount(value) > 0)) {
+ say_value("stream: focus:", key, value); /* (an empty window, the middle: often, unsaid) */
+ }
+ if (key_is(key, "AFWindowParams")) {
+ af_window(s, value);
+ } else if (key_is(key, "LockAFNow")) {
+ s->af_locked = true;
+ } else if (key_is(key, "AutoFocusNow") || key_is(key, "ContinuousAutoFocusNow") || key_is(key, "FocusNow") ||
+ key_is(key, "PeakTrackingFocusNow")) {
+ s->af_locked = false;
+ s->af_now = true;
+ }
+ return 0;
+ }
say_once("stream: set (not used)", key, value);
return 0;
}
@@ -671,7 +856,15 @@ output_buffer(struct output *out)
}
return NULL;
}
- if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess) {
+ if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && out->held != NULL) {
+ let_go(out); /* a pool that will not grow by one: the picture kept is the one to have */
+ (void)CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb);
+ if (!out->said_pool) {
+ out->said_pool = true;
+ say("%s: the graph's pool has no second buffer: the picture on the screen is drawn over", out->name);
+ }
+ }
+ if (pb == NULL) {
/*
* The preview pool holds one buffer (MinimumBufferCount 1): while the graph still has it,
* there is none to be had, and this frame is not given out. Said at most every 300th frame.
@@ -688,7 +881,68 @@ output_buffer(struct output *out)
typedef void (^pixel_buffer_handler)(CVPixelBufferRef, CMTime);
static void
-deliver(struct stream *s, struct output *out, const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw)
+let_go(struct output *out)
+{
+ if (out->held != NULL) {
+ CVPixelBufferRelease(out->held);
+ out->held = NULL;
+ }
+}
+
+static void
+md_put(CFMutableDictionaryRef m, int key, CFTypeRef value)
+{
+ if (md_key[key] != NULL && value != NULL) {
+ CFDictionarySetValue(m, md_key[key], value);
+ }
+ if (value != NULL) {
+ CFRelease(value);
+ }
+}
+
+/* the frame's metadata, as H10ISP's stream gives them (see md_names) */
+static CFDictionaryRef
+frame_metadata(const struct isp *isp, const struct redwood_camera_frame *f, const struct isp_stats *st, CFStringRef port)
+{
+ CFMutableDictionaryRef m;
+ double exposure = f->exposure_lines * (double)f->line_ns / 1e9, analog = f->gain_unity ? (double)f->gain / f->gain_unity : 1;
+ double total = analog * isp->digital, fps = f->frame_lines && f->line_ns ? 1e9 / ((double)f->frame_lines * f->line_ns) : 30;
+ /* (green's level for a second at 1x, a quarter of it: a room's 70 or so, daylight's tens of thousands) */
+ double lux = exposure > 0 && total > 0 ? st->mean[1] / (exposure * total) / 4 : 0;
+
+ if (md_attachment == NULL) {
+ return NULL;
+ }
+ m = (CFMutableDictionaryRef)new_dict();
+ md_put(m, MD_PORT, CFRetain(port));
+ md_put(m, MD_PTS, CMTimeCopyAsDictionary(CMClockMakeHostTimeFromSystemUnits(f->time), NULL));
+ md_put(m, MD_EXPOSURE, number_f(exposure));
+ md_put(m, MD_AGC, number_i((int64_t)(analog * 256 + 0.5)));
+ md_put(m, MD_ISP_DGAIN, number_i((int64_t)(isp->digital * 256 + 0.5)));
+ md_put(m, MD_SENSOR_DGAIN, number_i(256));
+ md_put(m, MD_ISO_BASE, number_f(32));
+ md_put(m, MD_FRAME_RATE, number_f(fps));
+ md_put(m, MD_LUX, number_i((int64_t)(lux + 0.5)));
+ md_put(m, MD_SNR, number_f(40 - 10 * log10(total < 1 ? 1 : total)));
+ md_put(m, MD_AF_STATUS, number_i(0));
+ md_put(m, MD_FOCUSING, number_i(0));
+ md_put(m, MD_AE_STABLE, CFRetain(kCFBooleanTrue));
+ md_put(m, MD_AE_LOCKED, CFRetain(kCFBooleanFalse));
+ md_put(m, MD_AWB_STABLE, CFRetain(kCFBooleanTrue));
+ md_put(m, MD_AWB_LOCKED, CFRetain(kCFBooleanFalse));
+ md_put(m, MD_AF_LOCKED, CFRetain(kCFBooleanFalse));
+ /*
+ * The zoom the picture was made with: the source node takes it for what the ISP applied
+ * (-[BWMultiStreamCameraSourceNode _ispAppliedZoomFactorFromSampleBufferMetadataDictionary:outputIndex:])
+ * and would make up the rest of what was asked for itself; without it, a zoomed preview stood still.
+ */
+ md_put(m, MD_ZOOM, number_f(isp->zoom));
+ return m;
+}
+
+static void
+deliver(struct stream *s, struct output *out, const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw,
+ CFDictionaryRef metadata)
{
CVPixelBufferRef pb = output_buffer(out);
uint32_t pf, width, height;
@@ -720,16 +974,304 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const struc
CVBufferSetAttachment(pb, kCVImageBufferTransferFunctionKey, kCVImageBufferTransferFunction_ITU_R_709_2,
kCVAttachmentMode_ShouldPropagate);
+ if (metadata != NULL) {
+ CFMutableDictionaryRef m = CFDictionaryCreateMutableCopy(NULL, 0, metadata);
+
+ CVBufferSetAttachment(pb, md_attachment, m, kCVAttachmentMode_ShouldPropagate);
+ CFRelease(m);
+ }
+
CMTime pts = CMClockMakeHostTimeFromSystemUnits(f->time);
((pixel_buffer_handler)out->handler)(pb, pts);
- CVPixelBufferRelease(pb);
+ /*
+ * The moving picture's buffer is kept until the next one is given. The preview's pool would
+ * otherwise hand the same buffer straight back -- the graph lets go of it as soon as it has passed
+ * it on -- while the window server is still copying it to the screen, and the next frame drawn
+ * into it showed as bands of two frames, torn across. Kept, it makes the pool take another.
+ */
+ if (strcmp(out->name, "StillImage") != 0) {
+ let_go(out);
+ out->held = pb;
+ } else {
+ CVPixelBufferRelease(pb);
+ }
if (out->delivered++ == 0) {
say("%s: the first picture given (%ux%u %c%c%c%c)", out->name, width, height, pf >> 24, pf >> 16 & 255,
pf >> 8 & 255, pf & 255);
}
}
+/*
+ * The scene's light as the cameras' tuning measures it (Qualcomm's lux index, higher is darker): about
+ * 46 a doubling of exposure times gain, 197 at a 33 ms exposure and 1x -- the black level's ranges in
+ * the tuning of the wide camera pair lux indexes with gains, 195-200 with 1-1.1x and 380-385 with
+ * 15.9-16x, at the longest exposure a frame at 30 a second allows.
+ */
+static float
+lux_index(const struct isp *isp, const struct redwood_camera_frame *f)
+{
+ double exposure = f->exposure_lines * (double)f->line_ns / 1e9 * f->gain / f->gain_unity * isp->digital;
+
+ return (float)(197 + 46 * log2(exposure > 1e-6 ? exposure / 0.0333 : 1e-6 / 0.0333));
+}
+
+/*
+ * The light's colour temperature: where the white balance is on the line of the tuning's reference
+ * lights; without them, guessed from how much more the blue is raised than the red, by what is typical of a phone's sensor --
+ * blue's gain over red's about 0.8 in daylight (6500 K), 1.3 under a fluorescent tube (4000 K), 2 under
+ * a filament lamp (2850 K) -- between those on the logarithm of the ratio, and no further.
+ */
+static float
+colour_temperature(const struct isp *isp)
+{
+ static const float ratio[3] = { 0.8f, 1.3f, 2.0f }, kelvin[3] = { 6500, 4000, 2850 };
+ float r = isp->wb[0] > 0 ? isp->wb[2] / isp->wb[0] : 1, cct = isp_cct(isp);
+
+ if (cct > 0) {
+ return cct; /* (by the tuning's reference lights) */
+ }
+
+ if (r <= ratio[0]) {
+ return kelvin[0];
+ }
+ for (int i = 1; i < 3; i++) {
+ if (r <= ratio[i]) {
+ float t = logf(r / ratio[i - 1]) / logf(ratio[i] / ratio[i - 1]);
+
+ return kelvin[i - 1] + t * (kelvin[i] - kelvin[i - 1]);
+ }
+ }
+ return kelvin[2];
+}
+
+/*
+ * Focus, where the camera has a lens motor (the frames' focus is a position, not REDWOOD_CAMERA_NO_FOCUS:
+ * the wide camera's): by contrast, as there is nothing else to go by -- the sensor's phase-detection
+ * pixels are not read. A search climbs from where the lens is: a step of AF_STEP one way, on that way
+ * while the middle of the picture gets sharper, back the other way if the first step was not, until
+ * two steps in a row are less sharp than the sharpest (or the travel, AF_NEAR to AF_FAR, ends); then
+ * around the sharpest in steps of AF_FINE, and there it stays. Each place is measured two frames after
+ * the lens was told to go there (it settles in less than one). The lens is only ever moved a step at a
+ * time: thrown across its travel it knocked, audibly, against its stop at 0. In focus, it stays until
+ * the middle has been a good deal less sharp or quite differently lit for a second, and then until
+ * the picture has kept still for a third of one: no searching while the phone is being moved.
+ * A focus asked for (a tap: AutoFocusNow, on the window AFWindowParams gave) sweeps instead: the
+ * lens is walked to the nearer end of its travel, then measured towards the other in steps of
+ * AF_STEP until it is past the sharpest (two steps in a row well under it), then around the sharpest
+ * -- it finds what was tapped however far out of focus it was, where the climb gave up on the flat
+ * of the curve.
+ */
+enum { AF_WATCH, AF_STEADY, AF_CLIMB, AF_SWEEP, AF_FINE_SEARCH, AF_SETTLE };
+#define AF_NEAR 16 /* (off the stop at 0) */
+#define AF_FAR 768 /* (its module file's actuator region goes to 700; near 1023 the PM8008i's LDO5 said over-current) */
+#define AF_STEP 48
+#define AF_FINE 12
+
+struct af {
+ int state;
+ uint32_t at, target; /* where the lens was told to go; where it is going a step at a time */
+ unsigned wait; /* frames before the next measurement */
+ int way; /* +1 nearer, -1 further */
+ bool turned;
+ unsigned worse; /* steps in a row less sharp than the sharpest */
+ uint32_t start, best, fine_to;
+ float best_sharpness;
+ float sharpness, level; /* in focus: the middle's, to watch */
+ float last_sharpness, last_level;
+ unsigned unlike, steady, steps;
+ bool now; /* asked for: no waiting for the picture to keep still */
+ bool sweep; /* ... and the whole travel measured */
+ float sum_sharpness, sum_level;
+ unsigned measured;
+};
+
+static void
+af_go(int device, struct af *af, uint32_t target)
+{
+ struct redwood_camera_command c = { .command = REDWOOD_CAMERA_FOCUS };
+
+ af->target = target < AF_NEAR ? AF_NEAR : target > AF_FAR ? AF_FAR : target;
+ c.a = af->at < af->target ? (af->target - af->at > AF_STEP ? af->at + AF_STEP : af->target) :
+ (af->at - af->target > AF_STEP ? af->at - AF_STEP : af->target);
+ af->at = c.a;
+ /*
+ * The lens settles in a few milliseconds; what is spoilt is the frame being exposed while it moved
+ * (written at a frame's end, the next one's exposure has begun): one frame let go by before a
+ * measurement, none on the way.
+ */
+ af->wait = af->at == af->target ? 1 : 0;
+ af->steps++;
+ af->sum_sharpness = af->sum_level = 0;
+ af->measured = 0;
+ (void)write(device, &c, sizeof(c));
+}
+
+static void
+af_frame(int device, struct af *af, const struct redwood_camera_frame *f, const uint8_t *raw, const float window[4], bool locked)
+{
+ float level, sharpness;
+ unsigned needed = f->gain >= 16 * f->gain_unity ? 2 : 1; /* (in the dark, two frames to a measurement) */
+
+ if (f->focus == REDWOOD_CAMERA_NO_FOCUS) {
+ return;
+ }
+ if (af->at == 0 && af->target == 0) {
+ af->at = af->target = f->focus; /* (where the kernel put it) */
+ }
+ if (af->wait > 0) {
+ af->wait--;
+ return;
+ }
+ if (af->at != af->target) {
+ af_go(device, af, af->target); /* (the next step of a longer way) */
+ return;
+ }
+ sharpness = isp_sharpness(f, raw, window[0], window[1], window[2], window[3], &level);
+ af->sum_sharpness += sharpness;
+ af->sum_level += level;
+ if (++af->measured < needed) {
+ return;
+ }
+ sharpness = af->sum_sharpness / af->measured;
+ level = af->sum_level / af->measured;
+ af->sum_sharpness = af->sum_level = 0;
+ af->measured = 0;
+ switch (af->state) {
+ case AF_WATCH:
+ if (locked || (af->sharpness > 0 && sharpness > af->sharpness * 0.6f && level > af->level * 0.7f &&
+ level < af->level * 1.4f)) {
+ af->unlike = 0;
+ return;
+ }
+ if (af->sharpness > 0 && ++af->unlike < 30) {
+ return;
+ }
+ af->state = AF_STEADY;
+ af->steady = 0;
+ af->last_sharpness = sharpness;
+ af->last_level = level;
+ return;
+ case AF_STEADY:
+ if (fabsf(sharpness - af->last_sharpness) < 0.15f * af->last_sharpness && fabsf(level - af->last_level) < 0.1f * af->last_level) {
+ af->steady++;
+ } else {
+ af->steady = 0;
+ }
+ af->last_sharpness = sharpness;
+ af->last_level = level;
+ if (af->steady < 10 && !af->now) {
+ return;
+ }
+ af->now = false;
+ if (af->sweep) {
+ af->sweep = false;
+ af->state = AF_SWEEP;
+ af->best_sharpness = -1;
+ af->worse = 0;
+ af->steps = 0;
+ af->way = af->at - AF_NEAR < AF_FAR - af->at ? 1 : -1;
+ af_go(device, af, af->way > 0 ? AF_NEAR : AF_FAR); /* (walked there a step at a time) */
+ return;
+ }
+ af->state = AF_CLIMB;
+ af->best = af->start = af->at;
+ af->best_sharpness = sharpness;
+ af->way = af->at + AF_STEP > AF_FAR ? -1 : 1;
+ af->turned = false;
+ af->worse = 0;
+ af->steps = 0;
+ af_go(device, af, (uint32_t)((int)af->at + af->way * AF_STEP));
+ return;
+ case AF_CLIMB: {
+ int next = (int)af->at + af->way * AF_STEP;
+ bool end = next < AF_NEAR || next > AF_FAR;
+
+ if (sharpness > af->best_sharpness * 1.02f) {
+ af->best = af->at;
+ af->best_sharpness = sharpness;
+ af->worse = 0;
+ } else {
+ af->worse++;
+ }
+ if (af->worse < 2 && !end) {
+ af_go(device, af, (uint32_t)next);
+ return;
+ }
+ next = (int)af->best - af->way * AF_STEP;
+ if (!af->turned && af->best == af->start && next >= AF_NEAR && next <= AF_FAR) {
+ /* not sharper that way at all: the other way, from where it started */
+ af->turned = true;
+ af->way = -af->way;
+ af->worse = 0;
+ af_go(device, af, (uint32_t)next);
+ return;
+ }
+ af->state = AF_FINE_SEARCH;
+ af->fine_to = af->best + 2 * AF_FINE > AF_FAR ? AF_FAR : af->best + 2 * AF_FINE;
+ af_go(device, af, af->best > 2 * AF_FINE + AF_NEAR ? af->best - 2 * AF_FINE : AF_NEAR);
+ return;
+ }
+ case AF_SWEEP:
+ say("focus: sweep: %u %.4f", af->at, sharpness);
+ if (sharpness > af->best_sharpness) {
+ af->best = af->at;
+ af->best_sharpness = sharpness;
+ af->worse = 0;
+ } else if (sharpness < af->best_sharpness * 0.85f) {
+ af->worse++;
+ }
+ if (af->worse < 2 && (int)af->at + af->way * AF_STEP >= AF_NEAR && (int)af->at + af->way * AF_STEP <= AF_FAR) {
+ af_go(device, af, (uint32_t)((int)af->at + af->way * AF_STEP));
+ return;
+ }
+ af->state = AF_FINE_SEARCH;
+ af->fine_to = af->best + 2 * AF_FINE > AF_FAR ? AF_FAR : af->best + 2 * AF_FINE;
+ af_go(device, af, af->best > 2 * AF_FINE + AF_NEAR ? af->best - 2 * AF_FINE : AF_NEAR);
+ return;
+ case AF_FINE_SEARCH:
+ if (sharpness > af->best_sharpness) {
+ af->best = af->at;
+ af->best_sharpness = sharpness;
+ }
+ if (af->at + AF_FINE <= af->fine_to) {
+ af_go(device, af, af->at + AF_FINE);
+ return;
+ }
+ af->state = AF_SETTLE;
+ af_go(device, af, af->best);
+ return;
+ case AF_SETTLE:
+ af->state = AF_WATCH;
+ af->sharpness = sharpness;
+ af->level = level;
+ af->unlike = 0;
+ say("focus: the lens at %u after %u moves (sharpness %.3f)", af->at, af->steps, sharpness);
+ return;
+ }
+}
+
+/* the sensor's colour tuning, when the image has one (fw/camera-<sensor>.isp, tools/camtuning.py) */
+static void
+load_tuning(struct isp *isp, uint32_t sensor)
+{
+ static const char *const names[] = { "ov02b10", "s5khm2sd", "ov16a1q", "s5k4h7" };
+ char path[96];
+
+ if (sensor >= sizeof(names) / sizeof(names[0])) {
+ return;
+ }
+ snprintf(path, sizeof(path), "/redwood/fw/camera-%s.isp", names[sensor]);
+ if (isp_load_tuning(isp, path)) {
+ say("the %s's colour tuning from %s: %u colour matrices' lights, %u lens shadings'", names[sensor], path,
+ isp->tuning->ccm.lights, isp->tuning->lsc.lights);
+ } else {
+ say("no colour tuning for the %s (%s): no colour matrix, no lens shading", names[sensor], path);
+ isp_set_ccm(isp, (const float[9]){ 1, 0, 0, 0, 1, 0, 0, 0, 1 });
+ isp->lsc.on = false;
+ }
+}
+
static void *
worker(void *context)
{
@@ -740,6 +1282,10 @@ worker(void *context)
struct isp_stats st;
uint8_t *buffer = NULL;
size_t bytes = 0;
+ uint32_t tuned_for = ~0u;
+ bool switched = false;
+ float lux = -1, cct = -1;
+ struct af af = { 0 };
int device = open(REDWOOD_CAMERA_DEVICE, O_RDWR), failures = 0;
pthread_setname_np("redwood-camera");
@@ -761,9 +1307,20 @@ worker(void *context)
}
continue;
}
+ if (head.sensor != s->camera->sensor) {
+ /* another camera's frames (the kernel's last sensor): changed, and the next ones are this one's */
+ if (!switched) {
+ struct redwood_camera_command c = { .command = REDWOOD_CAMERA_SENSOR, .a = s->camera->sensor, .b = ~0u };
+
+ switched = true;
+ say("the kernel's frames are camera %u's: to %u, the %s", head.sensor, s->camera->sensor, s->camera->name);
+ (void)write(device, &c, sizeof(c));
+ }
+ continue;
+ }
bytes = sizeof(head) + (size_t)head.stride * head.height;
buffer = malloc(bytes);
- say("frames of %ux%u from the kernel", head.width, head.height);
+ say("frames of %ux%u from the kernel (the %s)", head.width, head.height, s->camera->name);
continue;
}
n = read(device, buffer, bytes);
@@ -781,12 +1338,36 @@ worker(void *context)
const uint8_t *raw = buffer + f->header_bytes;
struct redwood_camera_command c;
- isp_measure(f, raw, &st);
+ isp_measure(&isp, f, raw, &st);
isp_awb(&isp, &st);
+ if (f->sensor != tuned_for) {
+ tuned_for = f->sensor;
+ load_tuning(&isp, f->sensor);
+ }
+ /*
+ * Slowly, a second or so: the colour temperature is guessed from the white balance, the tuning it
+ * picks changes what the white balance sees, and the colour matrix makes every swing of it plain.
+ */
+ lux = lux < 0 ? lux_index(&isp, f) : lux * 0.9f + lux_index(&isp, f) * 0.1f;
+ cct = cct < 0 ? colour_temperature(&isp) : cct * 0.97f + colour_temperature(&isp) * 0.03f;
+ isp_tune(&isp, lux, cct);
+ isp.zoom = s->zoom;
+ isp.turned = s->camera->turned;
if (isp_ae(&isp, f, &st, &c)) {
(void)write(device, &c, sizeof(c));
}
+ if (s->af_now) {
+ s->af_now = false;
+ /* (whatever it was doing: at once, from the next frame) */
+ af.state = AF_STEADY;
+ af.now = true;
+ af.sweep = true;
+ }
+ af_frame(device, &af, f, raw, s->af_window, s->af_locked);
s->frames++;
+
+ CFDictionaryRef metadata = frame_metadata(&isp, f, &st, s->port);
+
/*
* Only once the graph has started the stream. Its PixelBufferHandler blocks hand the buffer
* to multiStreamCameraSourceNode_sendBufferToBufferServicingThread, which reaches into the
@@ -794,6 +1375,14 @@ worker(void *context)
* it walks a pointer that is not there yet and mediaserverd dies of SIGSEGV. The frames
* before that are read and thrown away, which is also what settles the exposure.
*/
+ if (metadata != NULL) {
+ pthread_mutex_lock(&s->metadata_lock);
+ if (s->metadata != NULL) {
+ CFRelease(s->metadata);
+ }
+ s->metadata = CFRetain(metadata);
+ pthread_mutex_unlock(&s->metadata_lock);
+ }
pthread_mutex_lock(&s->lock);
for (int i = 0; s->graph_running && i < OUTPUTS; i++) {
struct output *out = &s->out[i];
@@ -811,16 +1400,20 @@ worker(void *context)
if (still && s->stills_wanted == 0) {
continue;
}
- deliver(s, out, &isp, f, raw);
+ deliver(s, out, &isp, f, raw, metadata);
if (still) {
s->stills_wanted--;
say("the photograph is given (%llu so far)", (unsigned long long)++s->stills_given);
}
}
pthread_mutex_unlock(&s->lock);
+ if (metadata != NULL) {
+ CFRelease(metadata);
+ }
if (s->frames == 1 || s->frames % 300 == 0) {
- say("frame %llu: exposure %.1f ms, gain %.1fx, digital %.2fx, green %.0f above black", f->sequence,
- f->exposure_lines * (double)f->line_ns / 1e6, (double)f->gain / f->gain_unity, isp.digital, st.mean[1]);
+ say("frame %llu: exposure %.1f ms, gain %.1fx, digital %.2fx, green %.0f above black; white balance R %.2f "
+ "B %.2f, lux index %.0f, about %.0f K", f->sequence, f->exposure_lines * (double)f->line_ns / 1e6,
+ (double)f->gain / f->gain_unity, isp.digital, st.mean[1], isp.wb[0], isp.wb[2], lux, cct);
}
}
free(buffer);
@@ -835,6 +1428,9 @@ worker(void *context)
* takes a moment to settle and the exposure longer), the second only when the graph starts the
* stream itself (`graph` true, the FigCaptureStream start method).
*/
+static CMBaseObjectRef all_streams[2]; /* the device's, back and front */
+static OSStatus stream_stop(CMBaseObjectRef o);
+
static OSStatus
stream_begin(CMBaseObjectRef o, bool graph)
{
@@ -849,6 +1445,13 @@ stream_begin(CMBaseObjectRef o, bool graph)
if (s->running) {
return 0;
}
+ /* one camera at a time: the kernel gives one sensor's frames (the other stream's worker stopped first) */
+ for (int i = 0; i < 2; i++) {
+ if (all_streams[i] != NULL && all_streams[i] != o && stream_of(all_streams[i])->running) {
+ say("stream: the %s's frames stop for the %s's", stream_of(all_streams[i])->camera->name, s->camera->name);
+ (void)stream_stop(all_streams[i]);
+ }
+ }
s->running = true;
s->frames = 0;
say("stream: frames begin (format %d)", s->format_index);
@@ -875,6 +1478,9 @@ stream_stop(CMBaseObjectRef o)
(unsigned long long)s->out[0].delivered);
pthread_mutex_lock(&s->lock);
s->graph_running = false;
+ for (int i = 0; i < OUTPUTS; i++) {
+ let_go(&s->out[i]);
+ }
pthread_mutex_unlock(&s->lock);
s->running = false;
if (s->worker_started) {
@@ -884,17 +1490,60 @@ stream_stop(CMBaseObjectRef o)
return 0;
}
+static void
+add_absent(const void *key, const void *value, void *context)
+{
+ if (!CFDictionaryContainsKey(context, key)) {
+ CFDictionarySetValue(context, key, value);
+ }
+}
+
+/*
+ * The frame's metadata into the sample buffer the graph made of our pixel buffer: the pixel buffer's
+ * own dictionary (deliver()), the same object, else the last frame's; whatever the graph has put in a
+ * dictionary of the sample buffer's already stays as it is.
+ */
static OSStatus
stream_add_attachments(CMBaseObjectRef o, CMSampleBufferRef sbuf, CFDictionaryRef options)
{
+ struct stream *s = stream_of(o);
static bool said;
+ CVImageBufferRef image = sbuf != NULL ? CMSampleBufferGetImageBuffer(sbuf) : NULL;
+ CFTypeRef ours = image != NULL && md_attachment != NULL ? CVBufferGetAttachment(image, md_attachment, NULL) : NULL, theirs;
+ CFMutableDictionaryRef m;
- (void)o;
- (void)sbuf;
if (!said) {
said = true;
- say("stream: AddAttachmentsToSampleBuffer (options %s)", options ? "given" : "none");
+ say("stream: AddAttachmentsToSampleBuffer (options %s): the frames' metadata %s", options ? "given" : "none",
+ md_attachment != NULL ? "attached" : "NOT THERE");
}
+ if (md_attachment == NULL || sbuf == NULL) {
+ return 0;
+ }
+ if (ours != NULL && CFGetTypeID(ours) == CFDictionaryGetTypeID()) {
+ CFRetain(ours);
+ } else {
+ pthread_mutex_lock(&s->metadata_lock);
+ ours = s->metadata != NULL ? CFDictionaryCreateMutableCopy(NULL, 0, s->metadata) : NULL;
+ pthread_mutex_unlock(&s->metadata_lock);
+ }
+ if (ours == NULL) {
+ return 0;
+ }
+ theirs = CMGetAttachment(sbuf, md_attachment, NULL);
+ if (theirs == ours) {
+ CFRelease(ours);
+ return 0;
+ }
+ if (theirs != NULL && CFGetTypeID(theirs) == CFDictionaryGetTypeID()) {
+ m = CFDictionaryCreateMutableCopy(NULL, 0, theirs);
+ CFDictionaryApplyFunction(ours, add_absent, m);
+ } else {
+ m = (CFMutableDictionaryRef)CFRetain(ours);
+ }
+ CMSetAttachment(sbuf, md_attachment, m, kCMAttachmentMode_ShouldPropagate);
+ CFRelease(m);
+ CFRelease(ours);
return 0;
}
@@ -928,6 +1577,9 @@ stream_finalize(CMBaseObjectRef o)
if (s->colour_info) {
CFRelease(s->colour_info);
}
+ if (s->metadata) {
+ CFRelease(s->metadata);
+ }
if (s->torch >= 0) {
close(s->torch);
}
@@ -948,7 +1600,7 @@ static const struct stream_class stream_derived = { 2, stream_start, stream_stop
static const struct vtable stream_vtable = { NULL, &stream_base, &stream_derived };
static CMBaseObjectRef
-stream_create(CFAllocatorRef allocator)
+stream_create(CFAllocatorRef allocator, const struct camera *camera, CFStringRef port)
{
CMBaseObjectRef o = NULL;
struct stream *s;
@@ -960,15 +1612,21 @@ stream_create(CFAllocatorRef allocator)
s = stream_of(o);
memset(s, 0, sizeof(*s));
pthread_mutex_init(&s->lock, NULL);
+ pthread_mutex_init(&s->metadata_lock, NULL);
s->torch = -1;
s->min_fps = 2;
+ s->zoom = 1;
+ s->af_window[0] = s->af_window[1] = 0.3f;
+ s->af_window[2] = s->af_window[3] = 0.7f;
s->max_fps = 30;
+ s->camera = camera;
+ s->port = port;
for (int i = 0; i < OUTPUTS; i++) {
s->out[i].name = output_names[i];
}
- /* "Photo" and "Video-60" of D22/AVCaptureSession.plist's back camera */
- CFDictionaryRef f[2] = { format(4032, 3024, 30, false), format(4224, 2376, 60, false) };
+ CFDictionaryRef f[2] = { format(camera->formats[0][0], camera->formats[0][1], camera->formats[0][2], false, camera->fov),
+ format(camera->formats[1][0], camera->formats[1][1], camera->formats[1][2], false, camera->fov) };
s->formats = CFArrayCreate(NULL, (const void **)f, 2, &kCFTypeArrayCallBacks);
s->supported = supported(stream_props, sizeof(stream_props) / sizeof(stream_props[0]));
@@ -1102,7 +1760,7 @@ OSStatus
redwood_camera_device_create(CFAllocatorRef allocator, CFDictionaryRef options, CMBaseObjectRef *out)
{
static bool found, usable;
- CMBaseObjectRef device = NULL, stream;
+ CMBaseObjectRef device = NULL, stream[2];
struct device *d;
(void)options;
@@ -1113,20 +1771,28 @@ redwood_camera_device_create(CFAllocatorRef allocator, CFDictionaryRef options,
if (!usable || out == NULL) {
return ERR_UNSUPPORTED;
}
- if ((stream = stream_create(allocator)) == NULL) {
+ if ((stream[0] = stream_create(allocator, &cameras[0], port_back)) == NULL) {
+ return ERR_UNSUPPORTED;
+ }
+ if ((stream[1] = stream_create(allocator, &cameras[1], port_front)) == NULL) {
+ CFRelease(stream[0]);
return ERR_UNSUPPORTED;
}
if (derived_create(allocator, &device_vtable, device_class_id(), &device) != 0 || device == NULL) {
- CFRelease(stream);
+ CFRelease(stream[0]);
+ CFRelease(stream[1]);
say("the device cannot be made");
return ERR_UNSUPPORTED;
}
d = derived_storage(device);
memset(d, 0, sizeof(*d));
- d->streams = CFArrayCreate(NULL, (const void **)&stream, 1, &kCFTypeArrayCallBacks);
+ d->streams = CFArrayCreate(NULL, (const void **)stream, 2, &kCFTypeArrayCallBacks);
+ all_streams[0] = stream[0]; /* (kept by the device's array) */
+ all_streams[1] = stream[1];
d->supported = supported(device_props, sizeof(device_props) / sizeof(device_props[0]));
- CFRelease(stream);
+ CFRelease(stream[0]);
+ CFRelease(stream[1]);
*out = device;
- say("the device made: one stream, the back camera");
+ say("the device made: two streams, the back camera and the front one");
return 0;
}
diff --git a/userland/src/camera_isp.c b/userland/src/camera_isp.c
index 6bfd14f..626472f 100644
--- a/userland/src/camera_isp.c
+++ b/userland/src/camera_isp.c
@@ -3,11 +3,15 @@
* cell, the nearest cell for each pixel of the picture (previews), or every pixel by bilinear
* demosaic (stills); per pixel, in this order: the black level off, lens shading (a grid of gains per
* colour, when there is one), the white balance's and AE's gains, the colour matrix (identity until
- * one is set), sRGB's curve. Grey-world white balance; no noise reduction yet.
+ * one is set), sRGB's curve. Grey-world white balance; no noise reduction yet. The colour matrix and
+ * the lens shading can come from the camera's own tuning (isp_load_tuning(), isp_tune()).
*/
+#include <fcntl.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
+#include <sys/stat.h>
#include "camera_isp.h"
#define WHITE 959 /* 1023 less the black level of 64 */
@@ -19,6 +23,7 @@ isp_init(struct isp *isp)
memset(isp, 0, sizeof(*isp));
isp->wb[0] = isp->wb[1] = isp->wb[2] = 1;
isp->digital = 1;
+ isp->zoom = 1;
for (int i = 0; i < 3; i++) {
isp->ccm[i][i] = 1;
}
@@ -56,29 +61,134 @@ cell(const uint8_t *top, size_t stride, uint32_t cx, int v[4])
}
}
+float
+isp_sharpness(const struct redwood_camera_frame *f, const uint8_t *raw, float x0, float y0, float x1, float y1, float *level)
+{
+ enum { BLOCK = 4, MAX_BLOCKS = 512 };
+ uint32_t cells_w = f->width / 2, cells_h = f->height / 2;
+ uint32_t cx0 = (uint32_t)(x0 * cells_w), cx1 = (uint32_t)(x1 * cells_w), cy0 = (uint32_t)(y0 * cells_h),
+ cy1 = (uint32_t)(y1 * cells_h);
+ float above[MAX_BLOCKS], here[MAX_BLOCKS];
+ double steps = 0, sum = 0;
+ uint32_t blocks_w, rows = 0, n = 0;
+ int r, b;
+
+ cell_places(f->bayer, &r, &b);
+ cx1 = cx1 > cells_w ? cells_w : cx1;
+ cy1 = cy1 > cells_h ? cells_h : cy1;
+ blocks_w = cx1 > cx0 ? (cx1 - cx0) / BLOCK : 0;
+ blocks_w = blocks_w > MAX_BLOCKS ? MAX_BLOCKS : blocks_w;
+ /*
+ * Green's mean over blocks of 4x4 cells, and the squares of the steps between neighbouring
+ * blocks: a cell's own noise is a quarter of itself in a block's mean, and the picture's edges are
+ * not -- in the dark, at 32x and more, the steps from cell to cell were the noise's alone and the
+ * focus went wherever it liked.
+ */
+ for (uint32_t by = cy0; by + BLOCK <= cy1; by += BLOCK, rows++) {
+ for (uint32_t bx = 0; bx < blocks_w; bx++) {
+ uint32_t g = 0;
+
+ for (uint32_t cy = by; cy < by + BLOCK; cy++) {
+ const uint8_t *top = raw + (size_t)cy * 2 * f->stride;
+
+ for (uint32_t cx = cx0 + bx * BLOCK; cx < cx0 + bx * BLOCK + BLOCK; cx++) {
+ int v[4];
+
+ cell(top, f->stride, cx, v);
+ for (int i = 0; i < 4; i++) {
+ g += i != r && i != b ? (uint32_t)v[i] : 0;
+ }
+ }
+ }
+ here[bx] = g / (2.0f * BLOCK * BLOCK);
+ sum += here[bx];
+ n++;
+ if (bx > 0) {
+ steps += (here[bx] - here[bx - 1]) * (here[bx] - here[bx - 1]);
+ }
+ if (rows > 0) {
+ steps += (here[bx] - above[bx]) * (here[bx] - above[bx]);
+ }
+ }
+ memcpy(above, here, sizeof(float) * blocks_w);
+ }
+ double mean = n ? sum / n : 0;
+
+ if (level != NULL) {
+ *level = (float)mean;
+ }
+ return mean > 0 ? (float)(steps / n / (mean * mean)) : 0;
+}
+
+static inline void lsc_at(const struct isp *isp, float u, float v, float g[3]);
+
void
-isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s)
+isp_measure(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s)
{
uint32_t cells_w = f->width / 2, cells_h = f->height / 2;
- double sum[3] = { 0, 0, 0 };
+ double sum[3] = { 0, 0, 0 }, grey[2] = { 0, 0 }, neutral[2] = { 0, 0 };
+ float gains[3] = { 1, 1, 1 };
int r, b;
memset(s, 0, sizeof(*s));
cell_places(f->bayer, &r, &b);
for (uint32_t cy = 0; cy < cells_h; cy += 4) {
const uint8_t *top = raw + (size_t)cy * 2 * f->stride;
+ float v_pos = cells_h > 1 ? (float)cy / (cells_h - 1) : 0;
for (uint32_t cx = 0; cx < cells_w; cx += 4) {
- int v[4], g = 0;
+ int v[4], g = 0, top_v = 0;
cell(top, f->stride, cx, v);
for (int i = 0; i < 4; i++) {
g += i != r && i != b ? v[i] : 0;
+ top_v = v[i] > top_v ? v[i] : top_v;
}
g /= 2;
sum[0] += v[r];
sum[1] += g;
sum[2] += v[b];
+ /*
+ * The white balance leaves out a cell with a colour near white -- a lamp, a screen:
+ * green, the most sensitive, stops there first and the cell would count as magenta --
+ * and one near black, where noise cut off at the black level is all there is.
+ */
+ if (top_v < WHITE * 9 / 10 && g >= 24) {
+ float cr = (float)v[r], cg = (float)g, cb = (float)v[b];
+
+ /*
+ * As the picture will have it: the lens shading raises the corners several times, and
+ * the light there counts for that much more than in the frame as it comes.
+ */
+ if (isp != NULL && isp->lsc.on) {
+ lsc_at(isp, cells_w > 1 ? (float)cx / (cells_w - 1) : 0, v_pos, gains);
+ cr *= gains[0];
+ cg *= gains[1];
+ cb *= gains[2];
+ }
+ /*
+ * Each cell's colour, not its light: summed, a lamp or a screen in the picture -- tens of
+ * times brighter than the room it is in -- would have the white balance to itself, and
+ * the room would be left the colour of its light.
+ */
+ grey[0] += cr / cg;
+ grey[1] += cb / cg;
+ s->grey_cells++;
+ /*
+ * Grey, or within a third of it in red and blue, with the white balance as it is: what
+ * the white balance should make grey, without the coloured things around it (the
+ * colour matrix makes any mistake in it twice as plain).
+ */
+ if (isp != NULL) {
+ float wr = cr * isp->wb[0] / cg, wb = cb * isp->wb[2] / cg;
+
+ if (wr > 0.75f && wr < 1.33f && wb > 0.75f && wb < 1.33f) {
+ neutral[0] += cr / cg;
+ neutral[1] += cb / cg;
+ s->neutral_cells++;
+ }
+ }
+ }
s->hist[g >> 4 > 63 ? 63 : g >> 4]++;
s->cells++;
}
@@ -86,6 +196,10 @@ isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct isp
for (int c = 0; c < 3; c++) {
s->mean[c] = s->cells ? sum[c] / s->cells : 0;
}
+ for (int c = 0; c < 2; c++) {
+ s->grey[c] = s->grey_cells ? grey[c] / s->grey_cells : 0;
+ s->neutral[c] = s->neutral_cells ? neutral[c] / s->neutral_cells : 0;
+ }
for (int i = 63, above = 0; i >= 0; i--) {
above += s->hist[i];
if (above * 100 >= (int)s->cells) {
@@ -155,25 +269,97 @@ isp_ae(struct isp *isp, const struct redwood_camera_frame *f, const struct isp_s
return true;
}
+/*
+ * The point of the reference lights' line nearest a light seen as red over green rg and blue over
+ * green bg, on their logarithms: where along it (segment i and how far into it, 0 to 1).
+ */
+static void
+on_reference_line(const struct isp_tuning *t, float rg, float bg, uint32_t *segment, float *along)
+{
+ float x = logf(rg), y = logf(bg), best = INFINITY;
+
+ *segment = 0;
+ *along = 0;
+ for (uint32_t i = 0; i + 1 < t->refs; i++) {
+ float x0 = logf(t->ref[i][0]), y0 = logf(t->ref[i][1]), dx = logf(t->ref[i + 1][0]) - x0,
+ dy = logf(t->ref[i + 1][1]) - y0, length = dx * dx + dy * dy;
+ float u = length > 0 ? ((x - x0) * dx + (y - y0) * dy) / length : 0, d;
+
+ u = u < 0 ? 0 : u > 1 ? 1 : u;
+ d = (x0 + u * dx - x) * (x0 + u * dx - x) + (y0 + u * dy - y) * (y0 + u * dy - y);
+ if (d < best) {
+ best = d;
+ *segment = i;
+ *along = u;
+ }
+ }
+}
+
+float
+isp_cct(const struct isp *isp)
+{
+ const struct isp_tuning *t = isp->tuning;
+ uint32_t i;
+ float u;
+
+ if (t == NULL || t->refs < 2 || isp->wb[0] <= 0 || isp->wb[2] <= 0) {
+ return 0;
+ }
+ on_reference_line(t, 1 / isp->wb[0], 1 / isp->wb[2], &i, &u);
+ return t->ref[i][2] + u * (t->ref[i + 1][2] - t->ref[i][2]);
+}
+
void
isp_awb(struct isp *isp, const struct isp_stats *s)
{
- if (s->mean[0] < 2 || s->mean[2] < 2 || s->mean[1] < 2) {
+ /*
+ * The nearly grey cells, when there are enough of them (1 in 32); else every cell neither near
+ * white nor near black, when there are enough of those (1 in 16); else every cell's light.
+ */
+ float r, b;
+
+ if (s->neutral_cells * 32 >= s->cells && s->neutral[0] > 0.01 && s->neutral[1] > 0.01) {
+ r = (float)(1 / s->neutral[0]);
+ b = (float)(1 / s->neutral[1]);
+ } else if (s->grey_cells * 16 >= s->cells && s->grey[0] > 0.01 && s->grey[1] > 0.01) {
+ r = (float)(1 / s->grey[0]);
+ b = (float)(1 / s->grey[1]);
+ } else if (s->mean[0] >= 2 && s->mean[1] >= 2 && s->mean[2] >= 2) {
+ r = (float)(s->mean[1] / s->mean[0]);
+ b = (float)(s->mean[1] / s->mean[2]);
+ } else {
return;
}
- float r = (float)(s->mean[1] / s->mean[0]), b = (float)(s->mean[1] / s->mean[2]);
+ /*
+ * Onto the line of the reference lights: a scene's own colours pull grey world off it -- a pink
+ * screen, a green wall -- and off it is what shows as green or magenta, while along it is a
+ * warmer or colder white. (The wide camera under a warm white LED: grey world of the room with a
+ * screen in it 15 % short in both red and blue, the light measured on a white sheet; kept to the
+ * line, 2 % off.)
+ */
+ if (isp->tuning != NULL && isp->tuning->refs >= 2) {
+ const struct isp_tuning *t = isp->tuning;
+ uint32_t i;
+ float u;
+
+ on_reference_line(t, 1 / r, 1 / b, &i, &u);
+ r = 1 / expf(logf(t->ref[i][0]) + u * (logf(t->ref[i + 1][0]) - logf(t->ref[i][0])));
+ b = 1 / expf(logf(t->ref[i][1]) + u * (logf(t->ref[i + 1][1]) - logf(t->ref[i][1])));
+ }
r = r < 0.5f ? 0.5f : r > 4 ? 4 : r;
b = b < 0.5f ? 0.5f : b > 4 ? 4 : b;
- isp->wb[0] = isp->wb[0] * 0.8f + r * 0.2f;
- isp->wb[2] = isp->wb[2] * 0.8f + b * 0.2f;
+ isp->wb[0] = isp->wb[0] * 0.9f + r * 0.1f;
+ isp->wb[2] = isp->wb[2] * 0.9f + b * 0.1f;
}
-/* the part of the sensor's cells the picture shows: its middle, in the picture's proportions */
+/* the part of the sensor's cells the picture shows: its middle, in the picture's proportions, 1/zoom of it each way */
static void
-crop(const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint32_t *x0, uint32_t *y0, uint32_t *cw, uint32_t *ch)
+crop(const struct isp *isp, const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint32_t *x0, uint32_t *y0,
+ uint32_t *cw, uint32_t *ch)
{
uint32_t w = f->width / 2, h = f->height / 2;
+ float zoom = isp->zoom < 1 ? 1 : isp->zoom;
if ((uint64_t)width * h > (uint64_t)height * w) {
*cw = w;
@@ -182,6 +368,8 @@ crop(const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint
*ch = h;
*cw = (uint32_t)((uint64_t)h * width / height);
}
+ *cw = (uint32_t)(*cw / zoom) < 2 ? 2 : (uint32_t)(*cw / zoom);
+ *ch = (uint32_t)(*ch / zoom) < 2 ? 2 : (uint32_t)(*ch / zoom);
*x0 = (w - *cw) / 2;
*y0 = (h - *ch) / 2;
}
@@ -264,9 +452,53 @@ render_line(const struct isp *isp, const struct redwood_camera_frame *f, const u
int r, b;
struct pipe p;
- crop(f, width, height, &x0, &y0, &cw, &ch);
+ crop(isp, f, width, height, &x0, &y0, &cw, &ch);
cell_places(f->bayer, &r, &b);
pipe_init(isp, &p);
+ if (isp->turned) {
+ row = height - 1 - row; /* (and each line from its end, below) */
+ }
+ if (cw < width && ch < height) {
+ /*
+ * Magnified (the zoom): each pixel between the four cells around it. The nearest cell blown up
+ * to a square of pixels showed every cell's own colour, its noise and the colours a 2x2 cell
+ * gets wrong at an edge, as coloured dots.
+ */
+ float fy = y0 + (row + 0.5f) * ch / height - 0.5f, ty, v_at;
+ uint32_t cy0 = fy < 0 ? 0 : (uint32_t)fy, cy1;
+
+ cy0 = cy0 > cells_h - 1 ? cells_h - 1 : cy0;
+ cy1 = cy0 + 1 > cells_h - 1 ? cells_h - 1 : cy0 + 1;
+ ty = fy < (float)cy0 ? 0 : fy - cy0;
+ v_at = cells_h > 1 ? (float)cy0 / (cells_h - 1) : 0;
+ const uint8_t *t0 = raw + (size_t)cy0 * 2 * f->stride, *t1 = raw + (size_t)cy1 * 2 * f->stride;
+ float gains[3] = { 1, 1, 1 };
+
+ for (uint32_t x = 0; x < width; x++) {
+ float fx = x0 + (x + 0.5f) * cw / width - 0.5f, tx, c[3] = { 0, 0, 0 };
+ uint32_t cx0 = fx < 0 ? 0 : (uint32_t)fx, cx1;
+
+ cx0 = cx0 > cells_w - 1 ? cells_w - 1 : cx0;
+ cx1 = cx0 + 1 > cells_w - 1 ? cells_w - 1 : cx0 + 1;
+ tx = fx < (float)cx0 ? 0 : fx - cx0;
+ for (int k = 0; k < 4; k++) {
+ int v[4];
+ float w = (k & 1 ? tx : 1 - tx) * (k & 2 ? ty : 1 - ty);
+
+ cell(k & 2 ? t1 : t0, f->stride, k & 1 ? cx1 : cx0, v);
+ c[0] += w * v[r];
+ c[2] += w * v[b];
+ for (int j = 0; j < 4; j++) {
+ c[1] += j != r && j != b ? w * v[j] * 0.5f : 0;
+ }
+ }
+ if (isp->lsc.on) {
+ lsc_at(isp, cells_w > 1 ? (float)cx0 / (cells_w - 1) : 0, v_at, gains);
+ }
+ pixel(isp, &p, c[0] * gains[0], c[1] * gains[1], c[2] * gains[2], rgb + 3 * (isp->turned ? width - 1 - x : x));
+ }
+ return;
+ }
uint32_t cy = y0 + (uint32_t)((uint64_t)row * ch / height);
const uint8_t *top = raw + (size_t)cy * 2 * f->stride;
uint64_t step = ((uint64_t)cw << 16) / width, at = (uint64_t)x0 << 16;
@@ -283,7 +515,7 @@ render_line(const struct isp *isp, const struct redwood_camera_frame *f, const u
if (isp->lsc.on) {
lsc_at(isp, cells_w > 1 ? (float)cx / (cells_w - 1) : 0, v_pos, gains);
}
- pixel(isp, &p, v[r] * gains[0], g * 0.5f * gains[1], v[b] * gains[2], rgb + 3 * x);
+ pixel(isp, &p, v[r] * gains[0], g * 0.5f * gains[1], v[b] * gains[2], rgb + 3 * (isp->turned ? width - 1 - x : x));
}
}
@@ -545,3 +777,159 @@ isp_lsc_from_flat(struct isp *isp, const struct redwood_camera_frame *f, const u
}
return true;
}
+
+/* one module's part of a tuning file (tools/camtuning.py): its ranges, and where each range's values are */
+static bool
+tune_set(const uint8_t **p, const uint8_t *end, struct isp_tune_set *set, size_t floats)
+{
+ uint32_t n;
+
+ if (end - *p < 4) {
+ return false;
+ }
+ memcpy(&n, *p, 4);
+ *p += 4;
+ if (n == 0 || n > TUNE_LIGHTS) {
+ return false;
+ }
+ set->lights = n;
+ for (uint32_t i = 0; i < n; i++) {
+ if (end - *p < 12) {
+ return false;
+ }
+ memcpy(&set->light[i].lux_from, *p, 4);
+ memcpy(&set->light[i].lux_to, *p + 4, 4);
+ memcpy(&set->light[i].ccts, *p + 8, 4);
+ *p += 12;
+ if (set->light[i].ccts == 0 || set->light[i].ccts > TUNE_CCTS) {
+ return false;
+ }
+ for (uint32_t j = 0; j < set->light[i].ccts; j++) {
+ if ((size_t)(end - *p) < (2 + floats) * 4) {
+ return false;
+ }
+ memcpy(&set->light[i].cct_from[j], *p, 4);
+ memcpy(&set->light[i].cct_to[j], *p + 4, 4);
+ set->light[i].v[j] = (const float *)(const void *)(*p + 8); /* (every field is 4 bytes) */
+ *p += (2 + floats) * 4;
+ }
+ }
+ return true;
+}
+
+bool
+isp_load_tuning(struct isp *isp, const char *path)
+{
+ struct isp_tuning *t = calloc(1, sizeof(*t));
+ int fd = open(path, O_RDONLY);
+ struct stat st;
+ const uint8_t *p, *end;
+ bool good = false;
+
+ if (isp->tuning != NULL) {
+ free(isp->tuning->file);
+ free(isp->tuning);
+ isp->tuning = NULL;
+ }
+ if (t != NULL && fd >= 0 && fstat(fd, &st) == 0 && st.st_size > 8 && st.st_size < (16 << 20) &&
+ (t->file = malloc((size_t)st.st_size)) != NULL && read(fd, t->file, (size_t)st.st_size) == st.st_size) {
+ p = t->file;
+ end = p + st.st_size;
+ good = memcmp(p, "RWCISP02", 8) == 0;
+ p += 8;
+ good = good && tune_set(&p, end, &t->ccm, 9) && tune_set(&p, end, &t->lsc, 4 * LSC_W * LSC_H) && end - p >= 4;
+ if (good) {
+ memcpy(&t->refs, p, 4);
+ p += 4;
+ good = t->refs <= 16 && (size_t)(end - p) >= t->refs * 12;
+ }
+ if (good) {
+ memcpy(t->ref, p, t->refs * 12);
+ }
+ }
+ if (fd >= 0) {
+ close(fd);
+ }
+ if (!good) {
+ if (t != NULL) {
+ free(t->file);
+ }
+ free(t);
+ return false;
+ }
+ isp->tuning = t;
+ return true;
+}
+
+/*
+ * Where x falls among ascending ranges [from[i], to[i]]: in one (a = b), or between two (a, b and how
+ * far from a's end to b's start, 0 to 1); before the first, the first; after the last, the last.
+ */
+static void
+pick(const float *from, const float *to, uint32_t n, float x, uint32_t *a, uint32_t *b, float *t)
+{
+ *a = *b = n - 1;
+ *t = 0;
+ for (uint32_t i = 0; i < n; i++) {
+ if (x <= to[i]) {
+ *a = *b = i;
+ if (i > 0 && x < from[i]) {
+ *a = i - 1;
+ *t = (x - to[i - 1]) / (from[i] - to[i - 1]);
+ }
+ return;
+ }
+ }
+}
+
+/* a module's values for this light and temperature: those of the ranges they fall in, or between */
+static void
+tune_blend(const struct isp_tune_set *set, float lux_index, float cct, size_t floats, float *out)
+{
+ float from[TUNE_LIGHTS], to[TUNE_LIGHTS], tl;
+ uint32_t la, lb;
+
+ for (uint32_t i = 0; i < set->lights; i++) {
+ from[i] = set->light[i].lux_from;
+ to[i] = set->light[i].lux_to;
+ }
+ pick(from, to, set->lights, lux_index, &la, &lb, &tl);
+ memset(out, 0, floats * sizeof(float));
+ for (int k = 0; k < 2; k++) {
+ uint32_t l = k ? lb : la, ca, cb;
+ float w = k ? tl : 1 - tl, tc;
+
+ if (w == 0) {
+ continue;
+ }
+ pick(set->light[l].cct_from, set->light[l].cct_to, set->light[l].ccts, cct, &ca, &cb, &tc);
+ for (size_t i = 0; i < floats; i++) {
+ out[i] += w * ((1 - tc) * set->light[l].v[ca][i] + tc * set->light[l].v[cb][i]);
+ }
+ }
+}
+
+void
+isp_tune(struct isp *isp, float lux_index, float cct)
+{
+ const struct isp_tuning *t = isp->tuning;
+ float m[9], g[4 * LSC_H * LSC_W];
+ const int grid = LSC_H * LSC_W;
+
+ if (t == NULL) {
+ return;
+ }
+ tune_blend(&t->ccm, lux_index, cct, 9, m);
+ isp_set_ccm(isp, m);
+ tune_blend(&t->lsc, lux_index, cct, 4 * (size_t)grid, g);
+ for (int gy = 0; gy < LSC_H; gy++) {
+ for (int gx = 0; gx < LSC_W; gx++) {
+ int at = gy * LSC_W + gx;
+
+ isp->lsc.gain[0][gy][gx] = g[at];
+ isp->lsc.gain[1][gy][gx] = (g[grid + at] + g[2 * grid + at]) / 2;
+ isp->lsc.gain[2][gy][gx] = g[3 * grid + at];
+ }
+ }
+ isp->lsc.on = true;
+}
diff --git a/userland/src/camera_isp.h b/userland/src/camera_isp.h
index 5c3f084..51a8dc2 100644
--- a/userland/src/camera_isp.h
+++ b/userland/src/camera_isp.h
@@ -19,11 +19,18 @@ struct isp_stats {
double bright; /* the level of green that 1 % of the cells are above */
uint32_t hist[64]; /* green above black, 16 steps a bin */
uint32_t cells;
+ /*
+ * Red over green and blue over green, lens-shaded, each cell's counted once however bright it is:
+ * the mean of the cells none of whose colours is near white or near black, and of those of them
+ * that are grey or nearly with the white balance as it is.
+ */
+ double grey[2], neutral[2];
+ uint32_t grey_cells, neutral_cells;
};
/*
* Lens shading: a gain for each colour at each point of a grid laid over the sensor (corners on its
- * corners), between the points bilinear. Off until it is given one (isp_lsc_from_flat()).
+ * corners), between the points bilinear. Off until it is given one (isp_lsc_from_flat(), isp_tune()).
*/
#define LSC_W 17
#define LSC_H 13
@@ -32,23 +39,64 @@ struct isp_lsc {
float gain[3][LSC_H][LSC_W];
};
+/*
+ * A camera's colour tuning, as tools/camtuning.py makes it out of the camera's tuning file of Android's
+ * camera stack: colour matrices and lens shading grids, each per range of the scene's light (a lux
+ * index, Qualcomm's measure: higher is darker) and within that per range of the light's colour
+ * temperature.
+ */
+#define TUNE_LIGHTS 6
+#define TUNE_CCTS 6
+struct isp_tune_set {
+ uint32_t lights;
+ struct {
+ float lux_from, lux_to;
+ uint32_t ccts;
+ float cct_from[TUNE_CCTS], cct_to[TUNE_CCTS];
+ const float *v[TUNE_CCTS]; /* a matrix by rows (9), or R, Gr, Gb and B grids (4 x 13 x 17) */
+ } light[TUNE_LIGHTS];
+};
+struct isp_tuning {
+ struct isp_tune_set ccm, lsc;
+ /*
+ * The white balance's reference lights, from the bluest to the reddest: each as the sensor sees a
+ * grey under it (red over green, blue over green), and the colour temperature its tables are
+ * picked by (tools/camtuning.py). Between them, the line the white balance keeps to.
+ */
+ uint32_t refs;
+ float ref[16][3];
+ void *file;
+};
+
struct isp {
float wb[3]; /* the white balance's gains, green 1 */
float digital; /* more gain, for what AE cannot reach with the sensor's */
+ float zoom; /* the picture is the middle 1/zoom of the sensor each way (1: the whole of it) */
+ bool turned; /* the picture turned half round (a sensor mounted the other way up) */
float ccm[3][3]; /* the colour matrix, camera RGB (white balanced) to linear sRGB; rows sum to 1 */
struct isp_lsc lsc;
uint8_t gamma[4096]; /* linear (959 above black is 3836) to 8 bits, sRGB's curve */
uint32_t asked_lines, asked_gain, asked_frame_lines; /* what AE asked for last */
bool asked;
uint32_t max_frame_lines; /* the longest frame AE may ask for (0: twice the shortest) */
+ struct isp_tuning *tuning; /* isp_load_tuning()'s, or none */
};
void isp_init(struct isp *isp);
-/* the levels, from every 4th cell of every 4th pair of lines */
-void isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s);
+/*
+ * How sharp the picture is in a part of the frame (fractions of it, x0 < x1, y0 < y1): the squares of
+ * green's steps between neighbouring blocks of 4x4 cells, over green's level squared -- higher is sharper,
+ * the same scene at any exposure the same; and green's mean level there (level, may be NULL).
+ */
+float isp_sharpness(const struct redwood_camera_frame *f, const uint8_t *raw, float x0, float y0, float x1, float y1, float *level);
+/* the levels, from every 4th cell of every 4th pair of lines (with the processor's lens shading and white balance, if isp) */
+void isp_measure(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s);
/* the exposure for the frames to come: true, and the command filled in, when it should change */
bool isp_ae(struct isp *isp, const struct redwood_camera_frame *f, const struct isp_stats *s, struct redwood_camera_command *c);
-/* the white balance: grey world, a little of the way at a time */
+/*
+ * The white balance: grey world -- of the nearly grey, when enough is -- kept to the line of the
+ * tuning's reference lights when there is one, a little of the way at a time.
+ */
void isp_awb(struct isp *isp, const struct isp_stats *s);
/*
* A picture of the frame, width x height, of the sensor's middle part with the picture's proportions
@@ -72,5 +120,11 @@ void isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_fra
*/
bool isp_lsc_from_flat(struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, float *min_ratio);
void isp_set_ccm(struct isp *isp, const float m[9]);
+/* the tuning in PATH (tools/camtuning.py's) in place of the one there was: false, and none, if it is not one */
+bool isp_load_tuning(struct isp *isp, const char *path);
+/* the colour matrix and lens shading for a scene of this lux index and colour temperature, from the tuning */
+void isp_tune(struct isp *isp, float lux_index, float cct);
+/* the colour temperature the white balance is at, by the tuning's reference lights (0: there are none) */
+float isp_cct(const struct isp *isp);
#endif
diff --git a/userland/src/camera_test.c b/userland/src/camera_test.c
index fffed67..42b07f4 100644
--- a/userland/src/camera_test.c
+++ b/userland/src/camera_test.c
@@ -128,7 +128,7 @@ cycle(int frames)
failures++;
continue;
}
- isp_measure(f, buffer + f->header_bytes, &s);
+ isp_measure(NULL, f, buffer + f->header_bytes, &s);
printf("camera_test: the %s camera (sensor %u): %ux%u, stride %u, Bayer %u; the first frame %.0f ms after it was asked "
"for; %d frames of %llu in %.2f s (%.1f a second); R %.1f G %.1f B %.1f above black\n", sensor_names[n], f->sensor,
f->width, f->height, f->stride, f->bayer, changed_ms, got, last_seq - first_seq + 1, ticks_ms(f->time - first) / 1000,
@@ -336,7 +336,7 @@ raw(int frames, bool ae, bool awb, bool keep_raw, bool full, int sensor)
got++;
first_seq = first_seq ? first_seq : f->sequence;
- isp_measure(f, raw, &s);
+ isp_measure(&isp, f, raw, &s);
if (awb) {
isp_awb(&isp, &s);
}
diff --git a/userland/src/inject_media.c b/userland/src/inject_media.c
index e86393b..e0bd78a 100644
--- a/userland/src/inject_media.c
+++ b/userland/src/inject_media.c
@@ -35,6 +35,7 @@
* arrives.
*/
#include <CoreFoundation/CoreFoundation.h>
+#include <dlfcn.h>
#include <fcntl.h>
#include <mach-o/dyld.h>
#include <objc/message.h>
@@ -283,6 +284,119 @@ audio_source_start(id self, SEL selector, id *error)
return true;
}
+/*
+ * A photograph's quality prioritization, "speed" at the most. Asked for "balanced" (Camera.app's) or
+ * "quality", the capture device takes a scene for a low-light one whenever a frame was exposed at
+ * least as long as the sensor's still image stabilization threshold
+ * (-[BWFigVideoCaptureDevice _isLowLightSceneUsingFrameStatisticsByPortType:]) -- a threshold from
+ * the sensor's tuning in CameraSetup.plist, 0 for a sensor it has no tuning for, as ours -- and
+ * then asks the stream for a bracket of four frames for SIS fusion (captureType 4, BWSISNode), which
+ * this port has nothing to fuse with: the photograph never came. With "speed" it asks for one frame
+ * (-_resolveStillImageCaptureTypeFromStatisticsByPortType:..., captureType 1), which the stream
+ * gives (camera_device.c, StillImageCaptureNow).
+ */
+static IMP original_quality_prioritization;
+
+static int
+quality_prioritization(id self, SEL selector)
+{
+ int asked = ((int (*)(id, SEL))original_quality_prioritization)(self, selector);
+ static bool said;
+
+ if (asked > 1 && !said) {
+ said = true;
+ say("redwood: mediaserverd: a photograph's quality prioritization %d taken as 1 (speed): one frame, no SIS bracket\n",
+ asked);
+ }
+ return asked > 1 ? 1 : asked;
+}
+
+/*
+ * A photograph's noise reduction (BWGNRNode, between the still image coordinator and the photo
+ * encoder) is a FigSampleBufferProcessor of the iPhone's, which is handed the picture and never
+ * hands one back here: the photograph stopped there, with no error, and the request with it (the
+ * shutter's second press asked for nothing more). The picture goes past it to the node's output as
+ * it came, as with no noise reduction to do.
+ */
+static void
+gnr_render(id self, SEL selector, void *sample_buffer, id input)
+{
+ id output = ((id (*)(id, SEL))objc_msgSend)(self, sel_registerName("output"));
+ static bool said;
+
+ (void)selector, (void)input;
+ if (!said) {
+ said = true;
+ say("redwood: mediaserverd: a photograph goes past the noise reduction, which has nothing to do it with here\n");
+ }
+ if (output != nil) {
+ ((void (*)(id, SEL, void *))objc_msgSend)(output, sel_registerName("emitSampleBuffer:"), sample_buffer);
+ }
+}
+
+/*
+ * A node's output given no sample buffer at all, which no node takes: the session's audio source
+ * (-[BWAudioSourceNode _generateSamples]), whose audio unit never started here (see audio_source_start
+ * above), fills the silence by a timer with buffers it fails to make and emits them all the same --
+ * NULL. Live Photo's staging node added that to an array (-[BWIrisStagingNode renderSampleBuffer:forInput:])
+ * and mediaserverd died of the NSInvalidArgumentException, every half a minute with Live Photo on, the
+ * preview black for a second each time. A NULL is not passed on.
+ */
+static IMP original_emit_sample_buffer;
+
+static void
+emit_sample_buffer(id self, SEL selector, void *sample_buffer)
+{
+ static bool said;
+
+ if (sample_buffer == NULL) {
+ if (!said) {
+ said = true;
+ say("redwood: mediaserverd: a node's output was given no sample buffer (the silent audio source's): not passed on\n");
+ }
+ return;
+ }
+ ((void (*)(id, SEL, void *))original_emit_sample_buffer)(self, selector, sample_buffer);
+}
+
+/*
+ * The photograph's encoder. Camera.app asks for HEIC -- HEVC inside a HEIF file, which CMPhoto makes
+ * through VideoToolbox -- and VideoToolbox here has no HEVC encoder but the iPhone's hardware ones
+ * (com.apple.videotoolbox.videoencoder.hevc and .hevc.gva, behind its AVE): the photo encoder node
+ * failed with kVTCouldNotFindVideoEncoderErr (-12908). Apple's software HEVC codec is in the system
+ * all the same, VCPHEVC.videocodec, whose decoder VideoToolbox takes here for every HEIC picture (see
+ * the top), and there is an encoder in it, HEVCVideoEncoder_CreateInstance, that iOS never registers:
+ * the codec's VCPHEVCRegisterEncoder is an empty function. It is registered here, as the codec
+ * registers its decoder (VTRegisterVideoDecoderWithInfo, not hardware accelerated).
+ */
+static void
+register_hevc_encoder(void)
+{
+ void *codec = dlopen("/System/Library/VideoCodecs/VCPHEVC.videocodec", RTLD_NOW | RTLD_LOCAL);
+ void *vt = dlopen("/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox", RTLD_NOW | RTLD_GLOBAL);
+ void *create = codec != NULL ? dlsym(codec, "HEVCVideoEncoder_CreateInstance") : NULL;
+ int (*register_encoder)(uint32_t, CFDictionaryRef, void *) = vt != NULL ? dlsym(vt, "VTRegisterVideoEncoderWithInfo") : NULL;
+ CFStringRef *id_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_EncoderID") : NULL;
+ CFStringRef *name_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_EncoderName") : NULL;
+ CFStringRef *hardware_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_IsHardwareAccelerated") : NULL;
+ int status;
+
+ if (create == NULL || register_encoder == NULL || id_key == NULL || name_key == NULL || hardware_key == NULL) {
+ say("redwood: mediaserverd: no software HEVC encoder to register (%s)\n", codec == NULL ? "no VCPHEVC.videocodec" :
+ create == NULL ? "no HEVCVideoEncoder_CreateInstance" : "VideoToolbox without VTRegisterVideoEncoderWithInfo");
+ return;
+ }
+ const void *keys[] = { *id_key, *name_key, *hardware_key };
+ const void *values[] = { CFSTR("com.apple.videotoolbox.videoencoder.hevc.vcp"), CFSTR("HEVC (VCPHEVC, software)"),
+ kCFBooleanFalse };
+ CFDictionaryRef info = CFDictionaryCreate(NULL, keys, values, 3, &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+
+ status = register_encoder('hvc1', info, create);
+ CFRelease(info);
+ say("redwood: mediaserverd: the software HEVC encoder of VCPHEVC.videocodec registered for 'hvc1' (%d)\n", status);
+}
+
struct method_spec {
const char *selector, *types;
IMP imp;
@@ -342,6 +456,10 @@ static struct replacement {
(IMP)video_stabilization_method, NULL, true, false },
{ "FigVideoCaptureConnectionConfiguration", "videoStabilizationType", 0x1a20d8f48UL,
(IMP)video_stabilization_type, NULL, true, false },
+ { "FigCaptureStillImageSettings", "qualityPrioritization", 0x1a1f533dcUL,
+ (IMP)quality_prioritization, &original_quality_prioritization, true, false },
+ { "BWGNRNode", "renderSampleBuffer:forInput:", 0x1a1ec83b8UL, (IMP)gnr_render, NULL, true, false },
+ { "BWNodeOutput", "emitSampleBuffer:", 0x1a1ea37e0UL, (IMP)emit_sample_buffer, &original_emit_sample_buffer, true, false },
};
static void
@@ -393,6 +511,7 @@ redwood_media_init(void)
}
if (camera) {
say("redwood: mediaserverd: the kernel has a camera (/dev/redwood-camera)\n");
+ register_hevc_encoder();
}
_dyld_register_func_for_add_image(image_added); /* also called for every image already there */
}