Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
Camera for iOS: photographs, colours by the cameras' own tuning, focus, zoom, the front camera
vcvkk98bc44afef0e
diff --git a/tools/camtuning.py b/tools/camtuning.pynew file mode 100755index 0000000..800744f--- /dev/null+++ b/tools/camtuning.py@@ -0,0 +1,127 @@+#!/usr/bin/env python3+"""Make a camera's colour tuning, as libredwood-media's processor reads it (userland/src/camera_isp.c,+isp_load_tuning), out of the camera's tuning file of Android's camera stack+(/vendor/lib64/camera/com.qti.tuned.<name>.bin of the phone: the vendor's, kept in fw-camera/ and never in+the repository; so is what this makes of it).++usage: camtuning.py TUNED.bin OUT.isp (camtuning.py TUNED.bin - lists what is there only)++The tuning file is a "QTI Chromatix Header" / "Parameter Parser V3.0.1" file, laid out as the sensors'+module files are (see camsensor.py): a table of symbols { id, name[32], kind, ?, parent, offset, size }+and the data they point into. Two of its modules are taken, each as Chromatix keeps it -- a tree whose+first level is the flash (led_idx_data: 0 without one, the only one taken), the next the scene's light+(aec_data: from a lux index to another, Qualcomm's measure of how dark the scene is: higher is darker),+the last the light's colour temperature (cct_data: from a CCT to another, in kelvin):++ cc13 (the colour matrix, IPE's): per light and temperature a 3x3 matrix from the sensor's colours,+ white balanced, to linear sRGB, by rows R, G, B (each row sums to 1)+ lsc34 (lens shading): per light and temperature a gain for each of R, Gr, Gb and B at each point of+ a 17 x 13 grid laid over the sensor (corners on its corners, rows from the top)++and, of its white balance (Qualcomm's AWB), the reference lights (refPtV1): ten of them for every camera+of this phone, each as the sensor sees a grey under it, red over green and blue over green -- from the+bluest daylight to a candle, in the order 10000 K, D75, D65, D50 (whose gains are the AWB's first ones,+initDataV1), cool white and TL84 fluorescent tubes, U30, A (a filament lamp), horizon, candle. Each is+given the colour temperature that the tables above are picked by for it: its own, but 3800 K for U30, a+3000 K light without the infrared a filament gives -- as a white LED is -- whose tables are TL84's (the+wide camera under a warm white LED: its colour shading measured on a white sheet is TL84's table's to+within 3 %, the 2550-2900 K one's is 12 % off in red).++OUT.isp: "RWCISP02"; the number of the matrices' light ranges and per range { lux index from, to (two+floats), the number of temperature ranges, per range { CCT from, to, the matrix (9 floats) } }; the+same for the lens shading, with per temperature range { CCT from, to, R, Gr, Gb, B grids (17 x 13 floats+each) }; the number of reference lights (0 where there are not the ten) and each { R/G, B/G, CCT }.+Everything little-endian, counts as 32-bit words.+"""+import struct+import sys++GRID = 17 * 13+++def symbols(data):+ count = struct.unpack_from('<I', data, 0xa4)[0]+ entries = [struct.unpack_from('<3I', data, 0xa8 + 12 * i) for i in range(count)]+ table, pool = entries[0][1], entries[1][1]+ syms, at = {}, table+ while at + 0x38 <= pool:+ sid = struct.unpack_from('<I', data, at)[0]+ name = data[at + 4:at + 0x24].split(b'\0')[0].decode('latin1')+ _, _, _, ref, size = struct.unpack_from('<5i', data, at + 0x24)+ syms[sid] = (name, data[pool + ref:pool + ref + size])+ at += 0x38+ return syms+++def module(syms, name, floats):+ """[(lux from, lux to, [(cct from, cct to, values)])] of the module's first tree, without the flash"""+ led = next(v for n, v in syms.values() if n == 'mod_%s_led_idx_data' % name)+ _, regions, sid = next(e for e in struct.iter_unpack('<3I', led) if e[0] == 0)+ out = []+ for i in range(regions):+ lux_from, lux_to, _, _, n, cct_sid = struct.unpack_from('<4f2I', syms[sid][1], 24 * i)+ cct = syms[cct_sid][1]+ size = len(cct) // n+ ranges = []+ for j in range(n):+ v = struct.unpack_from('<%df' % (2 + floats), cct, size * j)+ ranges.append((v[0], v[1], v[2:]))+ out.append((lux_from, lux_to, ranges))+ return out+++# the colour temperature each of the ten reference lights picks the tables by (see above)+REFERENCE_CCTS = (10000, 7500, 6500, 5000, 4150, 4000, 3800, 2850, 2300, 1900)+++def references(syms):+ """[(R/G, B/G, CCT)] of the AWB's reference lights, or [] where they are not the ten known"""+ v = next((v for n, v in syms.values() if n == 'refPtV1'), None)+ if v is None:+ return []+ f = struct.unpack('<%df' % (len(v) // 4), v)+ points = [(f[i], f[i + 1]) for i in range(6, len(f) - 1, 2) if f[i] > 0 and f[i + 1] > 0]+ if len(points) != len(REFERENCE_CCTS):+ return []+ return [(rg, bg, cct) for (rg, bg), cct in zip(points, REFERENCE_CCTS)]+++def main():+ if len(sys.argv) < 3:+ sys.exit(__doc__)+ data = open(sys.argv[1], 'rb').read()+ if not data.startswith(b'QTI Chromatix Header') or b'Parameter Parser V3.0.1' not in data[:0x60]:+ sys.exit('camtuning.py: not a tuning file of the kind known here')+ syms = symbols(data)+ ccm = module(syms, 'cc13', 9)+ lsc = module(syms, 'lsc34', 4 * GRID)+ refs = references(syms)++ for what, tree in (('colour matrix', ccm), ('lens shading', lsc)):+ for lux_from, lux_to, ranges in tree:+ for cct_from, cct_to, v in ranges:+ if what == 'colour matrix':+ shown = ' '.join('%.3f' % x for x in v)+ else:+ corners = [v[c * GRID] for c in range(4)]+ shown = 'corner gains R %.2f Gr %.2f Gb %.2f B %.2f' % tuple(corners)+ print('%-13s lux index %4.0f-%-4.0f CCT %5.0f-%-5.0f %s' % (what, lux_from, lux_to, cct_from, cct_to, shown))+ for rg, bg, cct in refs:+ print('reference light R/G %.3f B/G %.3f, tables of %.0f K' % (rg, bg, cct))+ if sys.argv[2] == '-':+ return++ out = bytearray(b'RWCISP02')+ for tree in (ccm, lsc):+ out += struct.pack('<I', len(tree))+ for lux_from, lux_to, ranges in tree:+ out += struct.pack('<2fI', lux_from, lux_to, len(ranges))+ for cct_from, cct_to, v in ranges:+ out += struct.pack('<2f%df' % len(v), cct_from, cct_to, *v)+ out += struct.pack('<I', len(refs))+ for ref in refs:+ out += struct.pack('<3f', *ref)+ open(sys.argv[2], 'wb').write(out)+++main()diff --git a/userland/build.sh b/userland/build.shindex 9089c92..2566c2a 100755--- a/userland/build.sh+++ b/userland/build.sh@@ -273,6 +273,14 @@ for module in ../fw-camera/com.qti.sensormodule.redwood_*.bin; domkdir -p root/fw../tools/camsensor.py "$module" "root/fw/camera-$sensor.tab" >/dev/nulldone+# ... and their colour tuning (libredwood-media's processor, src/camera_isp.c), made by tools/camtuning.py out of the+# cameras' tuning files beside them, com.qti.tuned.redwood_<maker>_<sensor>_<camera>.bin: camera-<sensor>.isp+for tuned in ../fw-camera/com.qti.tuned.redwood_*.bin; do+ [ -f "$tuned" ] || continue+ sensor=$(basename "$tuned" .bin | sed 's/^com\.qti\.tuned\.redwood_[a-z]*_\([a-z0-9]*\)_.*/\1/')+ mkdir -p root/fw+ ../tools/camtuning.py "$tuned" "root/fw/camera-$sensor.isp" >/dev/null+done# ... and with ADSP=1 the image of the audio DSP, from the same place into ../fw-audio (bsd/dev/redwood_adsp.c):# adsp.mdt and adsp.b02 to adsp.b36, 20 MB. Only when asked for: beside GL=1 the payload would be more than the 64 MB# the loader takes.diff --git a/userland/src/camera_device.c b/userland/src/camera_device.cindex 8d118d8..c1d37ed 100644--- a/userland/src/camera_device.c+++ b/userland/src/camera_device.c@@ -1,39 +1,34 @@/*- * The wide camera as iOS's capture stack wants a camera: a FigCaptureDevice with one FigCaptureStream- * (the back camera), made for mediaserverd's BWFigCaptureDeviceVendor in place of the one Apple's- * H10ISP.mediacapture makes of the iPhone's ISP (part of libredwood-media.dylib; inject_media.c- * gives the vendor our device's create function when the kernel has a camera: cam=5).+ * The phone's cameras as iOS's capture stack wants them: a FigCaptureDevice with two FigCaptureStreams,+ * the wide camera as the back camera and the front one as the front camera, made for mediaserverd's+ * BWFigCaptureDeviceVendor in place of the device Apple's H10ISP.mediacapture makes of the iPhone's ISP+ * (part of libredwood-media.dylib; inject_media.c gives the vendor our device's create function when+ * the kernel has a camera: cam=5).** Both are CoreMedia "derived objects" (CMDerivedObjectCreate): a table of functions for the base* class (properties) and one for the device or stream class, as H10ISP's are laid out in iOS 16.5:* device RequestStreams, RelinquishStreams (and a processing session and Pearl calibration, not here)* stream Start, Stop, AddAttachmentsToSampleBuffer- * The stream's frames go to the capture graph through the blocks it is given in its property+ * A stream's frames go to the capture graph through the blocks it is given in its property* "VideoOutputHandlers" ({ output: { PixelBufferHandler: ^(CVPixelBufferRef, CMTime) } }), for the- * outputs "VideoOutputConfigurations" describes (size, pixel format) and "VideoOutputsEnabled"+ * outputs "VideoOutputConfigurations" describes (the graph's own buffer pools) and "VideoOutputsEnabled"* switches on. A frame: read from the kernel's /dev/redwood-camera (RAW10), made into a picture by- * camera_isp.c (exposure, white balance, gamma, scaling), one pixel buffer for each output.+ * camera_isp.c (exposure, white balance kept to the sensor's reference lights, the colour matrix and+ * lens shading of the camera's own tuning, gamma, zoom), one pixel buffer for each output, with the+ * metadata the graph keeps its frame statistics and its zoom by. The kernel gives one sensor's frames+ * at a time: a stream that starts stops the other's frames and has the kernel change sensors.*- * The formats the stream offers are named as the iPhone X's back camera's are in CMCapture's- * D22/AVCaptureSession.plist (FigCaptureStreamFormats "Photo" and "Video-60", by the sensor sizes- * and frame rates the plist matches them by), so that its AVCaptureDevice formats and session presets- * resolve: the pictures are scaled from the sensor's 4000x3000 to whatever size is asked for.+ * The formats the streams offer are named as the iPhone X's cameras' are in CMCapture's+ * D22/AVCaptureSession.plist (FigCaptureStreamFormats, by the sensor sizes and frame rates the plist+ * matches them by), so that their AVCaptureDevice formats and session presets resolve: the pictures+ * are scaled from the sensor's frames to whatever size is asked for.*- * Everything asked of the device and the stream that is not answered here is said in mediaserverd's- * log ("redwood: camera: ..."), once per property.- *- * The graph builds, the session runs, and the pictures of the PrimaryScaler output are Camera.app's- * preview, steadily.+ * The shutter: StillImageCaptureNow asks for one frame of the whole sensor on the StillImage output+ * (the time machine's TimeMachineSuspendNow/ResumeNow are taken and ignored). Focus, where there is+ * a lens motor (the wide camera's): by contrast, continuous, and on the window a tap gives (af_frame()).*- * NOT FINISHED: no photograph is taken. The shutter reaches mediaserverd (-[BWStillImageCoordinatorNode- * initiateStillImageCaptureNowWithSettings:] and then -[BWFigVideoCaptureDevice- * _captureStillImageNowWithSettings:]), and there it quietly comes to nothing: the device's- * _convertStillImageCaptureSettings:toStillImageCaptureNowOptions:captureStreamPropertyValues: gives- * back no stream settings for this port, so _captureStillImageNowOnStreamWithOptions: writes nothing- * on the stream, reports success, and no "StillImageCaptureNow" ever arrives here. What this side- * would do with it is written below and waits for that: one frame of the sensor, the whole of it, to- * the StillImage output. (Those methods of CMCapture are objc_direct: they cannot be replaced to be- * looked into, which is as far as this went.)+ * Everything asked of the device and the streams that is not answered here is said in mediaserverd's+ * log ("redwood: camera: ..."), once per property.*/#include <CoreFoundation/CoreFoundation.h>#include <CoreMedia/CoreMedia.h>@@ -43,6 +38,7 @@#include <dlfcn.h>#include <errno.h>#include <fcntl.h>+#include <math.h>#include <pthread.h>#include <stdarg.h>#include <stdio.h>@@ -117,6 +113,40 @@ static void *(*derived_storage)(CMBaseObjectRef);static CMBaseClassID (*device_class_id)(void);static CMBaseClassID (*stream_class_id)(void);static CFStringRef port_back; /* kFigCapturePortType_BackFacingCamera */+static CFStringRef port_front; /* kFigCapturePortType_FrontFacingCamera */++/*+ * The frames' metadata: what H10ISP's stream attaches to every frame (a dictionary under+ * kFigCaptureSampleBufferAttachmentKey_MetadataDictionary, through the stream's+ * AddAttachmentsToSampleBuffer when the graph defers it), and what the capture device keeps its frame+ * statistics of. -[BWFrameStatisticsByPortType updateWithFrameMetadata:updateFocusDistance:] counts+ * a frame only for a buffer that has it, and -[BWFigVideoCaptureDevice+ * _stillImageCaptureInProgressSubstateForFrameStatisticsByPortType:...] puts a standard photograph+ * off until more than four frames have been counted (then only while AFStatus is 2, a coarse focus+ * scan): without it the shutter's capture was put off on every frame (the retry in+ * _updateStateUsingVideoSampleBuffer:fromCaptureStream:), for ever, and StillImageCaptureNow never+ * came. The dictionary goes on the pixel buffer too, and mutable, as the ISP's does: FigImageControl_Focus+ * (the device's camera controls, on every frame once the sample buffer has metadata) reads it from+ * the pixel buffer with no check that it is there -- CFDictionaryGetValue(NULL, ...): mediaserverd+ * died of SIGSEGV -- and adds its own keys to it. The keys are CoreMedia's; the values come from the+ * frame's header and our ISP's levels, in Apple's units where they are known (the exposure in seconds,+ * gains with 256 for 1x); lux and SNR are rough.+ */+enum {+ MD_PORT, MD_PTS, MD_EXPOSURE, MD_AGC, MD_ISP_DGAIN, MD_SENSOR_DGAIN, MD_ISO_BASE, MD_FRAME_RATE, MD_LUX, MD_SNR,+ MD_AF_STATUS, MD_FOCUSING, MD_AE_STABLE, MD_AE_LOCKED, MD_AWB_STABLE, MD_AWB_LOCKED, MD_AF_LOCKED, MD_ZOOM, MD_KEYS+};+static const char *const md_names[MD_KEYS] = {+ "kFigCaptureStreamMetadata_PortType", "kFigCaptureSampleBufferAttachmentKey_OriginalPresentationTimeStamp",+ "kFigCaptureStreamMetadata_ExposureTime", "kFigCaptureStreamMetadata_AGC", "kFigCaptureStreamMetadata_ispDGain",+ "kFigCaptureStreamMetadata_sensorDGain", "kFigCaptureStreamMetadata_ISOBase", "kFigCaptureStreamMetadata_CurrentFrameRate",+ "kFigCaptureStreamMetadata_LuxLevel", "kFigCaptureStreamMetadata_NormalizedSNR", "kFigCaptureStreamMetadata_AFStatus",+ "kFigCaptureStreamMetadata_FocusingMethod", "kFigCaptureStreamMetadata_AEStable", "kFigCaptureStreamMetadata_AELocked",+ "kFigCaptureStreamMetadata_AWBStable", "kFigCaptureStreamMetadata_AWBLocked", "kFigCaptureStreamMetadata_AFLocked",+ "kFigCaptureStreamMetadata_ZoomFactor",+};+static CFStringRef md_key[MD_KEYS];+static CFStringRef md_attachment; /* kFigCaptureSampleBufferAttachmentKey_MetadataDictionary */static boolfind_functions(void)@@ -129,7 +159,24 @@ find_functions(void)stream_class_id = dlsym(RTLD_DEFAULT, "FigCaptureStreamGetClassID");port = dlsym(RTLD_DEFAULT, "kFigCapturePortType_BackFacingCamera");port_back = port != NULL ? *port : NULL;- if (derived_create == NULL || derived_storage == NULL || device_class_id == NULL || stream_class_id == NULL || port_back == NULL) {+ port = dlsym(RTLD_DEFAULT, "kFigCapturePortType_FrontFacingCamera");+ port_front = port != NULL ? *port : NULL;+ port = dlsym(RTLD_DEFAULT, "kFigCaptureSampleBufferAttachmentKey_MetadataDictionary");+ md_attachment = port != NULL ? *port : NULL;+ for (int i = 0; i < MD_KEYS; i++) {+ port = dlsym(RTLD_DEFAULT, md_names[i]);+ md_key[i] = port != NULL ? *port : NULL;+ if (md_key[i] == NULL) {+ say("%s is not there: the frames' metadata go without it", md_names[i]);+ }+ }+ if (md_attachment == NULL || md_key[MD_PORT] == NULL) {+ say("kFigCaptureSampleBufferAttachmentKey_MetadataDictionary or the port's key is not there: frames without "+ "metadata, and the shutter will wait for ever");+ md_attachment = NULL;+ }+ if (derived_create == NULL || derived_storage == NULL || device_class_id == NULL || stream_class_id == NULL || port_back == NULL ||+ port_front == NULL) {say("CoreMedia's or CMCapture's functions are not there: no camera");return false;}@@ -142,23 +189,10 @@ static CFMutableSetRef said_names;static pthread_mutex_t said_lock = PTHREAD_MUTEX_INITIALIZER;static void-say_once(const char *what, CFStringRef key, CFTypeRef value)+say_value(const char *what, CFStringRef key, CFTypeRef value){char name[160] = "?", text[300] = "";- CFStringRef tag = CFStringCreateWithFormat(NULL, NULL, CFSTR("%s %@"), what, key);-- pthread_mutex_lock(&said_lock);- if (said_names == NULL) {- said_names = CFSetCreateMutable(NULL, 0, &kCFTypeSetCallBacks);- }- bool first = !CFSetContainsValue(said_names, tag);- CFSetAddValue(said_names, tag);- pthread_mutex_unlock(&said_lock);- CFRelease(tag);- if (!first) {- return;- }CFStringGetCString(key, name, sizeof(name), kCFStringEncodingUTF8);if (value != NULL) {CFStringRef d = CFCopyDescription(value);@@ -172,6 +206,25 @@ say_once(const char *what, CFStringRef key, CFTypeRef value)say("%s %s%s%s", what, name, value ? " = " : "", text);}+static void+say_once(const char *what, CFStringRef key, CFTypeRef value)+{+ CFStringRef tag = CFStringCreateWithFormat(NULL, NULL, CFSTR("%s %@"), what, key);++ pthread_mutex_lock(&said_lock);+ if (said_names == NULL) {+ said_names = CFSetCreateMutable(NULL, 0, &kCFTypeSetCallBacks);+ }+ bool first = !CFSetContainsValue(said_names, tag);++ CFSetAddValue(said_names, tag);+ pthread_mutex_unlock(&said_lock);+ CFRelease(tag);+ if (first) {+ say_value(what, key, value);+ }+}+/* ---- numbers and dictionaries ---------------------------------------------------------------------- */static CFNumberRef@@ -242,6 +295,11 @@ static const struct prop stream_props[] = {{ "RegionOfInterestForCameraControls", "Collection", "ReadWrite" }, { "AEMeteringMode", "Number", "ReadWrite" },{ "MaxFrameCoalescingDuration", "Number", "ReadWrite" }, { "LimitAERange", "Boolean", "ReadWrite" },{ "TimeMachineSuspendNow", "Boolean", "WriteOnly" }, { "TimeMachineResumeNow", "Boolean", "WriteOnly" },+ /* focus (the wide camera's lens motor; see af_frame()) */+ { "AutoFocusNow", "Collection", "WriteOnly" }, { "ContinuousAutoFocusNow", "Collection", "WriteOnly" },+ { "FocusNow", "Collection", "WriteOnly" }, { "PeakTrackingFocusNow", "Collection", "WriteOnly" },+ { "LockAFNow", "Boolean", "WriteOnly" }, { "ManualFocusNow", "Collection", "WriteOnly" },+ { "AFWindowParams", "Collection", "ReadWrite" }, { "AutoFocusMode", "Number", "ReadWrite" },}, device_props[] = {{ "StreamArray", "Collection", "ReadOnly" }, { "Clock", "Clock", "ReadOnly" }, { "ClientIsActive", "Boolean", "ReadWrite" },{ "SupportedSynchronizedStreamsGroups", "Collection", "ReadOnly" }, { "MemoryPoolID", "Number", "ReadWrite" },@@ -289,7 +347,7 @@ output_info(int width, int height)}static CFDictionaryRef-format(int sensor_width, int sensor_height, int max_fps, bool binned)+format(int sensor_width, int sensor_height, int max_fps, bool binned, double fov){CFMutableDictionaryRef f = (CFMutableDictionaryRef)new_dict(), outputs = (CFMutableDictionaryRef)new_dict();CMVideoFormatDescriptionRef fd = NULL;@@ -303,7 +361,7 @@ format(int sensor_width, int sensor_height, int max_fps, bool binned)put(f, "VideoIsBinned", CFRetain(binned ? kCFBooleanTrue : kCFBooleanFalse));put(f, "VideoBinningFactorHorizontal", number_i(binned ? 2 : 1));put(f, "VideoBinningFactorVertical", number_i(binned ? 2 : 1));- put(f, "VideoFieldOfView", number_f(65.0));+ put(f, "VideoFieldOfView", number_f(fov));put(f, "VideoMinIntegrationTime", number_i(100));put(f, "VideoMaxIntegrationTime", number_i(1000000 / 15));put(f, "VideoRawBitDepth", number_i(10));@@ -345,18 +403,51 @@ struct output {void * handler; /* a block: ^(CVPixelBufferRef, CMTime) */bool said_shape;uint64_t delivered, refused;+ CVPixelBufferRef held; /* the picture given last, kept from the pool (see deliver) */+ bool said_pool;};+static void let_go(struct output *out);+#define OUTPUTS 3static const char *const output_names[OUTPUTS] = { "PrimaryScaler", "SecondaryScaler", "StillImage" };+#define MAX_ZOOM 16 /* (the most the iPhone X's wide camera zooms digitally) */++/*+ * The device's cameras, a stream each, as the iPhone X's are in CMCapture's D22/AVCaptureSession.plist+ * (its devices' uniqueIDs, and the stream formats its formats and presets are matched to by sensor size+ * and frame rate): the wide camera as the back camera, the front one as the front camera. The kernel+ * gives one sensor's frames at a time; a stream's frames start with its own sensor's (see worker()).+ */+static const struct camera {+ uint32_t sensor; /* the kernel's (the frames' sensor, REDWOOD_CAMERA_SENSOR) */+ const char *name, *unique_id;+ bool turned; /* its sensor the other way up from the iPhone's: the pictures turned half round */+ double focal_length, f_number, pixel_size, fov;+ int formats[2][3]; /* sensor width, height, frames a second of "Photo" and of a video format */+} cameras[2] = {+ { 1, "Back Camera", "com.apple.avfoundation.avcapturedevice.built-in_video:0", false, 4.0, 1.8, 1.22, 65.0,+ { { 4032, 3024, 30 }, { 4224, 2376, 60 } } }, /* "Photo", "Video-60" */+ { 2, "Front Camera", "com.apple.avfoundation.avcapturedevice.built-in_video:1", true, 2.87, 2.2, 1.0, 70.0,+ { { 3088, 2320, 30 }, { 3840, 2160, 30 } } }, /* "Photo", "Video-30" */+};+struct stream {CFArrayRef formats;CFDictionaryRef supported;CFDictionaryRef colour_info; /* from VideoOutputsColorInfo */+ CFDictionaryRef metadata; /* the last frame's (frame_metadata()), for a buffer that has none of its own */+ pthread_mutex_t metadata_lock; /* (not `lock': AddAttachmentsToSampleBuffer may come while a frame is given out) */unsigned stills_wanted; /* photographs asked for by StillImageCaptureNow, not yet given */int format_index;double min_fps, max_fps;+ float zoom; /* ZoomFactor: the picture is the middle 1/zoom of the sensor each way */+ const struct camera *camera;+ CFStringRef port;+ bool af_now; /* a focus asked for (AutoFocusNow and the like): a search at the next frame */+ bool af_locked; /* LockAFNow: no search but one asked for */+ float af_window[4]; /* AFWindowParams: where to focus, as fractions of the frame (x0, y0, x1, y1) */struct output out[OUTPUTS];pthread_mutex_t lock;pthread_t worker;@@ -392,11 +483,11 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato(void)allocator;if (key_is(key, "PortType")) {- value = CFRetain(port_back);+ value = CFRetain(s->port);} else if (key_is(key, "UniqueID") || key_is(key, "ModelID")) {- value = CFRetain(CFSTR("com.apple.avfoundation.avcapturedevice.built-in_video:0"));+ value = CFStringCreateWithCString(NULL, s->camera->unique_id, kCFStringEncodingUTF8);} else if (key_is(key, "NonLocalizedName") || key_is(key, "LocalizedName")) {- value = CFRetain(CFSTR("Back Camera"));+ value = CFStringCreateWithCString(NULL, s->camera->name, kCFStringEncodingUTF8);} else if (key_is(key, "Manufacturer")) {value = CFRetain(CFSTR("Apple Inc."));} else if (key_is(key, "SupportedFormatsArray")) {@@ -452,22 +543,22 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocato} else if (key_is(key, "SupportedPropertiesDictionary")) {value = CFRetain(s->supported);} else if (key_is(key, "FocalLength")) {- value = number_f(4.0); /* (the iPhone X's wide camera's, in millimetres) */+ value = number_f(s->camera->focal_length); /* (the iPhone X's cameras', in millimetres) */} else if (key_is(key, "LensFNumber")) {- value = number_f(1.8);+ value = number_f(s->camera->f_number);} else if (key_is(key, "PixelSize")) {/** The capture-graph builder REQUIRES this of the stream: -[BWFigVideoCaptureStream* initWithCaptureStream:...] does a getProperty: (not IfSupported:) for it, and without it the* init fails, so FigVideoCaptureSourcesActivateAndCreateDevices errors and the graph is never- * built. The sensor's pixel pitch in micrometres (the iPhone X wide camera's); any value in+ * built. The sensor's pixel pitch in micrometres (the iPhone X cameras'); any value in* (0, 100) is accepted.*/- value = number_f(1.22);+ value = number_f(s->camera->pixel_size);} else if (key_is(key, "MaxZoomFactor")) {- value = number_f(16);+ value = number_f(MAX_ZOOM);} else if (key_is(key, "ZoomFactor")) {- value = number_f(1);+ value = number_f(s->zoom);} else if (key_is(key, "SupportedColorSpaces")) {CFNumberRef srgb = number_i(0);@@ -488,6 +579,57 @@ stream_copy_property(CMBaseObjectRef o, CFStringRef key, CFAllocatorRef allocatostatic void torch_set(struct stream *s, CFTypeRef value);+static double+dict_number(CFDictionaryRef d, const char *name, double otherwise)+{+ CFStringRef k = CFStringCreateWithCString(NULL, name, kCFStringEncodingUTF8);+ CFTypeRef v = CFDictionaryGetValue(d, k);+ double n = otherwise;++ CFRelease(k);+ if (v != NULL && CFGetTypeID(v) == CFNumberGetTypeID()) {+ CFNumberGetValue(v, kCFNumberDoubleType, &n);+ }+ return n;+}++/* AFWindowParams: the first window, as fractions of the frame; none (or not one) is the middle */+static void+af_window(struct stream *s, CFTypeRef value)+{+ CFDictionaryRef w = NULL;+ double x0, y0, width, height;++ if (value != NULL && CFGetTypeID(value) == CFArrayGetTypeID() && CFArrayGetCount(value) > 0) {+ w = CFArrayGetValueAtIndex(value, 0);+ }+ if (w == NULL || CFGetTypeID(w) != CFDictionaryGetTypeID() || (width = dict_number(w, "width", 0)) <= 0 ||+ (height = dict_number(w, "height", 0)) <= 0) {+ s->af_window[0] = s->af_window[1] = 0.3f;+ s->af_window[2] = s->af_window[3] = 0.7f;+ return;+ }+ x0 = dict_number(w, "hStart", 0) / 1000;+ y0 = dict_number(w, "vStart", 0) / 1000;+ width /= 1000;+ height /= 1000;+ /* (at least a tenth of the frame each way, about the middle of what was asked) */+ if (width < 0.1) {+ x0 -= (0.1 - width) / 2;+ width = 0.1;+ }+ if (height < 0.1) {+ y0 -= (0.1 - height) / 2;+ height = 0.1;+ }+ x0 = x0 < 0 ? 0 : x0 + width > 1 ? 1 - width : x0;+ y0 = y0 < 0 ? 0 : y0 + height > 1 ? 1 - height : y0;+ s->af_window[0] = (float)x0;+ s->af_window[1] = (float)y0;+ s->af_window[2] = (float)(x0 + width);+ s->af_window[3] = (float)(y0 + height);+}+static OSStatusstream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value){@@ -506,6 +648,24 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)say("stream: format %lld", (long long)i);return 0;}+ if (key_is(key, "ZoomFactor")) {+ /*+ * The zoom: the camera's pictures -- moving and still alike -- made of the middle of the sensor+ * (src/camera_isp.c, crop()), as the iPhone's ISP crops for it. Digital: no more detail than+ * that part of the sensor has.+ */+ double zoom = 1;++ if (value != NULL && CFGetTypeID(value) == CFNumberGetTypeID()) {+ CFNumberGetValue(value, kCFNumberDoubleType, &zoom);+ }+ zoom = zoom < 1 ? 1 : zoom > MAX_ZOOM ? MAX_ZOOM : zoom;+ if (fabs(zoom - s->zoom) >= 0.25 || (zoom == 1) != (s->zoom == 1)) {+ say("stream: zoom %.2fx", zoom);+ }+ s->zoom = (float)zoom;+ return 0;+ }if (key_is(key, "MinimumFrameRate") || key_is(key, "MaximumFrameRate")) {double fps = 0;@@ -541,6 +701,7 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)if (out->config != NULL) {CFRelease(out->config);}+ let_go(out);out->config = CFGetTypeID(values[i]) == CFDictionaryGetTypeID() ? CFRetain(values[i]) : NULL;} else if (enabled) {int64_t on = 1;@@ -592,6 +753,30 @@ stream_set_property(CMBaseObjectRef o, CFStringRef key, CFTypeRef value)torch_set(s, value);return 0;}+ if (key_is(key, "AutoFocusNow") || key_is(key, "ContinuousAutoFocusNow") || key_is(key, "FocusNow") ||+ key_is(key, "PeakTrackingFocusNow") || key_is(key, "LockAFNow") || key_is(key, "ManualFocusNow") ||+ key_is(key, "AFWindowParams") || key_is(key, "AutoFocusMode")) {+ /*+ * Camera.app's tap: AutoFocusMode "AutoFocusModeWeightedSum", AFWindowParams an array of+ * { hStart, vStart, width, height, weight } in thousandths of the frame (the first window is+ * the one taken here; an empty value is the middle), then AutoFocusNow (true). LockAFNow+ * keeps the lens where it is until focus is asked for again.+ */+ if (!key_is(key, "AFWindowParams") || (value != NULL && CFGetTypeID(value) == CFArrayGetTypeID() &&+ CFArrayGetCount(value) > 0)) {+ say_value("stream: focus:", key, value); /* (an empty window, the middle: often, unsaid) */+ }+ if (key_is(key, "AFWindowParams")) {+ af_window(s, value);+ } else if (key_is(key, "LockAFNow")) {+ s->af_locked = true;+ } else if (key_is(key, "AutoFocusNow") || key_is(key, "ContinuousAutoFocusNow") || key_is(key, "FocusNow") ||+ key_is(key, "PeakTrackingFocusNow")) {+ s->af_locked = false;+ s->af_now = true;+ }+ return 0;+ }say_once("stream: set (not used)", key, value);return 0;}@@ -671,7 +856,15 @@ output_buffer(struct output *out)}return NULL;}- if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess) {+ if (CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb) != kCVReturnSuccess && out->held != NULL) {+ let_go(out); /* a pool that will not grow by one: the picture kept is the one to have */+ (void)CVPixelBufferPoolCreatePixelBuffer(NULL, pool, &pb);+ if (!out->said_pool) {+ out->said_pool = true;+ say("%s: the graph's pool has no second buffer: the picture on the screen is drawn over", out->name);+ }+ }+ if (pb == NULL) {/** The preview pool holds one buffer (MinimumBufferCount 1): while the graph still has it,* there is none to be had, and this frame is not given out. Said at most every 300th frame.@@ -688,7 +881,68 @@ output_buffer(struct output *out)typedef void (^pixel_buffer_handler)(CVPixelBufferRef, CMTime);static void-deliver(struct stream *s, struct output *out, const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw)+let_go(struct output *out)+{+ if (out->held != NULL) {+ CVPixelBufferRelease(out->held);+ out->held = NULL;+ }+}++static void+md_put(CFMutableDictionaryRef m, int key, CFTypeRef value)+{+ if (md_key[key] != NULL && value != NULL) {+ CFDictionarySetValue(m, md_key[key], value);+ }+ if (value != NULL) {+ CFRelease(value);+ }+}++/* the frame's metadata, as H10ISP's stream gives them (see md_names) */+static CFDictionaryRef+frame_metadata(const struct isp *isp, const struct redwood_camera_frame *f, const struct isp_stats *st, CFStringRef port)+{+ CFMutableDictionaryRef m;+ double exposure = f->exposure_lines * (double)f->line_ns / 1e9, analog = f->gain_unity ? (double)f->gain / f->gain_unity : 1;+ double total = analog * isp->digital, fps = f->frame_lines && f->line_ns ? 1e9 / ((double)f->frame_lines * f->line_ns) : 30;+ /* (green's level for a second at 1x, a quarter of it: a room's 70 or so, daylight's tens of thousands) */+ double lux = exposure > 0 && total > 0 ? st->mean[1] / (exposure * total) / 4 : 0;++ if (md_attachment == NULL) {+ return NULL;+ }+ m = (CFMutableDictionaryRef)new_dict();+ md_put(m, MD_PORT, CFRetain(port));+ md_put(m, MD_PTS, CMTimeCopyAsDictionary(CMClockMakeHostTimeFromSystemUnits(f->time), NULL));+ md_put(m, MD_EXPOSURE, number_f(exposure));+ md_put(m, MD_AGC, number_i((int64_t)(analog * 256 + 0.5)));+ md_put(m, MD_ISP_DGAIN, number_i((int64_t)(isp->digital * 256 + 0.5)));+ md_put(m, MD_SENSOR_DGAIN, number_i(256));+ md_put(m, MD_ISO_BASE, number_f(32));+ md_put(m, MD_FRAME_RATE, number_f(fps));+ md_put(m, MD_LUX, number_i((int64_t)(lux + 0.5)));+ md_put(m, MD_SNR, number_f(40 - 10 * log10(total < 1 ? 1 : total)));+ md_put(m, MD_AF_STATUS, number_i(0));+ md_put(m, MD_FOCUSING, number_i(0));+ md_put(m, MD_AE_STABLE, CFRetain(kCFBooleanTrue));+ md_put(m, MD_AE_LOCKED, CFRetain(kCFBooleanFalse));+ md_put(m, MD_AWB_STABLE, CFRetain(kCFBooleanTrue));+ md_put(m, MD_AWB_LOCKED, CFRetain(kCFBooleanFalse));+ md_put(m, MD_AF_LOCKED, CFRetain(kCFBooleanFalse));+ /*+ * The zoom the picture was made with: the source node takes it for what the ISP applied+ * (-[BWMultiStreamCameraSourceNode _ispAppliedZoomFactorFromSampleBufferMetadataDictionary:outputIndex:])+ * and would make up the rest of what was asked for itself; without it, a zoomed preview stood still.+ */+ md_put(m, MD_ZOOM, number_f(isp->zoom));+ return m;+}++static void+deliver(struct stream *s, struct output *out, const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw,+ CFDictionaryRef metadata){CVPixelBufferRef pb = output_buffer(out);uint32_t pf, width, height;@@ -720,16 +974,304 @@ deliver(struct stream *s, struct output *out, const struct isp *isp, const strucCVBufferSetAttachment(pb, kCVImageBufferTransferFunctionKey, kCVImageBufferTransferFunction_ITU_R_709_2,kCVAttachmentMode_ShouldPropagate);+ if (metadata != NULL) {+ CFMutableDictionaryRef m = CFDictionaryCreateMutableCopy(NULL, 0, metadata);++ CVBufferSetAttachment(pb, md_attachment, m, kCVAttachmentMode_ShouldPropagate);+ CFRelease(m);+ }+CMTime pts = CMClockMakeHostTimeFromSystemUnits(f->time);((pixel_buffer_handler)out->handler)(pb, pts);- CVPixelBufferRelease(pb);+ /*+ * The moving picture's buffer is kept until the next one is given. The preview's pool would+ * otherwise hand the same buffer straight back -- the graph lets go of it as soon as it has passed+ * it on -- while the window server is still copying it to the screen, and the next frame drawn+ * into it showed as bands of two frames, torn across. Kept, it makes the pool take another.+ */+ if (strcmp(out->name, "StillImage") != 0) {+ let_go(out);+ out->held = pb;+ } else {+ CVPixelBufferRelease(pb);+ }if (out->delivered++ == 0) {say("%s: the first picture given (%ux%u %c%c%c%c)", out->name, width, height, pf >> 24, pf >> 16 & 255,pf >> 8 & 255, pf & 255);}}+/*+ * The scene's light as the cameras' tuning measures it (Qualcomm's lux index, higher is darker): about+ * 46 a doubling of exposure times gain, 197 at a 33 ms exposure and 1x -- the black level's ranges in+ * the tuning of the wide camera pair lux indexes with gains, 195-200 with 1-1.1x and 380-385 with+ * 15.9-16x, at the longest exposure a frame at 30 a second allows.+ */+static float+lux_index(const struct isp *isp, const struct redwood_camera_frame *f)+{+ double exposure = f->exposure_lines * (double)f->line_ns / 1e9 * f->gain / f->gain_unity * isp->digital;++ return (float)(197 + 46 * log2(exposure > 1e-6 ? exposure / 0.0333 : 1e-6 / 0.0333));+}++/*+ * The light's colour temperature: where the white balance is on the line of the tuning's reference+ * lights; without them, guessed from how much more the blue is raised than the red, by what is typical of a phone's sensor --+ * blue's gain over red's about 0.8 in daylight (6500 K), 1.3 under a fluorescent tube (4000 K), 2 under+ * a filament lamp (2850 K) -- between those on the logarithm of the ratio, and no further.+ */+static float+colour_temperature(const struct isp *isp)+{+ static const float ratio[3] = { 0.8f, 1.3f, 2.0f }, kelvin[3] = { 6500, 4000, 2850 };+ float r = isp->wb[0] > 0 ? isp->wb[2] / isp->wb[0] : 1, cct = isp_cct(isp);++ if (cct > 0) {+ return cct; /* (by the tuning's reference lights) */+ }++ if (r <= ratio[0]) {+ return kelvin[0];+ }+ for (int i = 1; i < 3; i++) {+ if (r <= ratio[i]) {+ float t = logf(r / ratio[i - 1]) / logf(ratio[i] / ratio[i - 1]);++ return kelvin[i - 1] + t * (kelvin[i] - kelvin[i - 1]);+ }+ }+ return kelvin[2];+}++/*+ * Focus, where the camera has a lens motor (the frames' focus is a position, not REDWOOD_CAMERA_NO_FOCUS:+ * the wide camera's): by contrast, as there is nothing else to go by -- the sensor's phase-detection+ * pixels are not read. A search climbs from where the lens is: a step of AF_STEP one way, on that way+ * while the middle of the picture gets sharper, back the other way if the first step was not, until+ * two steps in a row are less sharp than the sharpest (or the travel, AF_NEAR to AF_FAR, ends); then+ * around the sharpest in steps of AF_FINE, and there it stays. Each place is measured two frames after+ * the lens was told to go there (it settles in less than one). The lens is only ever moved a step at a+ * time: thrown across its travel it knocked, audibly, against its stop at 0. In focus, it stays until+ * the middle has been a good deal less sharp or quite differently lit for a second, and then until+ * the picture has kept still for a third of one: no searching while the phone is being moved.+ * A focus asked for (a tap: AutoFocusNow, on the window AFWindowParams gave) sweeps instead: the+ * lens is walked to the nearer end of its travel, then measured towards the other in steps of+ * AF_STEP until it is past the sharpest (two steps in a row well under it), then around the sharpest+ * -- it finds what was tapped however far out of focus it was, where the climb gave up on the flat+ * of the curve.+ */+enum { AF_WATCH, AF_STEADY, AF_CLIMB, AF_SWEEP, AF_FINE_SEARCH, AF_SETTLE };+#define AF_NEAR 16 /* (off the stop at 0) */+#define AF_FAR 768 /* (its module file's actuator region goes to 700; near 1023 the PM8008i's LDO5 said over-current) */+#define AF_STEP 48+#define AF_FINE 12++struct af {+ int state;+ uint32_t at, target; /* where the lens was told to go; where it is going a step at a time */+ unsigned wait; /* frames before the next measurement */+ int way; /* +1 nearer, -1 further */+ bool turned;+ unsigned worse; /* steps in a row less sharp than the sharpest */+ uint32_t start, best, fine_to;+ float best_sharpness;+ float sharpness, level; /* in focus: the middle's, to watch */+ float last_sharpness, last_level;+ unsigned unlike, steady, steps;+ bool now; /* asked for: no waiting for the picture to keep still */+ bool sweep; /* ... and the whole travel measured */+ float sum_sharpness, sum_level;+ unsigned measured;+};++static void+af_go(int device, struct af *af, uint32_t target)+{+ struct redwood_camera_command c = { .command = REDWOOD_CAMERA_FOCUS };++ af->target = target < AF_NEAR ? AF_NEAR : target > AF_FAR ? AF_FAR : target;+ c.a = af->at < af->target ? (af->target - af->at > AF_STEP ? af->at + AF_STEP : af->target) :+ (af->at - af->target > AF_STEP ? af->at - AF_STEP : af->target);+ af->at = c.a;+ /*+ * The lens settles in a few milliseconds; what is spoilt is the frame being exposed while it moved+ * (written at a frame's end, the next one's exposure has begun): one frame let go by before a+ * measurement, none on the way.+ */+ af->wait = af->at == af->target ? 1 : 0;+ af->steps++;+ af->sum_sharpness = af->sum_level = 0;+ af->measured = 0;+ (void)write(device, &c, sizeof(c));+}++static void+af_frame(int device, struct af *af, const struct redwood_camera_frame *f, const uint8_t *raw, const float window[4], bool locked)+{+ float level, sharpness;+ unsigned needed = f->gain >= 16 * f->gain_unity ? 2 : 1; /* (in the dark, two frames to a measurement) */++ if (f->focus == REDWOOD_CAMERA_NO_FOCUS) {+ return;+ }+ if (af->at == 0 && af->target == 0) {+ af->at = af->target = f->focus; /* (where the kernel put it) */+ }+ if (af->wait > 0) {+ af->wait--;+ return;+ }+ if (af->at != af->target) {+ af_go(device, af, af->target); /* (the next step of a longer way) */+ return;+ }+ sharpness = isp_sharpness(f, raw, window[0], window[1], window[2], window[3], &level);+ af->sum_sharpness += sharpness;+ af->sum_level += level;+ if (++af->measured < needed) {+ return;+ }+ sharpness = af->sum_sharpness / af->measured;+ level = af->sum_level / af->measured;+ af->sum_sharpness = af->sum_level = 0;+ af->measured = 0;+ switch (af->state) {+ case AF_WATCH:+ if (locked || (af->sharpness > 0 && sharpness > af->sharpness * 0.6f && level > af->level * 0.7f &&+ level < af->level * 1.4f)) {+ af->unlike = 0;+ return;+ }+ if (af->sharpness > 0 && ++af->unlike < 30) {+ return;+ }+ af->state = AF_STEADY;+ af->steady = 0;+ af->last_sharpness = sharpness;+ af->last_level = level;+ return;+ case AF_STEADY:+ if (fabsf(sharpness - af->last_sharpness) < 0.15f * af->last_sharpness && fabsf(level - af->last_level) < 0.1f * af->last_level) {+ af->steady++;+ } else {+ af->steady = 0;+ }+ af->last_sharpness = sharpness;+ af->last_level = level;+ if (af->steady < 10 && !af->now) {+ return;+ }+ af->now = false;+ if (af->sweep) {+ af->sweep = false;+ af->state = AF_SWEEP;+ af->best_sharpness = -1;+ af->worse = 0;+ af->steps = 0;+ af->way = af->at - AF_NEAR < AF_FAR - af->at ? 1 : -1;+ af_go(device, af, af->way > 0 ? AF_NEAR : AF_FAR); /* (walked there a step at a time) */+ return;+ }+ af->state = AF_CLIMB;+ af->best = af->start = af->at;+ af->best_sharpness = sharpness;+ af->way = af->at + AF_STEP > AF_FAR ? -1 : 1;+ af->turned = false;+ af->worse = 0;+ af->steps = 0;+ af_go(device, af, (uint32_t)((int)af->at + af->way * AF_STEP));+ return;+ case AF_CLIMB: {+ int next = (int)af->at + af->way * AF_STEP;+ bool end = next < AF_NEAR || next > AF_FAR;++ if (sharpness > af->best_sharpness * 1.02f) {+ af->best = af->at;+ af->best_sharpness = sharpness;+ af->worse = 0;+ } else {+ af->worse++;+ }+ if (af->worse < 2 && !end) {+ af_go(device, af, (uint32_t)next);+ return;+ }+ next = (int)af->best - af->way * AF_STEP;+ if (!af->turned && af->best == af->start && next >= AF_NEAR && next <= AF_FAR) {+ /* not sharper that way at all: the other way, from where it started */+ af->turned = true;+ af->way = -af->way;+ af->worse = 0;+ af_go(device, af, (uint32_t)next);+ return;+ }+ af->state = AF_FINE_SEARCH;+ af->fine_to = af->best + 2 * AF_FINE > AF_FAR ? AF_FAR : af->best + 2 * AF_FINE;+ af_go(device, af, af->best > 2 * AF_FINE + AF_NEAR ? af->best - 2 * AF_FINE : AF_NEAR);+ return;+ }+ case AF_SWEEP:+ say("focus: sweep: %u %.4f", af->at, sharpness);+ if (sharpness > af->best_sharpness) {+ af->best = af->at;+ af->best_sharpness = sharpness;+ af->worse = 0;+ } else if (sharpness < af->best_sharpness * 0.85f) {+ af->worse++;+ }+ if (af->worse < 2 && (int)af->at + af->way * AF_STEP >= AF_NEAR && (int)af->at + af->way * AF_STEP <= AF_FAR) {+ af_go(device, af, (uint32_t)((int)af->at + af->way * AF_STEP));+ return;+ }+ af->state = AF_FINE_SEARCH;+ af->fine_to = af->best + 2 * AF_FINE > AF_FAR ? AF_FAR : af->best + 2 * AF_FINE;+ af_go(device, af, af->best > 2 * AF_FINE + AF_NEAR ? af->best - 2 * AF_FINE : AF_NEAR);+ return;+ case AF_FINE_SEARCH:+ if (sharpness > af->best_sharpness) {+ af->best = af->at;+ af->best_sharpness = sharpness;+ }+ if (af->at + AF_FINE <= af->fine_to) {+ af_go(device, af, af->at + AF_FINE);+ return;+ }+ af->state = AF_SETTLE;+ af_go(device, af, af->best);+ return;+ case AF_SETTLE:+ af->state = AF_WATCH;+ af->sharpness = sharpness;+ af->level = level;+ af->unlike = 0;+ say("focus: the lens at %u after %u moves (sharpness %.3f)", af->at, af->steps, sharpness);+ return;+ }+}++/* the sensor's colour tuning, when the image has one (fw/camera-<sensor>.isp, tools/camtuning.py) */+static void+load_tuning(struct isp *isp, uint32_t sensor)+{+ static const char *const names[] = { "ov02b10", "s5khm2sd", "ov16a1q", "s5k4h7" };+ char path[96];++ if (sensor >= sizeof(names) / sizeof(names[0])) {+ return;+ }+ snprintf(path, sizeof(path), "/redwood/fw/camera-%s.isp", names[sensor]);+ if (isp_load_tuning(isp, path)) {+ say("the %s's colour tuning from %s: %u colour matrices' lights, %u lens shadings'", names[sensor], path,+ isp->tuning->ccm.lights, isp->tuning->lsc.lights);+ } else {+ say("no colour tuning for the %s (%s): no colour matrix, no lens shading", names[sensor], path);+ isp_set_ccm(isp, (const float[9]){ 1, 0, 0, 0, 1, 0, 0, 0, 1 });+ isp->lsc.on = false;+ }+}+static void *worker(void *context){@@ -740,6 +1282,10 @@ worker(void *context)struct isp_stats st;uint8_t *buffer = NULL;size_t bytes = 0;+ uint32_t tuned_for = ~0u;+ bool switched = false;+ float lux = -1, cct = -1;+ struct af af = { 0 };int device = open(REDWOOD_CAMERA_DEVICE, O_RDWR), failures = 0;pthread_setname_np("redwood-camera");@@ -761,9 +1307,20 @@ worker(void *context)}continue;}+ if (head.sensor != s->camera->sensor) {+ /* another camera's frames (the kernel's last sensor): changed, and the next ones are this one's */+ if (!switched) {+ struct redwood_camera_command c = { .command = REDWOOD_CAMERA_SENSOR, .a = s->camera->sensor, .b = ~0u };++ switched = true;+ say("the kernel's frames are camera %u's: to %u, the %s", head.sensor, s->camera->sensor, s->camera->name);+ (void)write(device, &c, sizeof(c));+ }+ continue;+ }bytes = sizeof(head) + (size_t)head.stride * head.height;buffer = malloc(bytes);- say("frames of %ux%u from the kernel", head.width, head.height);+ say("frames of %ux%u from the kernel (the %s)", head.width, head.height, s->camera->name);continue;}n = read(device, buffer, bytes);@@ -781,12 +1338,36 @@ worker(void *context)const uint8_t *raw = buffer + f->header_bytes;struct redwood_camera_command c;- isp_measure(f, raw, &st);+ isp_measure(&isp, f, raw, &st);isp_awb(&isp, &st);+ if (f->sensor != tuned_for) {+ tuned_for = f->sensor;+ load_tuning(&isp, f->sensor);+ }+ /*+ * Slowly, a second or so: the colour temperature is guessed from the white balance, the tuning it+ * picks changes what the white balance sees, and the colour matrix makes every swing of it plain.+ */+ lux = lux < 0 ? lux_index(&isp, f) : lux * 0.9f + lux_index(&isp, f) * 0.1f;+ cct = cct < 0 ? colour_temperature(&isp) : cct * 0.97f + colour_temperature(&isp) * 0.03f;+ isp_tune(&isp, lux, cct);+ isp.zoom = s->zoom;+ isp.turned = s->camera->turned;if (isp_ae(&isp, f, &st, &c)) {(void)write(device, &c, sizeof(c));}+ if (s->af_now) {+ s->af_now = false;+ /* (whatever it was doing: at once, from the next frame) */+ af.state = AF_STEADY;+ af.now = true;+ af.sweep = true;+ }+ af_frame(device, &af, f, raw, s->af_window, s->af_locked);s->frames++;++ CFDictionaryRef metadata = frame_metadata(&isp, f, &st, s->port);+/** Only once the graph has started the stream. Its PixelBufferHandler blocks hand the buffer* to multiStreamCameraSourceNode_sendBufferToBufferServicingThread, which reaches into the@@ -794,6 +1375,14 @@ worker(void *context)* it walks a pointer that is not there yet and mediaserverd dies of SIGSEGV. The frames* before that are read and thrown away, which is also what settles the exposure.*/+ if (metadata != NULL) {+ pthread_mutex_lock(&s->metadata_lock);+ if (s->metadata != NULL) {+ CFRelease(s->metadata);+ }+ s->metadata = CFRetain(metadata);+ pthread_mutex_unlock(&s->metadata_lock);+ }pthread_mutex_lock(&s->lock);for (int i = 0; s->graph_running && i < OUTPUTS; i++) {struct output *out = &s->out[i];@@ -811,16 +1400,20 @@ worker(void *context)if (still && s->stills_wanted == 0) {continue;}- deliver(s, out, &isp, f, raw);+ deliver(s, out, &isp, f, raw, metadata);if (still) {s->stills_wanted--;say("the photograph is given (%llu so far)", (unsigned long long)++s->stills_given);}}pthread_mutex_unlock(&s->lock);+ if (metadata != NULL) {+ CFRelease(metadata);+ }if (s->frames == 1 || s->frames % 300 == 0) {- say("frame %llu: exposure %.1f ms, gain %.1fx, digital %.2fx, green %.0f above black", f->sequence,- f->exposure_lines * (double)f->line_ns / 1e6, (double)f->gain / f->gain_unity, isp.digital, st.mean[1]);+ say("frame %llu: exposure %.1f ms, gain %.1fx, digital %.2fx, green %.0f above black; white balance R %.2f "+ "B %.2f, lux index %.0f, about %.0f K", f->sequence, f->exposure_lines * (double)f->line_ns / 1e6,+ (double)f->gain / f->gain_unity, isp.digital, st.mean[1], isp.wb[0], isp.wb[2], lux, cct);}}free(buffer);@@ -835,6 +1428,9 @@ worker(void *context)* takes a moment to settle and the exposure longer), the second only when the graph starts the* stream itself (`graph` true, the FigCaptureStream start method).*/+static CMBaseObjectRef all_streams[2]; /* the device's, back and front */+static OSStatus stream_stop(CMBaseObjectRef o);+static OSStatusstream_begin(CMBaseObjectRef o, bool graph){@@ -849,6 +1445,13 @@ stream_begin(CMBaseObjectRef o, bool graph)if (s->running) {return 0;}+ /* one camera at a time: the kernel gives one sensor's frames (the other stream's worker stopped first) */+ for (int i = 0; i < 2; i++) {+ if (all_streams[i] != NULL && all_streams[i] != o && stream_of(all_streams[i])->running) {+ say("stream: the %s's frames stop for the %s's", stream_of(all_streams[i])->camera->name, s->camera->name);+ (void)stream_stop(all_streams[i]);+ }+ }s->running = true;s->frames = 0;say("stream: frames begin (format %d)", s->format_index);@@ -875,6 +1478,9 @@ stream_stop(CMBaseObjectRef o)(unsigned long long)s->out[0].delivered);pthread_mutex_lock(&s->lock);s->graph_running = false;+ for (int i = 0; i < OUTPUTS; i++) {+ let_go(&s->out[i]);+ }pthread_mutex_unlock(&s->lock);s->running = false;if (s->worker_started) {@@ -884,17 +1490,60 @@ stream_stop(CMBaseObjectRef o)return 0;}+static void+add_absent(const void *key, const void *value, void *context)+{+ if (!CFDictionaryContainsKey(context, key)) {+ CFDictionarySetValue(context, key, value);+ }+}++/*+ * The frame's metadata into the sample buffer the graph made of our pixel buffer: the pixel buffer's+ * own dictionary (deliver()), the same object, else the last frame's; whatever the graph has put in a+ * dictionary of the sample buffer's already stays as it is.+ */static OSStatusstream_add_attachments(CMBaseObjectRef o, CMSampleBufferRef sbuf, CFDictionaryRef options){+ struct stream *s = stream_of(o);static bool said;+ CVImageBufferRef image = sbuf != NULL ? CMSampleBufferGetImageBuffer(sbuf) : NULL;+ CFTypeRef ours = image != NULL && md_attachment != NULL ? CVBufferGetAttachment(image, md_attachment, NULL) : NULL, theirs;+ CFMutableDictionaryRef m;- (void)o;- (void)sbuf;if (!said) {said = true;- say("stream: AddAttachmentsToSampleBuffer (options %s)", options ? "given" : "none");+ say("stream: AddAttachmentsToSampleBuffer (options %s): the frames' metadata %s", options ? "given" : "none",+ md_attachment != NULL ? "attached" : "NOT THERE");}+ if (md_attachment == NULL || sbuf == NULL) {+ return 0;+ }+ if (ours != NULL && CFGetTypeID(ours) == CFDictionaryGetTypeID()) {+ CFRetain(ours);+ } else {+ pthread_mutex_lock(&s->metadata_lock);+ ours = s->metadata != NULL ? CFDictionaryCreateMutableCopy(NULL, 0, s->metadata) : NULL;+ pthread_mutex_unlock(&s->metadata_lock);+ }+ if (ours == NULL) {+ return 0;+ }+ theirs = CMGetAttachment(sbuf, md_attachment, NULL);+ if (theirs == ours) {+ CFRelease(ours);+ return 0;+ }+ if (theirs != NULL && CFGetTypeID(theirs) == CFDictionaryGetTypeID()) {+ m = CFDictionaryCreateMutableCopy(NULL, 0, theirs);+ CFDictionaryApplyFunction(ours, add_absent, m);+ } else {+ m = (CFMutableDictionaryRef)CFRetain(ours);+ }+ CMSetAttachment(sbuf, md_attachment, m, kCMAttachmentMode_ShouldPropagate);+ CFRelease(m);+ CFRelease(ours);return 0;}@@ -928,6 +1577,9 @@ stream_finalize(CMBaseObjectRef o)if (s->colour_info) {CFRelease(s->colour_info);}+ if (s->metadata) {+ CFRelease(s->metadata);+ }if (s->torch >= 0) {close(s->torch);}@@ -948,7 +1600,7 @@ static const struct stream_class stream_derived = { 2, stream_start, stream_stopstatic const struct vtable stream_vtable = { NULL, &stream_base, &stream_derived };static CMBaseObjectRef-stream_create(CFAllocatorRef allocator)+stream_create(CFAllocatorRef allocator, const struct camera *camera, CFStringRef port){CMBaseObjectRef o = NULL;struct stream *s;@@ -960,15 +1612,21 @@ stream_create(CFAllocatorRef allocator)s = stream_of(o);memset(s, 0, sizeof(*s));pthread_mutex_init(&s->lock, NULL);+ pthread_mutex_init(&s->metadata_lock, NULL);s->torch = -1;s->min_fps = 2;+ s->zoom = 1;+ s->af_window[0] = s->af_window[1] = 0.3f;+ s->af_window[2] = s->af_window[3] = 0.7f;s->max_fps = 30;+ s->camera = camera;+ s->port = port;for (int i = 0; i < OUTPUTS; i++) {s->out[i].name = output_names[i];}- /* "Photo" and "Video-60" of D22/AVCaptureSession.plist's back camera */- CFDictionaryRef f[2] = { format(4032, 3024, 30, false), format(4224, 2376, 60, false) };+ CFDictionaryRef f[2] = { format(camera->formats[0][0], camera->formats[0][1], camera->formats[0][2], false, camera->fov),+ format(camera->formats[1][0], camera->formats[1][1], camera->formats[1][2], false, camera->fov) };s->formats = CFArrayCreate(NULL, (const void **)f, 2, &kCFTypeArrayCallBacks);s->supported = supported(stream_props, sizeof(stream_props) / sizeof(stream_props[0]));@@ -1102,7 +1760,7 @@ OSStatusredwood_camera_device_create(CFAllocatorRef allocator, CFDictionaryRef options, CMBaseObjectRef *out){static bool found, usable;- CMBaseObjectRef device = NULL, stream;+ CMBaseObjectRef device = NULL, stream[2];struct device *d;(void)options;@@ -1113,20 +1771,28 @@ redwood_camera_device_create(CFAllocatorRef allocator, CFDictionaryRef options,if (!usable || out == NULL) {return ERR_UNSUPPORTED;}- if ((stream = stream_create(allocator)) == NULL) {+ if ((stream[0] = stream_create(allocator, &cameras[0], port_back)) == NULL) {+ return ERR_UNSUPPORTED;+ }+ if ((stream[1] = stream_create(allocator, &cameras[1], port_front)) == NULL) {+ CFRelease(stream[0]);return ERR_UNSUPPORTED;}if (derived_create(allocator, &device_vtable, device_class_id(), &device) != 0 || device == NULL) {- CFRelease(stream);+ CFRelease(stream[0]);+ CFRelease(stream[1]);say("the device cannot be made");return ERR_UNSUPPORTED;}d = derived_storage(device);memset(d, 0, sizeof(*d));- d->streams = CFArrayCreate(NULL, (const void **)&stream, 1, &kCFTypeArrayCallBacks);+ d->streams = CFArrayCreate(NULL, (const void **)stream, 2, &kCFTypeArrayCallBacks);+ all_streams[0] = stream[0]; /* (kept by the device's array) */+ all_streams[1] = stream[1];d->supported = supported(device_props, sizeof(device_props) / sizeof(device_props[0]));- CFRelease(stream);+ CFRelease(stream[0]);+ CFRelease(stream[1]);*out = device;- say("the device made: one stream, the back camera");+ say("the device made: two streams, the back camera and the front one");return 0;}diff --git a/userland/src/camera_isp.c b/userland/src/camera_isp.cindex 6bfd14f..626472f 100644--- a/userland/src/camera_isp.c+++ b/userland/src/camera_isp.c@@ -3,11 +3,15 @@* cell, the nearest cell for each pixel of the picture (previews), or every pixel by bilinear* demosaic (stills); per pixel, in this order: the black level off, lens shading (a grid of gains per* colour, when there is one), the white balance's and AE's gains, the colour matrix (identity until- * one is set), sRGB's curve. Grey-world white balance; no noise reduction yet.+ * one is set), sRGB's curve. Grey-world white balance; no noise reduction yet. The colour matrix and+ * the lens shading can come from the camera's own tuning (isp_load_tuning(), isp_tune()).*/+#include <fcntl.h>#include <math.h>#include <stdlib.h>#include <string.h>+#include <unistd.h>+#include <sys/stat.h>#include "camera_isp.h"#define WHITE 959 /* 1023 less the black level of 64 */@@ -19,6 +23,7 @@ isp_init(struct isp *isp)memset(isp, 0, sizeof(*isp));isp->wb[0] = isp->wb[1] = isp->wb[2] = 1;isp->digital = 1;+ isp->zoom = 1;for (int i = 0; i < 3; i++) {isp->ccm[i][i] = 1;}@@ -56,29 +61,134 @@ cell(const uint8_t *top, size_t stride, uint32_t cx, int v[4])}}+float+isp_sharpness(const struct redwood_camera_frame *f, const uint8_t *raw, float x0, float y0, float x1, float y1, float *level)+{+ enum { BLOCK = 4, MAX_BLOCKS = 512 };+ uint32_t cells_w = f->width / 2, cells_h = f->height / 2;+ uint32_t cx0 = (uint32_t)(x0 * cells_w), cx1 = (uint32_t)(x1 * cells_w), cy0 = (uint32_t)(y0 * cells_h),+ cy1 = (uint32_t)(y1 * cells_h);+ float above[MAX_BLOCKS], here[MAX_BLOCKS];+ double steps = 0, sum = 0;+ uint32_t blocks_w, rows = 0, n = 0;+ int r, b;++ cell_places(f->bayer, &r, &b);+ cx1 = cx1 > cells_w ? cells_w : cx1;+ cy1 = cy1 > cells_h ? cells_h : cy1;+ blocks_w = cx1 > cx0 ? (cx1 - cx0) / BLOCK : 0;+ blocks_w = blocks_w > MAX_BLOCKS ? MAX_BLOCKS : blocks_w;+ /*+ * Green's mean over blocks of 4x4 cells, and the squares of the steps between neighbouring+ * blocks: a cell's own noise is a quarter of itself in a block's mean, and the picture's edges are+ * not -- in the dark, at 32x and more, the steps from cell to cell were the noise's alone and the+ * focus went wherever it liked.+ */+ for (uint32_t by = cy0; by + BLOCK <= cy1; by += BLOCK, rows++) {+ for (uint32_t bx = 0; bx < blocks_w; bx++) {+ uint32_t g = 0;++ for (uint32_t cy = by; cy < by + BLOCK; cy++) {+ const uint8_t *top = raw + (size_t)cy * 2 * f->stride;++ for (uint32_t cx = cx0 + bx * BLOCK; cx < cx0 + bx * BLOCK + BLOCK; cx++) {+ int v[4];++ cell(top, f->stride, cx, v);+ for (int i = 0; i < 4; i++) {+ g += i != r && i != b ? (uint32_t)v[i] : 0;+ }+ }+ }+ here[bx] = g / (2.0f * BLOCK * BLOCK);+ sum += here[bx];+ n++;+ if (bx > 0) {+ steps += (here[bx] - here[bx - 1]) * (here[bx] - here[bx - 1]);+ }+ if (rows > 0) {+ steps += (here[bx] - above[bx]) * (here[bx] - above[bx]);+ }+ }+ memcpy(above, here, sizeof(float) * blocks_w);+ }+ double mean = n ? sum / n : 0;++ if (level != NULL) {+ *level = (float)mean;+ }+ return mean > 0 ? (float)(steps / n / (mean * mean)) : 0;+}++static inline void lsc_at(const struct isp *isp, float u, float v, float g[3]);+void-isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s)+isp_measure(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s){uint32_t cells_w = f->width / 2, cells_h = f->height / 2;- double sum[3] = { 0, 0, 0 };+ double sum[3] = { 0, 0, 0 }, grey[2] = { 0, 0 }, neutral[2] = { 0, 0 };+ float gains[3] = { 1, 1, 1 };int r, b;memset(s, 0, sizeof(*s));cell_places(f->bayer, &r, &b);for (uint32_t cy = 0; cy < cells_h; cy += 4) {const uint8_t *top = raw + (size_t)cy * 2 * f->stride;+ float v_pos = cells_h > 1 ? (float)cy / (cells_h - 1) : 0;for (uint32_t cx = 0; cx < cells_w; cx += 4) {- int v[4], g = 0;+ int v[4], g = 0, top_v = 0;cell(top, f->stride, cx, v);for (int i = 0; i < 4; i++) {g += i != r && i != b ? v[i] : 0;+ top_v = v[i] > top_v ? v[i] : top_v;}g /= 2;sum[0] += v[r];sum[1] += g;sum[2] += v[b];+ /*+ * The white balance leaves out a cell with a colour near white -- a lamp, a screen:+ * green, the most sensitive, stops there first and the cell would count as magenta --+ * and one near black, where noise cut off at the black level is all there is.+ */+ if (top_v < WHITE * 9 / 10 && g >= 24) {+ float cr = (float)v[r], cg = (float)g, cb = (float)v[b];++ /*+ * As the picture will have it: the lens shading raises the corners several times, and+ * the light there counts for that much more than in the frame as it comes.+ */+ if (isp != NULL && isp->lsc.on) {+ lsc_at(isp, cells_w > 1 ? (float)cx / (cells_w - 1) : 0, v_pos, gains);+ cr *= gains[0];+ cg *= gains[1];+ cb *= gains[2];+ }+ /*+ * Each cell's colour, not its light: summed, a lamp or a screen in the picture -- tens of+ * times brighter than the room it is in -- would have the white balance to itself, and+ * the room would be left the colour of its light.+ */+ grey[0] += cr / cg;+ grey[1] += cb / cg;+ s->grey_cells++;+ /*+ * Grey, or within a third of it in red and blue, with the white balance as it is: what+ * the white balance should make grey, without the coloured things around it (the+ * colour matrix makes any mistake in it twice as plain).+ */+ if (isp != NULL) {+ float wr = cr * isp->wb[0] / cg, wb = cb * isp->wb[2] / cg;++ if (wr > 0.75f && wr < 1.33f && wb > 0.75f && wb < 1.33f) {+ neutral[0] += cr / cg;+ neutral[1] += cb / cg;+ s->neutral_cells++;+ }+ }+ }s->hist[g >> 4 > 63 ? 63 : g >> 4]++;s->cells++;}@@ -86,6 +196,10 @@ isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct ispfor (int c = 0; c < 3; c++) {s->mean[c] = s->cells ? sum[c] / s->cells : 0;}+ for (int c = 0; c < 2; c++) {+ s->grey[c] = s->grey_cells ? grey[c] / s->grey_cells : 0;+ s->neutral[c] = s->neutral_cells ? neutral[c] / s->neutral_cells : 0;+ }for (int i = 63, above = 0; i >= 0; i--) {above += s->hist[i];if (above * 100 >= (int)s->cells) {@@ -155,25 +269,97 @@ isp_ae(struct isp *isp, const struct redwood_camera_frame *f, const struct isp_sreturn true;}+/*+ * The point of the reference lights' line nearest a light seen as red over green rg and blue over+ * green bg, on their logarithms: where along it (segment i and how far into it, 0 to 1).+ */+static void+on_reference_line(const struct isp_tuning *t, float rg, float bg, uint32_t *segment, float *along)+{+ float x = logf(rg), y = logf(bg), best = INFINITY;++ *segment = 0;+ *along = 0;+ for (uint32_t i = 0; i + 1 < t->refs; i++) {+ float x0 = logf(t->ref[i][0]), y0 = logf(t->ref[i][1]), dx = logf(t->ref[i + 1][0]) - x0,+ dy = logf(t->ref[i + 1][1]) - y0, length = dx * dx + dy * dy;+ float u = length > 0 ? ((x - x0) * dx + (y - y0) * dy) / length : 0, d;++ u = u < 0 ? 0 : u > 1 ? 1 : u;+ d = (x0 + u * dx - x) * (x0 + u * dx - x) + (y0 + u * dy - y) * (y0 + u * dy - y);+ if (d < best) {+ best = d;+ *segment = i;+ *along = u;+ }+ }+}++float+isp_cct(const struct isp *isp)+{+ const struct isp_tuning *t = isp->tuning;+ uint32_t i;+ float u;++ if (t == NULL || t->refs < 2 || isp->wb[0] <= 0 || isp->wb[2] <= 0) {+ return 0;+ }+ on_reference_line(t, 1 / isp->wb[0], 1 / isp->wb[2], &i, &u);+ return t->ref[i][2] + u * (t->ref[i + 1][2] - t->ref[i][2]);+}+voidisp_awb(struct isp *isp, const struct isp_stats *s){- if (s->mean[0] < 2 || s->mean[2] < 2 || s->mean[1] < 2) {+ /*+ * The nearly grey cells, when there are enough of them (1 in 32); else every cell neither near+ * white nor near black, when there are enough of those (1 in 16); else every cell's light.+ */+ float r, b;++ if (s->neutral_cells * 32 >= s->cells && s->neutral[0] > 0.01 && s->neutral[1] > 0.01) {+ r = (float)(1 / s->neutral[0]);+ b = (float)(1 / s->neutral[1]);+ } else if (s->grey_cells * 16 >= s->cells && s->grey[0] > 0.01 && s->grey[1] > 0.01) {+ r = (float)(1 / s->grey[0]);+ b = (float)(1 / s->grey[1]);+ } else if (s->mean[0] >= 2 && s->mean[1] >= 2 && s->mean[2] >= 2) {+ r = (float)(s->mean[1] / s->mean[0]);+ b = (float)(s->mean[1] / s->mean[2]);+ } else {return;}- float r = (float)(s->mean[1] / s->mean[0]), b = (float)(s->mean[1] / s->mean[2]);+ /*+ * Onto the line of the reference lights: a scene's own colours pull grey world off it -- a pink+ * screen, a green wall -- and off it is what shows as green or magenta, while along it is a+ * warmer or colder white. (The wide camera under a warm white LED: grey world of the room with a+ * screen in it 15 % short in both red and blue, the light measured on a white sheet; kept to the+ * line, 2 % off.)+ */+ if (isp->tuning != NULL && isp->tuning->refs >= 2) {+ const struct isp_tuning *t = isp->tuning;+ uint32_t i;+ float u;++ on_reference_line(t, 1 / r, 1 / b, &i, &u);+ r = 1 / expf(logf(t->ref[i][0]) + u * (logf(t->ref[i + 1][0]) - logf(t->ref[i][0])));+ b = 1 / expf(logf(t->ref[i][1]) + u * (logf(t->ref[i + 1][1]) - logf(t->ref[i][1])));+ }r = r < 0.5f ? 0.5f : r > 4 ? 4 : r;b = b < 0.5f ? 0.5f : b > 4 ? 4 : b;- isp->wb[0] = isp->wb[0] * 0.8f + r * 0.2f;- isp->wb[2] = isp->wb[2] * 0.8f + b * 0.2f;+ isp->wb[0] = isp->wb[0] * 0.9f + r * 0.1f;+ isp->wb[2] = isp->wb[2] * 0.9f + b * 0.1f;}-/* the part of the sensor's cells the picture shows: its middle, in the picture's proportions */+/* the part of the sensor's cells the picture shows: its middle, in the picture's proportions, 1/zoom of it each way */static void-crop(const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint32_t *x0, uint32_t *y0, uint32_t *cw, uint32_t *ch)+crop(const struct isp *isp, const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint32_t *x0, uint32_t *y0,+ uint32_t *cw, uint32_t *ch){uint32_t w = f->width / 2, h = f->height / 2;+ float zoom = isp->zoom < 1 ? 1 : isp->zoom;if ((uint64_t)width * h > (uint64_t)height * w) {*cw = w;@@ -182,6 +368,8 @@ crop(const struct redwood_camera_frame *f, uint32_t width, uint32_t height, uint*ch = h;*cw = (uint32_t)((uint64_t)h * width / height);}+ *cw = (uint32_t)(*cw / zoom) < 2 ? 2 : (uint32_t)(*cw / zoom);+ *ch = (uint32_t)(*ch / zoom) < 2 ? 2 : (uint32_t)(*ch / zoom);*x0 = (w - *cw) / 2;*y0 = (h - *ch) / 2;}@@ -264,9 +452,53 @@ render_line(const struct isp *isp, const struct redwood_camera_frame *f, const uint r, b;struct pipe p;- crop(f, width, height, &x0, &y0, &cw, &ch);+ crop(isp, f, width, height, &x0, &y0, &cw, &ch);cell_places(f->bayer, &r, &b);pipe_init(isp, &p);+ if (isp->turned) {+ row = height - 1 - row; /* (and each line from its end, below) */+ }+ if (cw < width && ch < height) {+ /*+ * Magnified (the zoom): each pixel between the four cells around it. The nearest cell blown up+ * to a square of pixels showed every cell's own colour, its noise and the colours a 2x2 cell+ * gets wrong at an edge, as coloured dots.+ */+ float fy = y0 + (row + 0.5f) * ch / height - 0.5f, ty, v_at;+ uint32_t cy0 = fy < 0 ? 0 : (uint32_t)fy, cy1;++ cy0 = cy0 > cells_h - 1 ? cells_h - 1 : cy0;+ cy1 = cy0 + 1 > cells_h - 1 ? cells_h - 1 : cy0 + 1;+ ty = fy < (float)cy0 ? 0 : fy - cy0;+ v_at = cells_h > 1 ? (float)cy0 / (cells_h - 1) : 0;+ const uint8_t *t0 = raw + (size_t)cy0 * 2 * f->stride, *t1 = raw + (size_t)cy1 * 2 * f->stride;+ float gains[3] = { 1, 1, 1 };++ for (uint32_t x = 0; x < width; x++) {+ float fx = x0 + (x + 0.5f) * cw / width - 0.5f, tx, c[3] = { 0, 0, 0 };+ uint32_t cx0 = fx < 0 ? 0 : (uint32_t)fx, cx1;++ cx0 = cx0 > cells_w - 1 ? cells_w - 1 : cx0;+ cx1 = cx0 + 1 > cells_w - 1 ? cells_w - 1 : cx0 + 1;+ tx = fx < (float)cx0 ? 0 : fx - cx0;+ for (int k = 0; k < 4; k++) {+ int v[4];+ float w = (k & 1 ? tx : 1 - tx) * (k & 2 ? ty : 1 - ty);++ cell(k & 2 ? t1 : t0, f->stride, k & 1 ? cx1 : cx0, v);+ c[0] += w * v[r];+ c[2] += w * v[b];+ for (int j = 0; j < 4; j++) {+ c[1] += j != r && j != b ? w * v[j] * 0.5f : 0;+ }+ }+ if (isp->lsc.on) {+ lsc_at(isp, cells_w > 1 ? (float)cx0 / (cells_w - 1) : 0, v_at, gains);+ }+ pixel(isp, &p, c[0] * gains[0], c[1] * gains[1], c[2] * gains[2], rgb + 3 * (isp->turned ? width - 1 - x : x));+ }+ return;+ }uint32_t cy = y0 + (uint32_t)((uint64_t)row * ch / height);const uint8_t *top = raw + (size_t)cy * 2 * f->stride;uint64_t step = ((uint64_t)cw << 16) / width, at = (uint64_t)x0 << 16;@@ -283,7 +515,7 @@ render_line(const struct isp *isp, const struct redwood_camera_frame *f, const uif (isp->lsc.on) {lsc_at(isp, cells_w > 1 ? (float)cx / (cells_w - 1) : 0, v_pos, gains);}- pixel(isp, &p, v[r] * gains[0], g * 0.5f * gains[1], v[b] * gains[2], rgb + 3 * x);+ pixel(isp, &p, v[r] * gains[0], g * 0.5f * gains[1], v[b] * gains[2], rgb + 3 * (isp->turned ? width - 1 - x : x));}}@@ -545,3 +777,159 @@ isp_lsc_from_flat(struct isp *isp, const struct redwood_camera_frame *f, const u}return true;}++/* one module's part of a tuning file (tools/camtuning.py): its ranges, and where each range's values are */+static bool+tune_set(const uint8_t **p, const uint8_t *end, struct isp_tune_set *set, size_t floats)+{+ uint32_t n;++ if (end - *p < 4) {+ return false;+ }+ memcpy(&n, *p, 4);+ *p += 4;+ if (n == 0 || n > TUNE_LIGHTS) {+ return false;+ }+ set->lights = n;+ for (uint32_t i = 0; i < n; i++) {+ if (end - *p < 12) {+ return false;+ }+ memcpy(&set->light[i].lux_from, *p, 4);+ memcpy(&set->light[i].lux_to, *p + 4, 4);+ memcpy(&set->light[i].ccts, *p + 8, 4);+ *p += 12;+ if (set->light[i].ccts == 0 || set->light[i].ccts > TUNE_CCTS) {+ return false;+ }+ for (uint32_t j = 0; j < set->light[i].ccts; j++) {+ if ((size_t)(end - *p) < (2 + floats) * 4) {+ return false;+ }+ memcpy(&set->light[i].cct_from[j], *p, 4);+ memcpy(&set->light[i].cct_to[j], *p + 4, 4);+ set->light[i].v[j] = (const float *)(const void *)(*p + 8); /* (every field is 4 bytes) */+ *p += (2 + floats) * 4;+ }+ }+ return true;+}++bool+isp_load_tuning(struct isp *isp, const char *path)+{+ struct isp_tuning *t = calloc(1, sizeof(*t));+ int fd = open(path, O_RDONLY);+ struct stat st;+ const uint8_t *p, *end;+ bool good = false;++ if (isp->tuning != NULL) {+ free(isp->tuning->file);+ free(isp->tuning);+ isp->tuning = NULL;+ }+ if (t != NULL && fd >= 0 && fstat(fd, &st) == 0 && st.st_size > 8 && st.st_size < (16 << 20) &&+ (t->file = malloc((size_t)st.st_size)) != NULL && read(fd, t->file, (size_t)st.st_size) == st.st_size) {+ p = t->file;+ end = p + st.st_size;+ good = memcmp(p, "RWCISP02", 8) == 0;+ p += 8;+ good = good && tune_set(&p, end, &t->ccm, 9) && tune_set(&p, end, &t->lsc, 4 * LSC_W * LSC_H) && end - p >= 4;+ if (good) {+ memcpy(&t->refs, p, 4);+ p += 4;+ good = t->refs <= 16 && (size_t)(end - p) >= t->refs * 12;+ }+ if (good) {+ memcpy(t->ref, p, t->refs * 12);+ }+ }+ if (fd >= 0) {+ close(fd);+ }+ if (!good) {+ if (t != NULL) {+ free(t->file);+ }+ free(t);+ return false;+ }+ isp->tuning = t;+ return true;+}++/*+ * Where x falls among ascending ranges [from[i], to[i]]: in one (a = b), or between two (a, b and how+ * far from a's end to b's start, 0 to 1); before the first, the first; after the last, the last.+ */+static void+pick(const float *from, const float *to, uint32_t n, float x, uint32_t *a, uint32_t *b, float *t)+{+ *a = *b = n - 1;+ *t = 0;+ for (uint32_t i = 0; i < n; i++) {+ if (x <= to[i]) {+ *a = *b = i;+ if (i > 0 && x < from[i]) {+ *a = i - 1;+ *t = (x - to[i - 1]) / (from[i] - to[i - 1]);+ }+ return;+ }+ }+}++/* a module's values for this light and temperature: those of the ranges they fall in, or between */+static void+tune_blend(const struct isp_tune_set *set, float lux_index, float cct, size_t floats, float *out)+{+ float from[TUNE_LIGHTS], to[TUNE_LIGHTS], tl;+ uint32_t la, lb;++ for (uint32_t i = 0; i < set->lights; i++) {+ from[i] = set->light[i].lux_from;+ to[i] = set->light[i].lux_to;+ }+ pick(from, to, set->lights, lux_index, &la, &lb, &tl);+ memset(out, 0, floats * sizeof(float));+ for (int k = 0; k < 2; k++) {+ uint32_t l = k ? lb : la, ca, cb;+ float w = k ? tl : 1 - tl, tc;++ if (w == 0) {+ continue;+ }+ pick(set->light[l].cct_from, set->light[l].cct_to, set->light[l].ccts, cct, &ca, &cb, &tc);+ for (size_t i = 0; i < floats; i++) {+ out[i] += w * ((1 - tc) * set->light[l].v[ca][i] + tc * set->light[l].v[cb][i]);+ }+ }+}++void+isp_tune(struct isp *isp, float lux_index, float cct)+{+ const struct isp_tuning *t = isp->tuning;+ float m[9], g[4 * LSC_H * LSC_W];+ const int grid = LSC_H * LSC_W;++ if (t == NULL) {+ return;+ }+ tune_blend(&t->ccm, lux_index, cct, 9, m);+ isp_set_ccm(isp, m);+ tune_blend(&t->lsc, lux_index, cct, 4 * (size_t)grid, g);+ for (int gy = 0; gy < LSC_H; gy++) {+ for (int gx = 0; gx < LSC_W; gx++) {+ int at = gy * LSC_W + gx;++ isp->lsc.gain[0][gy][gx] = g[at];+ isp->lsc.gain[1][gy][gx] = (g[grid + at] + g[2 * grid + at]) / 2;+ isp->lsc.gain[2][gy][gx] = g[3 * grid + at];+ }+ }+ isp->lsc.on = true;+}diff --git a/userland/src/camera_isp.h b/userland/src/camera_isp.hindex 5c3f084..51a8dc2 100644--- a/userland/src/camera_isp.h+++ b/userland/src/camera_isp.h@@ -19,11 +19,18 @@ struct isp_stats {double bright; /* the level of green that 1 % of the cells are above */uint32_t hist[64]; /* green above black, 16 steps a bin */uint32_t cells;+ /*+ * Red over green and blue over green, lens-shaded, each cell's counted once however bright it is:+ * the mean of the cells none of whose colours is near white or near black, and of those of them+ * that are grey or nearly with the white balance as it is.+ */+ double grey[2], neutral[2];+ uint32_t grey_cells, neutral_cells;};/** Lens shading: a gain for each colour at each point of a grid laid over the sensor (corners on its- * corners), between the points bilinear. Off until it is given one (isp_lsc_from_flat()).+ * corners), between the points bilinear. Off until it is given one (isp_lsc_from_flat(), isp_tune()).*/#define LSC_W 17#define LSC_H 13@@ -32,23 +39,64 @@ struct isp_lsc {float gain[3][LSC_H][LSC_W];};+/*+ * A camera's colour tuning, as tools/camtuning.py makes it out of the camera's tuning file of Android's+ * camera stack: colour matrices and lens shading grids, each per range of the scene's light (a lux+ * index, Qualcomm's measure: higher is darker) and within that per range of the light's colour+ * temperature.+ */+#define TUNE_LIGHTS 6+#define TUNE_CCTS 6+struct isp_tune_set {+ uint32_t lights;+ struct {+ float lux_from, lux_to;+ uint32_t ccts;+ float cct_from[TUNE_CCTS], cct_to[TUNE_CCTS];+ const float *v[TUNE_CCTS]; /* a matrix by rows (9), or R, Gr, Gb and B grids (4 x 13 x 17) */+ } light[TUNE_LIGHTS];+};+struct isp_tuning {+ struct isp_tune_set ccm, lsc;+ /*+ * The white balance's reference lights, from the bluest to the reddest: each as the sensor sees a+ * grey under it (red over green, blue over green), and the colour temperature its tables are+ * picked by (tools/camtuning.py). Between them, the line the white balance keeps to.+ */+ uint32_t refs;+ float ref[16][3];+ void *file;+};+struct isp {float wb[3]; /* the white balance's gains, green 1 */float digital; /* more gain, for what AE cannot reach with the sensor's */+ float zoom; /* the picture is the middle 1/zoom of the sensor each way (1: the whole of it) */+ bool turned; /* the picture turned half round (a sensor mounted the other way up) */float ccm[3][3]; /* the colour matrix, camera RGB (white balanced) to linear sRGB; rows sum to 1 */struct isp_lsc lsc;uint8_t gamma[4096]; /* linear (959 above black is 3836) to 8 bits, sRGB's curve */uint32_t asked_lines, asked_gain, asked_frame_lines; /* what AE asked for last */bool asked;uint32_t max_frame_lines; /* the longest frame AE may ask for (0: twice the shortest) */+ struct isp_tuning *tuning; /* isp_load_tuning()'s, or none */};void isp_init(struct isp *isp);-/* the levels, from every 4th cell of every 4th pair of lines */-void isp_measure(const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s);+/*+ * How sharp the picture is in a part of the frame (fractions of it, x0 < x1, y0 < y1): the squares of+ * green's steps between neighbouring blocks of 4x4 cells, over green's level squared -- higher is sharper,+ * the same scene at any exposure the same; and green's mean level there (level, may be NULL).+ */+float isp_sharpness(const struct redwood_camera_frame *f, const uint8_t *raw, float x0, float y0, float x1, float y1, float *level);+/* the levels, from every 4th cell of every 4th pair of lines (with the processor's lens shading and white balance, if isp) */+void isp_measure(const struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, struct isp_stats *s);/* the exposure for the frames to come: true, and the command filled in, when it should change */bool isp_ae(struct isp *isp, const struct redwood_camera_frame *f, const struct isp_stats *s, struct redwood_camera_command *c);-/* the white balance: grey world, a little of the way at a time */+/*+ * The white balance: grey world -- of the nearly grey, when enough is -- kept to the line of the+ * tuning's reference lights when there is one, a little of the way at a time.+ */void isp_awb(struct isp *isp, const struct isp_stats *s);/** A picture of the frame, width x height, of the sensor's middle part with the picture's proportions@@ -72,5 +120,11 @@ void isp_render_full_nv12(const struct isp *isp, const struct redwood_camera_fra*/bool isp_lsc_from_flat(struct isp *isp, const struct redwood_camera_frame *f, const uint8_t *raw, float *min_ratio);void isp_set_ccm(struct isp *isp, const float m[9]);+/* the tuning in PATH (tools/camtuning.py's) in place of the one there was: false, and none, if it is not one */+bool isp_load_tuning(struct isp *isp, const char *path);+/* the colour matrix and lens shading for a scene of this lux index and colour temperature, from the tuning */+void isp_tune(struct isp *isp, float lux_index, float cct);+/* the colour temperature the white balance is at, by the tuning's reference lights (0: there are none) */+float isp_cct(const struct isp *isp);#endifdiff --git a/userland/src/camera_test.c b/userland/src/camera_test.cindex fffed67..42b07f4 100644--- a/userland/src/camera_test.c+++ b/userland/src/camera_test.c@@ -128,7 +128,7 @@ cycle(int frames)failures++;continue;}- isp_measure(f, buffer + f->header_bytes, &s);+ isp_measure(NULL, f, buffer + f->header_bytes, &s);printf("camera_test: the %s camera (sensor %u): %ux%u, stride %u, Bayer %u; the first frame %.0f ms after it was asked ""for; %d frames of %llu in %.2f s (%.1f a second); R %.1f G %.1f B %.1f above black\n", sensor_names[n], f->sensor,f->width, f->height, f->stride, f->bayer, changed_ms, got, last_seq - first_seq + 1, ticks_ms(f->time - first) / 1000,@@ -336,7 +336,7 @@ raw(int frames, bool ae, bool awb, bool keep_raw, bool full, int sensor)got++;first_seq = first_seq ? first_seq : f->sequence;- isp_measure(f, raw, &s);+ isp_measure(&isp, f, raw, &s);if (awb) {isp_awb(&isp, &s);}diff --git a/userland/src/inject_media.c b/userland/src/inject_media.cindex e86393b..e0bd78a 100644--- a/userland/src/inject_media.c+++ b/userland/src/inject_media.c@@ -35,6 +35,7 @@* arrives.*/#include <CoreFoundation/CoreFoundation.h>+#include <dlfcn.h>#include <fcntl.h>#include <mach-o/dyld.h>#include <objc/message.h>@@ -283,6 +284,119 @@ audio_source_start(id self, SEL selector, id *error)return true;}+/*+ * A photograph's quality prioritization, "speed" at the most. Asked for "balanced" (Camera.app's) or+ * "quality", the capture device takes a scene for a low-light one whenever a frame was exposed at+ * least as long as the sensor's still image stabilization threshold+ * (-[BWFigVideoCaptureDevice _isLowLightSceneUsingFrameStatisticsByPortType:]) -- a threshold from+ * the sensor's tuning in CameraSetup.plist, 0 for a sensor it has no tuning for, as ours -- and+ * then asks the stream for a bracket of four frames for SIS fusion (captureType 4, BWSISNode), which+ * this port has nothing to fuse with: the photograph never came. With "speed" it asks for one frame+ * (-_resolveStillImageCaptureTypeFromStatisticsByPortType:..., captureType 1), which the stream+ * gives (camera_device.c, StillImageCaptureNow).+ */+static IMP original_quality_prioritization;++static int+quality_prioritization(id self, SEL selector)+{+ int asked = ((int (*)(id, SEL))original_quality_prioritization)(self, selector);+ static bool said;++ if (asked > 1 && !said) {+ said = true;+ say("redwood: mediaserverd: a photograph's quality prioritization %d taken as 1 (speed): one frame, no SIS bracket\n",+ asked);+ }+ return asked > 1 ? 1 : asked;+}++/*+ * A photograph's noise reduction (BWGNRNode, between the still image coordinator and the photo+ * encoder) is a FigSampleBufferProcessor of the iPhone's, which is handed the picture and never+ * hands one back here: the photograph stopped there, with no error, and the request with it (the+ * shutter's second press asked for nothing more). The picture goes past it to the node's output as+ * it came, as with no noise reduction to do.+ */+static void+gnr_render(id self, SEL selector, void *sample_buffer, id input)+{+ id output = ((id (*)(id, SEL))objc_msgSend)(self, sel_registerName("output"));+ static bool said;++ (void)selector, (void)input;+ if (!said) {+ said = true;+ say("redwood: mediaserverd: a photograph goes past the noise reduction, which has nothing to do it with here\n");+ }+ if (output != nil) {+ ((void (*)(id, SEL, void *))objc_msgSend)(output, sel_registerName("emitSampleBuffer:"), sample_buffer);+ }+}++/*+ * A node's output given no sample buffer at all, which no node takes: the session's audio source+ * (-[BWAudioSourceNode _generateSamples]), whose audio unit never started here (see audio_source_start+ * above), fills the silence by a timer with buffers it fails to make and emits them all the same --+ * NULL. Live Photo's staging node added that to an array (-[BWIrisStagingNode renderSampleBuffer:forInput:])+ * and mediaserverd died of the NSInvalidArgumentException, every half a minute with Live Photo on, the+ * preview black for a second each time. A NULL is not passed on.+ */+static IMP original_emit_sample_buffer;++static void+emit_sample_buffer(id self, SEL selector, void *sample_buffer)+{+ static bool said;++ if (sample_buffer == NULL) {+ if (!said) {+ said = true;+ say("redwood: mediaserverd: a node's output was given no sample buffer (the silent audio source's): not passed on\n");+ }+ return;+ }+ ((void (*)(id, SEL, void *))original_emit_sample_buffer)(self, selector, sample_buffer);+}++/*+ * The photograph's encoder. Camera.app asks for HEIC -- HEVC inside a HEIF file, which CMPhoto makes+ * through VideoToolbox -- and VideoToolbox here has no HEVC encoder but the iPhone's hardware ones+ * (com.apple.videotoolbox.videoencoder.hevc and .hevc.gva, behind its AVE): the photo encoder node+ * failed with kVTCouldNotFindVideoEncoderErr (-12908). Apple's software HEVC codec is in the system+ * all the same, VCPHEVC.videocodec, whose decoder VideoToolbox takes here for every HEIC picture (see+ * the top), and there is an encoder in it, HEVCVideoEncoder_CreateInstance, that iOS never registers:+ * the codec's VCPHEVCRegisterEncoder is an empty function. It is registered here, as the codec+ * registers its decoder (VTRegisterVideoDecoderWithInfo, not hardware accelerated).+ */+static void+register_hevc_encoder(void)+{+ void *codec = dlopen("/System/Library/VideoCodecs/VCPHEVC.videocodec", RTLD_NOW | RTLD_LOCAL);+ void *vt = dlopen("/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox", RTLD_NOW | RTLD_GLOBAL);+ void *create = codec != NULL ? dlsym(codec, "HEVCVideoEncoder_CreateInstance") : NULL;+ int (*register_encoder)(uint32_t, CFDictionaryRef, void *) = vt != NULL ? dlsym(vt, "VTRegisterVideoEncoderWithInfo") : NULL;+ CFStringRef *id_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_EncoderID") : NULL;+ CFStringRef *name_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_EncoderName") : NULL;+ CFStringRef *hardware_key = vt != NULL ? dlsym(vt, "kVTVideoEncoderList_IsHardwareAccelerated") : NULL;+ int status;++ if (create == NULL || register_encoder == NULL || id_key == NULL || name_key == NULL || hardware_key == NULL) {+ say("redwood: mediaserverd: no software HEVC encoder to register (%s)\n", codec == NULL ? "no VCPHEVC.videocodec" :+ create == NULL ? "no HEVCVideoEncoder_CreateInstance" : "VideoToolbox without VTRegisterVideoEncoderWithInfo");+ return;+ }+ const void *keys[] = { *id_key, *name_key, *hardware_key };+ const void *values[] = { CFSTR("com.apple.videotoolbox.videoencoder.hevc.vcp"), CFSTR("HEVC (VCPHEVC, software)"),+ kCFBooleanFalse };+ CFDictionaryRef info = CFDictionaryCreate(NULL, keys, values, 3, &kCFTypeDictionaryKeyCallBacks,+ &kCFTypeDictionaryValueCallBacks);++ status = register_encoder('hvc1', info, create);+ CFRelease(info);+ say("redwood: mediaserverd: the software HEVC encoder of VCPHEVC.videocodec registered for 'hvc1' (%d)\n", status);+}+struct method_spec {const char *selector, *types;IMP imp;@@ -342,6 +456,10 @@ static struct replacement {(IMP)video_stabilization_method, NULL, true, false },{ "FigVideoCaptureConnectionConfiguration", "videoStabilizationType", 0x1a20d8f48UL,(IMP)video_stabilization_type, NULL, true, false },+ { "FigCaptureStillImageSettings", "qualityPrioritization", 0x1a1f533dcUL,+ (IMP)quality_prioritization, &original_quality_prioritization, true, false },+ { "BWGNRNode", "renderSampleBuffer:forInput:", 0x1a1ec83b8UL, (IMP)gnr_render, NULL, true, false },+ { "BWNodeOutput", "emitSampleBuffer:", 0x1a1ea37e0UL, (IMP)emit_sample_buffer, &original_emit_sample_buffer, true, false },};static void@@ -393,6 +511,7 @@ redwood_media_init(void)}if (camera) {say("redwood: mediaserverd: the kernel has a camera (/dev/redwood-camera)\n");+ register_hevc_encoder();}_dyld_register_func_for_add_image(image_added); /* also called for every image already there */}