Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

Camera: the preview runs steadily -- no capture audio, and a camera that survives a restart

vcvkk1c320adfb065
diff --git a/userland/src/camera_device.c b/userland/src/camera_device.c
index 28388c8..2761bd4 100644
--- a/userland/src/camera_device.c
+++ b/userland/src/camera_device.c
@@ -22,9 +22,10 @@
* Everything asked of the device and the stream that is not answered here is said in mediaserverd's
* log ("redwood: camera: ..."), once per property.
*
- * The graph builds, the session runs, and the pictures of the PrimaryScaler output reach Camera.app's
- * preview. NOT FINISHED: a photograph is not taken yet ("StillImageCaptureNow" is not answered), and
- * the session is rebuilt every few seconds, so the preview comes in bursts rather than steadily.
+ * The graph builds, the session runs, and the pictures of the PrimaryScaler output are Camera.app's
+ * preview, steadily. NOT FINISHED: a photograph is not taken yet -- "StillImageCaptureNow" is not
+ * answered and the StillImage output is passed over, it being the whole sensor (4032x3024), which is
+ * not something to make thirty times a second.
*/
#include <CoreFoundation/CoreFoundation.h>
#include <CoreMedia/CoreMedia.h>
@@ -839,7 +840,8 @@ stream_stop(CMBaseObjectRef o)
{
struct stream *s = stream_of(o);
- say("stream: stop");
+ say("stream: stop (%llu frames, %llu pictures given)", (unsigned long long)s->frames,
+ (unsigned long long)s->out[0].delivered);
pthread_mutex_lock(&s->lock);
s->graph_running = false;
pthread_mutex_unlock(&s->lock);
diff --git a/userland/src/inject_media.c b/userland/src/inject_media.c
index 04b4dce..e86393b 100644
--- a/userland/src/inject_media.c
+++ b/userland/src/inject_media.c
@@ -242,6 +242,47 @@ video_stabilization_type(id self, SEL selector)
return 0; /* kFigCaptureVideoStabilizationType_None */
}
+/*
+ * The microphone, which the capture stack has none of here. A capture session's sound comes through
+ * CoreMedia's own capture audio session -- BWAudioSourceNode's start sets its properties and reads
+ * its pull format, all of it through a brokered plug-in of the audio HAL -- and that plug-in is not
+ * on this phone: the node's start comes back with 'nope', the HAL's word for "no such thing".
+ *
+ * Camera.app's session has an audio input in it, because it is a session for photographs and for
+ * video both. The graph starts its source nodes late (startDeferredSourceNodesIfNeeded) and takes
+ * the first error it gets for the whole graph's, so the session reported AVCaptureSessionRuntimeError
+ * to the application, which took the session down and built it again, two seconds later, for ever:
+ * the camera's preview came in bursts of five frames with the sensor powering down in between.
+ *
+ * The node is told that it started. It then produces no sound -- which is all a phone with no
+ * capture audio has to offer -- and the rest of the graph, the preview and the photograph, runs.
+ * Only that one error is swallowed: a microphone that is there and fails for another reason still
+ * takes the session down, as it should. (The kernel's own microphone, adsp=1 mic=4, is a different
+ * path: AVAudioRecorder and Voice Memos record through it, and nothing here touches it.)
+ */
+#define ERROR_NO_SUCH_THING 1852797029 /* 'nope' */
+
+static IMP original_audio_source_start;
+
+static BOOL
+audio_source_start(id self, SEL selector, id *error)
+{
+ BOOL started = ((BOOL (*)(id, SEL, id *))original_audio_source_start)(self, selector, error);
+ id made = error != NULL ? *error : NULL;
+ static bool said;
+
+ if (started || made == NULL || ((long (*)(id, SEL))objc_msgSend)(made, sel_registerName("code")) != ERROR_NO_SUCH_THING) {
+ return started;
+ }
+ *error = NULL; /* (autoreleased by the capture stack: not ours to give back) */
+ if (!said) {
+ said = true;
+ say("redwood: mediaserverd: there is no capture audio on this phone: the session's audio source gives nothing, "
+ "and the camera runs without it\n");
+ }
+ return true;
+}
+
struct method_spec {
const char *selector, *types;
IMP imp;
@@ -296,6 +337,7 @@ static struct replacement {
(IMP)prewarm_portrait_filter, NULL, false, false },
{ "BWFigCaptureDeviceVendor", "sharedCaptureDeviceVendor", 0x1a1ef0d04UL,
(IMP)shared_vendor, &original_shared_vendor, false, false },
+ { "BWAudioSourceNode", "start:", 0x1a1eefa18UL, (IMP)audio_source_start, &original_audio_source_start, true, false },
{ "FigVideoCaptureConnectionConfiguration", "videoStabilizationMethod", 0x1a1e99ef8UL,
(IMP)video_stabilization_method, NULL, true, false },
{ "FigVideoCaptureConnectionConfiguration", "videoStabilizationType", 0x1a20d8f48UL,
diff --git a/xnu/files/bsd/dev/redwood_camera.c b/xnu/files/bsd/dev/redwood_camera.c
index cfe26f8..f3b5703 100644
--- a/xnu/files/bsd/dev/redwood_camera.c
+++ b/xnu/files/bsd/dev/redwood_camera.c
@@ -620,12 +620,31 @@ sensor_ldo(unsigned n, unsigned mv)
return false;
}
+/*
+ * The PM8008i's I2C is bit-banged, on a bus whose lines the phone's own firmware holds as well, and
+ * a transfer now and then goes unanswered -- more often the longer the run, and most often right
+ * after the camera has been switched off and on again. Taken for a fault the first time, the camera
+ * was gone for the rest of the run (give_up: nothing more is written); asked again, the chip
+ * answers. The number of goes that were needed is said at the end of the run.
+ */
+#define PM8008_GOES 4
+
+static uint64_t pm8008_retries;
+
static bool
pm8008_get(uint8_t address, uint16_t reg, uint8_t *values, unsigned count)
{
uint8_t where[2] = { (uint8_t)(reg >> 8), (uint8_t)reg };
- return i2c_transfer(&pmic_bus, address, where, 2, values, count);
+ for (unsigned go = 0; go < PM8008_GOES; go++) {
+ if (i2c_transfer(&pmic_bus, address, where, 2, values, count)) {
+ pm8008_retries += go;
+ return true;
+ }
+ wait_ms(1);
+ }
+ pm8008_retries += PM8008_GOES;
+ return false;
}
/* The registers the PM8008i may be written in: the chip enable, and the set point, mode and enable of the sensor's LDOs. */
@@ -642,13 +661,17 @@ pm8008_put(uint8_t address, uint16_t reg, uint8_t value, bool force)
give_up(allowed ? "a write to the PM8008i after a fault" : "a PM8008i register that is not to be written");
return false;
}
- if (!i2c_transfer(&pmic_bus, address, out, 3, NULL, 0) || !pm8008_get(address, reg, &back, 1) || back != value) {
- say("camera: PM8008i %#04x:%#06x <- %#04x: %s (reads %#04x)\n", address, reg, value, back == 0xff ? "no answer" :
- "does not read back", back);
- give_up("a write to the PM8008i that failed");
- return false;
+ for (unsigned go = 0; go < PM8008_GOES; go++) {
+ if (i2c_transfer(&pmic_bus, address, out, 3, NULL, 0) && pm8008_get(address, reg, &back, 1) && back == value) {
+ pm8008_retries += go;
+ return true;
+ }
+ wait_ms(1);
}
- return true;
+ say("camera: PM8008i %#04x:%#06x <- %#04x: %s (reads %#04x), %u goes\n", address, reg, value, back == 0xff ? "no answer" :
+ "does not read back", back, PM8008_GOES);
+ give_up("a write to the PM8008i that failed");
+ return false;
}
static void
@@ -1221,7 +1244,15 @@ power_down(void)
supply_off(&hw->vana);
supply_off(&hw->vdig);
gdsc_off();
- if (chip_enable_switched) {
+ /*
+ * The PM8008i itself stays as it is between one use of the camera and the next: out of reset and
+ * enabled, with its LDOs off (they are switched above). Put back into reset it does come out
+ * again -- it answers for its revision at 0x08 -- but its regulators are deaf from then on (a
+ * write to 0x09 reads back 0xff), and the camera is gone for the rest of the run. It is put back
+ * as it was found when the kernel leaves (redwood_camera_quiesce), which is where Android gets
+ * the phone back.
+ */
+ if (leaving && chip_enable_switched) {
uint8_t enable = 0;
if (pm8008_get(PM8008_CHIP, MISC_CHIP_ENABLE, &enable, 1) &&
@@ -1232,7 +1263,7 @@ power_down(void)
}
chip_enable_switched = false;
}
- if (pgpio_changed) {
+ if (leaving && pgpio_changed) {
uint8_t now[5];
bool all = pgpio_read_all(now);
@@ -1250,7 +1281,11 @@ power_down(void)
reset_driven = false;
}
powered = false;
- say("camera: powered down\n");
+ if (pm8008_retries != 0) {
+ say("camera: powered down (the PM8008i needed %llu more goes on its I2C)\n", pm8008_retries);
+ } else {
+ say("camera: powered down\n");
+ }
}
/* The PM8008i out of reset (its reset pin as Android drives it) and enabled. */
@@ -1263,7 +1298,9 @@ pm8008_up(void)
give_up("the PM8350C's GPIO 3 cannot be read");
return false;
}
- memcpy(pgpio_found_regs, regs, sizeof(regs));
+ if (!pgpio_changed) { /* (as Android left it, which is what the run's end puts back) */
+ memcpy(pgpio_found_regs, regs, sizeof(regs));
+ }
if (!pgpio_active(regs)) {
uint8_t want[5] = { (uint8_t)(regs[0] & ~7), (uint8_t)((regs[1] & ~7) | 5), (uint8_t)((regs[2] & ~3) | 1), 0x80,
(uint8_t)(regs[4] | 0x80) };
@@ -1329,10 +1366,28 @@ read_chip_id(void)
unsigned out = ctab->address_bytes == 2 ? 2 : 1, in = ctab->data_bytes >= 1 && ctab->data_bytes <= 4 ? ctab->data_bytes : 2;
uint32_t chip = 0;
- if (!i2c_transfer(&cci_bus, sensor_address(), out == 2 ? where : where + 1, out, id, in)) {
+ /*
+ * The first word asked of the sensor after its supplies have come up. Powered up again within a
+ * run -- the capture graph stops and starts the stream as the application reconfigures its
+ * session -- it is not always ready by the time the power sequence's own waits are over, and one
+ * unanswered transfer used to end the camera for the rest of the run. It is asked again, for as
+ * long as the sensor's own data sheet would have it waiting.
+ */
+ unsigned go;
+
+ for (go = 0; go < 20; go++) {
+ if (i2c_transfer(&cci_bus, sensor_address(), out == 2 ? where : where + 1, out, id, in)) {
+ break;
+ }
+ wait_ms(2);
+ }
+ if (go == 20) {
say("camera: the %s sensor does not answer at %#04x\n", hw->name, sensor_address());
return false;
}
+ if (go != 0) {
+ say("camera: the %s sensor answered at %#04x after %u more goes (%u ms)\n", hw->name, sensor_address(), go, go * 2);
+ }
for (unsigned i = 0; i < in; i++) {
chip = chip << 8 | id[i];
}