Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
GPU path: no frame shown half drawn; Mesa rebuilt when its patches change
vcvkk69f31e34d422
diff --git a/prebuilt/SHA256SUMS b/prebuilt/SHA256SUMSindex 7771edd..054d885 100644--- a/prebuilt/SHA256SUMS+++ b/prebuilt/SHA256SUMS@@ -1,6 +1,6 @@-Built 2026-10-13 02:27 from this tree.+Built 2026-10-13 05:22 from this tree.-165b31a6f71a22a06e7d7d6a08573f9a031e9353c68394a40c36f2ad9a60fa22 prebuilt/mach.development.bcm2837+8d9b6f3faec637b1f6e9dce8023db3cb150ad2a311067d575160bdcf5dec1a9c prebuilt/mach.development.bcm28371332a598d2dec00a1d8e930044e4f00a12ad44c6e5e50eecf4b7fe7fae803bf2 prebuilt/loader.efid2a35a4a1d6658ffb3696914f727661b52c9333de8a3af8a31642b8023ce03ac prebuilt/userland/child1a413befd511384052e1962c1394adce20f718e1f2657ce63ab09828ebc4bd1b prebuilt/userland/cpu_testdiff --git a/prebuilt/mach.development.bcm2837 b/prebuilt/mach.development.bcm2837index 32e4c09..b8c0094 100755Binary files a/prebuilt/mach.development.bcm2837 and b/prebuilt/mach.development.bcm2837 differdiff --git a/tools/mk-mesa.sh b/tools/mk-mesa.shindex 8744dae..42a08b8 100755--- a/tools/mk-mesa.sh+++ b/tools/mk-mesa.sh@@ -31,11 +31,17 @@ CC=$ROOT/xnu-deps/host/bin/xnu-clangmkdir -p "$B/out"cd "$B"-if [ ! -d mesa-$VERSION ]; then+# The sources are the archive's with tools/mesa/*.patch on top. When the patches change, a tree that was+# patched with the earlier ones is of no use (and builds without a word into a library that does not match+# the rest): the sources are taken from the archive again and everything is built anew, about a minute.+PATCHES=$(cat "$ROOT"/tools/mesa/*.patch | sha256sum | cut -d' ' -f1)+if [ ! -d mesa-$VERSION ] || [ "$(cat mesa-$VERSION/.redwood-patches 2>/dev/null)" != "$PATCHES" ]; then[ -f mesa-$VERSION.tar.xz ] || curl -sL --fail -o mesa-$VERSION.tar.xz https://archive.mesa3d.org/mesa-$VERSION.tar.xzecho "$SHA256 mesa-$VERSION.tar.xz" | sha256sum -c --quiet+ rm -rf mesa-$VERSION build-*tar xJf mesa-$VERSION.tar.xzfor p in "$ROOT"/tools/mesa/*.patch; do patch -s --no-backup-if-mismatch -d mesa-$VERSION -p1 < "$p"; done+ echo "$PATCHES" > mesa-$VERSION/.redwood-patchesfi# ours, kept in the repository: always the current onesrm -rf mesa-$VERSION/src/redwood-drmdiff --git a/xnu/files/bsd/dev/redwood_adreno.c b/xnu/files/bsd/dev/redwood_adreno.cindex 9a6a54d..061f87b 100644--- a/xnu/files/bsd/dev/redwood_adreno.c+++ b/xnu/files/bsd/dev/redwood_adreno.c@@ -65,8 +65,8 @@* with, through the service RedwoodGPU (see "the GPU kept on" below); it is* given back when the run ends (adreno_dirty=1: left as a panic leaves it)* adreno_zap=0 step 19 has been done since the phone started: not again- * adreno_speed=N step 17 asks the GMU for the 3D core's Nth speed instead of the first:- * 1 315 MHz, 2 450 MHz, 3 550 MHz (the most this phone's GPU is rated for)+ * adreno_speed=N step 17 asks the GMU for the 3D core's Nth speed: 1 315 MHz, 2 450 MHz,+ * 3 550 MHz (the default: the most this phone's GPU is rated for)* adreno_blind=1 step 20 starts the command processor with its microcode nowhere, and* adreno_stream1=N step 18 leaves the core's second stream out (0) or gives it an empty* context of its own (2): what the SMMU then records shows what the core@@ -75,7 +75,7 @@* the machine is still alive, and what was written last* adreno_return=1 when all is over, go back to the loader instead of starting the system* display_probe=1 before any step: read what the firmware left the display controller- * doing (display_probe() below; nothing is written)+ * doing (display_probe() below; one clock is turned on, nothing else written)* The system does not go on starting until all of this is over, so that the end of the log is* ours; the touch screen driver's start would talk across it (leave touchscreen=1 out).*@@ -587,6 +587,18 @@ step_cx_power(void)return false;}printf("adreno: %u: CX is on\n", step);+ if (rd(gpucc.base + GPUCC_GX_GDSCR) & GDSC_POWER_ON) {+ /*+ * The 3D core kept its power while the rest of the block lost it: an ending that went+ * wrong (a GMU that did not answer, before gpu_block_stop() switched the core off+ * itself). Such a core is where it was, in the middle of things, and does not take the+ * steps' commands (seen: step 21); switching it off from here has hung the whole phone+ * (seen once). Left alone: no GPU until the phone has been restarted.+ */+ printf("adreno: %u: THE 3D CORE HAS POWER ALREADY, left on by an earlier kernel: not usable until the phone is "+ "restarted\n", step);+ return false;+ }return true;}@@ -2305,7 +2317,7 @@ step_core_power(void)} speeds[] = { { 0, 0 }, { 315000, 64 }, { 450000, 128 }, { 550000, 192 } };static struct hfi_perf_table perf;static struct hfi_bw_table bw;- uint32_t core_start[2] = { 0, 0 }, start[1] = { 0 }, vote[4] = { 0, 1, 1, HFI_NO_INDEX }, granted = 0, speed = 1;+ uint32_t core_start[2] = { 0, 0 }, start[1] = { 0 }, vote[4] = { 0, 1, 3, HFI_NO_INDEX }, granted = 0, speed = 0;uint16_t gfx[16], cx[16], mx[16];unsigned gfx_levels, cx_levels, mx_levels;@@ -2954,22 +2966,32 @@ step_frame(void)* core given back, the GMU told to drop its votes ("slumber") and stopped, its RPMh channel put* to sleep; then the processor into reset and its stream sent nowhere.*/-/* What step 4 does when it finds the GPU block running: see there. Nothing here has been set up by this kernel. */+/*+ * The 3D core's own power switch is the GMU's to operate. Where the GMU is gone and has left it+ * on (it is in reset; it did not answer when told to slumber), the switch is turned off from+ * here (vendor's a6xx_gmu_suspend): a core that keeps its power while the rest of the block+ * loses it is found by the next start as the last one left it, in the middle of things.+ */static void-stale_stop(void)+gx_off(void){- uint32_t gx;+ uint32_t gx = get(&gpucc, "gx_gdscr", GPUCC_GX_GDSCR);- need(&gmu);- need(&rscc);- put(&gmu, "cm3_sysreset", GMU_CM3_SYSRESET, 1);- gx = get(&gpucc, "gx_gdscr", GPUCC_GX_GDSCR);if (gx & GDSC_POWER_ON) {- /* the 3D core's switch is the GMU's to operate, and the GMU is gone: vendor's a6xx_gmu_suspend */put(&gpucc, "gx_gdscr", GPUCC_GX_GDSCR, gx | GDSC_SW_COLLAPSE);wait_for(&gpucc, GPUCC_GX_GDSCR, GDSC_POWER_ON, 0, 100);get(&gpucc, "gx_gdscr", GPUCC_GX_GDSCR);}+}++/* What step 4 does when it finds the GPU block running: see there. Nothing here has been set up by this kernel. */+static void+stale_stop(void)+{+ need(&gmu);+ need(&rscc);+ put(&gmu, "cm3_sysreset", GMU_CM3_SYSRESET, 1);+ gx_off();if (rd(rscc.base + RSCC_SEQ_MEM_0_DRV0) == 0xeaaae5a0 && !(rd(rscc.base + RSCC_RSC_STATUS0_DRV0) & RSCC_ASLEEP)) {put(&gmu, "rscc_control_req", GMU_RSCC_CONTROL_REQ, 1);wait_for(&rscc, RSCC_RSC_STATUS0_DRV0, RSCC_ASLEEP, RSCC_ASLEEP, 100);@@ -2991,6 +3013,38 @@ stale_stop(void)get(&gpucc, "cx_hw_ctrl", GPUCC_CX_HW_CTRL);}+/*+ * The GPU block's bus interface is told to let what is under way finish and to start nothing+ * more (Linux's a6xx_bus_clear_pending_transactions). (Without it, a core whose SMMU context is+ * taken away under a draw leaves the SMMU waiting for good, and the CX half cannot be switched+ * off any more: only a restart of the phone gets the GPU back.)+ * `for_good`: the 3D core may be in the middle of a frame and nobody will tell it to stop (a+ * panic; a core that does not come to rest): its own side is halted too, and all of it stays+ * halted, so that nothing of it reaches memory after this, whatever becomes of its tables.+ * Otherwise the halt is lifted again at once, as Linux does: the GMU reaches its memory through+ * the same interface, and one that cannot read its messages does not answer "slumber".+ */+static void+core_bus_halt(bool for_good)+{+ if (for_good) {+ wr(gpu.base + RBBM_GBIF_HALT, 1);+ for (int i = 0; i < 100000 && !(rd(gpu.base + RBBM_GBIF_HALT_ACK) & 1); i++) {+ }+ }+ wr(gbif.base + GBIF_HALT, 1);+ for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 1); i++) {+ }+ wr(gbif.base + GBIF_HALT, 2);+ for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 2); i++) {+ }+ if (!for_good) {+ wr(gbif.base + GBIF_HALT, 0);+ }+}++static bool core_stuck; /* the 3D core has not finished what it was given, and is not waited for */+static voidgpu_block_stop(void){@@ -2999,6 +3053,16 @@ gpu_block_stop(void)step = 0;newest_step = ~0u;+ if (core_runs && core_is_on) {+ /* (what the core looks like when it is given back: a GMU that then does not answer may have its reasons here) */+ printf("adreno: %u: the 3D core before it is stopped: status %#x, interrupts %#x, ring read up to %u of %u; the GMU "+ "sees it busy: %#x\n", step, rd(gpu.base + RBBM_STATUS), rd(gpu.base + RBBM_INT_0_STATUS),+ rd(gpu.base + CP_RB_RPTR), ring_at, rd(gmu.base + GMU_AO_GPU_CX_BUSY_STATUS));+ core_bus_halt(core_stuck);+ printf("adreno: %u: the GPU block's bus has been halted%s (rbbm_gbif_halt_ack %#x, gbif halt_ack %#x)\n", step,+ core_stuck ? ", the 3D core's side too, and stays so" : " and let go again", rd(gpu.base + RBBM_GBIF_HALT_ACK),+ rd(gbif.base + GBIF_HALT_ACK));+ }if (core_is_on) {put(&gmu, "host2gmu_intr_set", GMU_HOST2GMU_INTR_SET, 1u << 31);core_is_on = false;@@ -3015,6 +3079,7 @@ gpu_block_stop(void)if (gmu_runs) {put(&gmu, "cm3_sysreset", GMU_CM3_SYSRESET, 1);gmu_runs = false;+ gx_off(); /* (a GMU that has slumbered has done it itself) */}/* vendor's a6xx_rscc_sleep_sequence: the GMU's votes in RPMh are dropped, by the always-on sequencer */if (rsc_awake && rd(rscc.base + RSCC_SEQ_MEM_0_DRV0) == 0xeaaae5a0) {@@ -3079,21 +3144,7 @@ redwood_adreno_quiesce(void)gmu_runs = false;}if (core_runs) {- /*- * The 3D core may be in the middle of a frame. Its bus interface is told to let what- * is under way finish and to start nothing more (Linux's- * a6xx_bus_clear_pending_transactions), and stays so: nothing of it reaches memory- * after this, whatever becomes of its tables.- */- wr(gpu.base + RBBM_GBIF_HALT, 1);- for (int i = 0; i < 100000 && !(rd(gpu.base + RBBM_GBIF_HALT_ACK) & 1); i++) {- }- wr(gbif.base + GBIF_HALT, 1);- for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 1); i++) {- }- wr(gbif.base + GBIF_HALT, 2);- for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 2); i++) {- }+ core_bus_halt(true);core_runs = false;}if (gmu_space_is_on) {@@ -3143,6 +3194,7 @@ static uint32_t gpu_fence; /* the number of the last submission */static uint32_t gpu_timed; /* ... of the last whose times have been added up */static uint64_t gpu_ticks; /* how long the core has worked on the submissions added up */static uint32_t gpu_timed_count;+static uint32_t gpu_timed_by[7]; /* ... how many of them under 0.1, 0.25, 0.5, 1, 2, 4 ms, and longer */static uint32_t gpu_exact = 1; /* boot-arg gpu_exact=0: the core does not wait to be idle before it notes the time */static struct {uint64_t start, size;@@ -3166,9 +3218,22 @@ voidredwood_gpu_stop(void){if (gpu_kept) {- gpu_kept = false;+ uint32_t waited = 0;++ gpu_kept = false; /* (nothing more is taken) */quiet = false;- printf("adreno: the run is about to end: the GPU is given back\n");+ /*+ * What the core has been given, it finishes first: told to slumber in the middle of a+ * frame, the GMU may not answer. Half a second at the most; then its bus is halted+ * for good, and its power switched off from here.+ */+ while (!redwood_gpu_done(gpu_fence) && waited < 5000) {+ sleep_us(100);+ waited++;+ }+ core_stuck = !redwood_gpu_done(gpu_fence);+ printf("adreno: the run is about to end: the GPU is given back (%s after %u us)\n",+ core_stuck ? "STILL AT WORK" : "it had finished what it was given", 100 * waited);gpu_block_stop();}}@@ -3352,19 +3417,32 @@ gpu_times(void)uint32_t slot = ++gpu_timed % GPU_TIMES;if (gpu_timed != 0 && times[2 * slot + 1] > times[2 * slot]) {- gpu_ticks += times[2 * slot + 1] - times[2 * slot];+ static const uint32_t under[6] = { 1920, 4800, 9600, 19200, 38400, 76800 }; /* (19.2 ticks a microsecond) */+ uint64_t ticks = times[2 * slot + 1] - times[2 * slot];+ unsigned by = 0;++ while (by < 6 && ticks >= under[by]) {+ by++;+ }+ gpu_timed_by[by]++;+ gpu_ticks += ticks;gpu_timed_count++;}}}-/* How long the 3D core has worked since this was last asked, and on how many submissions. */+/*+ * How long the 3D core has worked since this was last asked, on how many submissions, and how+ * many of those took it under 0.1, 0.25, 0.5, 1, 2, 4 milliseconds and longer.+ */void-redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions)+redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions, uint32_t by[7]){gpu_times();*microseconds = gpu_ticks * 10 / 192;*submissions = gpu_timed_count;+ memcpy(by, gpu_timed_by, sizeof(gpu_timed_by));+ bzero(gpu_timed_by, sizeof(gpu_timed_by));gpu_ticks = 0;gpu_timed_count = 0;}@@ -3448,23 +3526,58 @@ static bool(*const steps[])(void) = {};#define STEPS ((uint32_t)(sizeof(steps) / sizeof(steps[0])))+/* ---- beside the steps: how warm the GPU is ------------------------------------------------------------------ */+/*- * Called while the system starts, before there is a root file system, and the start waits: for- * as long as the steps and the watch after them last, nothing else writes to the log, so that- * its end -- all that the loader's notes keep -- is about what is being done here.+ * The SoC's temperature sensors report by themselves, in a block that is always on (dt: tsens1,+ * whose readings are at 0xc265000; Linux's tsens-v2.c): a status word a sensor, the last reading+ * in its low 12 bits, signed, in tenths of a degree, and bit 21 when there is one. Sensors 1 and+ * 2 sit at the GPU (dt: gpuss-0, gpuss-1). Read, never written: once a minute while the GPU is+ * kept, for the line beside the kernel's line about memory (osfmk/arm64/loader_return.c). A+ * sensor caught between two readings has none for a moment and is asked again; one that still+ * has none is put down as -273.0.*/+static struct block tsens = { "tsens1", 0x0c265000, 0x200 };+#define TSENS_STATUS(sensor) (0xa0 + 4 * (sensor))+#define TSENS_VALID (1u << 21)++void+redwood_adreno_minute(void)+{+ int32_t tenths[2];++ if (!gpu_kept) {+ return;+ }+ need(&tsens);+ for (unsigned i = 0; i < 2; i++) {+ tenths[i] = -2730;+ for (unsigned again = 0; again < 16 && tenths[i] == -2730; again++) {+ uint32_t status = rd(tsens.base + TSENS_STATUS(1 + i));++ if (status & TSENS_VALID) {+ tenths[i] = (int32_t)(status << 20) >> 20;+ }+ }+ }+ printf("adreno: the GPU's sensors read %d.%d and %d.%d degrees; its clock is set to %u MHz\n", tenths[0] / 10,+ tenths[0] % 10 < 0 ? -(tenths[0] % 10) : tenths[0] % 10, tenths[1] / 10,+ tenths[1] % 10 < 0 ? -(tenths[1] % 10) : tenths[1] % 10, core_khz / 1000);+}+/* ---- beside the steps: what the firmware left the display doing (boot-arg display_probe=1) ---------------- *//** The panel is a DSI video panel (dt: qcom,mdss_dsi_m20_36_02_0a_dsc_vid: 1080 x 2400 at 120* frames a second, 2432 lines with the porches; or at 60 with a longer vertical front porch):* the display controller sends it the framebuffer over and over at the rate of its timing- * registers, which nobody has touched since the firmware. This reads them, and nothing is- * written. Each read is behind a check that what it reads has its clock: the bus clock of the- * display's clock controller (by GCC, which answers always), in that controller the display- * block's power switch and bus clock, then the timing of the interface the panel hangs on- * (Linux's dpu_hw_intf.c has the registers' meaning). Every read is announced and waited for- * (adreno_pause), as in the newest step.+ * registers, which nobody has touched since the firmware. This reads them. Each read is behind+ * a check that what it reads has its clock: the bus clock of the display's clock controller (by+ * GCC, which answers always; the firmware leaves it off, Linux keeps it on always, and so it is+ * turned on here: the one write), in that controller the display block's power switch and bus+ * clock, then the timing of the interface the panel hangs on (Linux's dpu_hw_intf.c has the+ * registers' meaning). Every access is announced and waited for (adreno_pause), as in the+ * newest step.*/static struct block dispcc = { "dispcc", 0x0af00000, 0x2000 }; /* dt: clock-controller@af00000 */static struct block disp_intf = { "intf1", 0x0ae36000, 0x300 }; /* dt: mdss_dsi0's mdp_intf_base */@@ -3492,8 +3605,13 @@ display_probe(void)fine = 1;announce("the display: has its clock controller its bus clock (gcc 0x127004; GCC answers always)");if (get(&gcc, "disp_ahb_cbcr", GCC_DISP_AHB_CBCR) & CBCR_OFF) {- printf("adreno: 0: the display's clock controller has no bus clock: nothing more is read\n");- goto out;+ announce("turn the bus clock of the display's clock controller on (gcc 0x127004 bit 0: THE ONE WRITE)");+ put(&gcc, "disp_ahb_cbcr", GCC_DISP_AHB_CBCR, rd(gcc.base + GCC_DISP_AHB_CBCR) | CBCR_ENABLE);+ sleep_ms(10);+ if (get(&gcc, "disp_ahb_cbcr", GCC_DISP_AHB_CBCR) & CBCR_OFF) {+ printf("adreno: 0: the display's clock controller has no bus clock, and does not get one: nothing more is read\n");+ goto out;+ }}announce("the display block's power switch and clocks, in its clock controller (0xaf00000: NOT READ BEFORE)");if (!(get(&dispcc, "mdss_core_gdscr", DISPCC_MDSS_CORE_GDSCR) & GDSC_POWER_ON) ||@@ -3528,6 +3646,11 @@ out:newest_step = 0;}+/*+ * Called while the system starts, before there is a root file system, and the start waits: for+ * as long as the steps and the watch after them last, nothing else writes to the log, so that+ * its end -- all that the loader's notes keep -- is about what is being done here.+ */voidredwood_adreno_init(void){diff --git a/xnu/files/bsd/sys/redwood_input.h b/xnu/files/bsd/sys/redwood_input.hindex 42cbdc1..e1babac 100644--- a/xnu/files/bsd/sys/redwood_input.h+++ b/xnu/files/bsd/sys/redwood_input.h@@ -45,6 +45,7 @@ void redwood_cpufreq_init(void); /* boot-arg cpufreq=1: the phone's procevoid redwood_adreno_init(void); /* boot-arg adreno=N: the first N steps of waking the phone's GPU */void redwood_adreno_quiesce(void); /* on the way back to the loader: the GPU block stopped, out of our memory */void redwood_adreno_stop(void); /* before that, where the ending is no panic's: the GPU given back in order */+void redwood_adreno_minute(void); /* once a minute: a line about how warm the GPU is, if it is in use *//** The GPU kept on (boot-arg adreno_keep=1), for iokit/Kernel/RedwoodGPU.cpp: see "the GPU kept on"@@ -61,7 +62,7 @@ void redwood_gpu_to_memory(void *start, uint64_t size);uint32_t redwood_gpu_submit(const uint64_t *addresses, const uint32_t *words, uint32_t count);bool redwood_gpu_done(uint32_t fence);uint32_t redwood_gpu_last(void);-void redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions);+void redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions, uint32_t by[7]);void redwood_gpu_report(const char *why);void redwood_gpu_stop(void);#endifdiff --git a/xnu/files/iokit/Kernel/MiniIOSurface.cpp b/xnu/files/iokit/Kernel/MiniIOSurface.cppindex efaafbb..cfa201e 100644--- a/xnu/files/iokit/Kernel/MiniIOSurface.cpp+++ b/xnu/files/iokit/Kernel/MiniIOSurface.cpp@@ -82,6 +82,7 @@ static MiniIOSurface * gSurfaceTable[MINISURF_TABLE_SIZE]; /* index = surfastatic uint32_t gSurfaceNext = 1;static uint32_t gSurfaceCount;static uint64_t gSurfaceBytes;+static uint32_t gSurfaceUnheld; /* surfaces around client memory that could not be made (said a few times) */static uint64_t gSurfaceLogged[2]; /* selectors already complained about *//* Boot-arg surftrace=N: log the first N surfaces created (default 24). */static TUNABLE(uint32_t, gSurfaceTrace, "surftrace", 24);@@ -95,6 +96,12 @@ voidMiniIOSurface::free(void){OSSafeReleaseNULL(values);+ OSSafeReleaseNULL(clientMap);+ if (client != NULL) {+ client->complete();+ client->release();+ client = NULL;+ }OSSafeReleaseNULL(memory);OSObject::free();}@@ -274,9 +281,10 @@ surfaceCreate(OSDictionary * props, IOReturn * error)}/** A surface around memory the client has already: CoreAnimation's OpenGL ES renderer makes- * them around pictures it has in memory (an application's icon), to hand them to the GPU as- * they are. Here the surface is one of our own with a copy of that memory in it, made now:- * good for a picture that does not change, which is what these are.+ * them around pictures it has in memory (an application's icon, a layer's contents as the+ * application has drawn them), to hand them to the GPU as they are. The surface's pixels+ * are that memory itself, not a copy: the picture goes on being drawn into after the+ * surface is made, and whoever looks at the surface is to see what is there now.*/wrapped = OSDynamicCast(OSNumber, props->getObject("IOSurfaceAddress"));@@ -387,10 +395,15 @@ surfaceCreate(OSDictionary * props, IOReturn * error)s->release();return NULL;}- alloc = alignUp(alloc, (uint32_t) PAGE_SIZE);+ if (wrapped == NULL) {+ alloc = alignUp(alloc, (uint32_t) PAGE_SIZE);+ } else {+ /* (the memory is as big as its owner says, and not a byte of ours is added to it) */+ alloc = dictNumber(props, "IOSurfaceAllocSize", (uint32_t) needed);+ }s->memory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,- kIODirectionInOut | kIOMemoryKernelUserShared, (vm_size_t) PAGE_SIZE + alloc, PAGE_SIZE);+ kIODirectionInOut | kIOMemoryKernelUserShared, (vm_size_t) PAGE_SIZE + (wrapped ? 0 : alloc), PAGE_SIZE);if (s->memory == NULL) {IOLog("MiniIOSurface: no memory for %u bytes (%u surfaces, %llu MB in use)\n", alloc, gSurfaceCount,gSurfaceBytes >> 20);@@ -400,15 +413,34 @@ surfaceCreate(OSDictionary * props, IOReturn * error)}s->kbase = (uint8_t *) s->memory->getBytesNoCopy();s->values = OSDictionary::withCapacity(4);+ s->kpixels = s->kbase + PAGE_SIZE;if (wrapped != NULL) {- uint32_t given = dictNumber(props, "IOSurfaceAllocSize", alloc);-- if (copyin((user_addr_t) wrapped->unsigned64BitValue(), s->kbase + PAGE_SIZE, given < alloc ? given : alloc) != 0) {- IOLog("MiniIOSurface: the memory to make a surface around cannot be read\n");+ s->clientTask = current_task();+ s->clientAddress = wrapped->unsigned64BitValue();+ IOReturn held = kIOReturnNoMemory;++ /* (held for writing if it can be written, which a picture that is only shown need not be) */+ for (unsigned attempt = 0; attempt < 2 && held != kIOReturnSuccess; attempt++) {+ OSSafeReleaseNULL(s->client);+ s->client = IOMemoryDescriptor::withAddressRange(s->clientAddress, alloc,+ attempt == 0 ? kIODirectionInOut : kIODirectionOut, s->clientTask);+ held = s->client != NULL ? s->client->prepare() : kIOReturnNoMemory;+ }+ if (held == kIOReturnSuccess) {+ s->clientMap = s->client->createMappingInTask(kernel_task, 0, kIOMapAnywhere | kIOMapReadOnly);+ } else {+ OSSafeReleaseNULL(s->client);+ }+ if (s->clientMap == NULL) {+ if (gSurfaceUnheld++ < 8) {+ IOLog("MiniIOSurface: the memory to make a surface around (%u bytes at %#llx) cannot be %s (%#x)\n", alloc,+ s->clientAddress, held == kIOReturnSuccess ? "seen from here" : "held", held);+ }s->release();*error = kIOReturnVMError;return NULL;}+ s->kpixels = (uint8_t *) s->clientMap->getVirtualAddress();}info->allocSize = alloc;@@ -619,6 +651,7 @@ MiniIOSurfaceStart(IOService * provider)struct minisurf_client_ref {MiniIOSurface * surface; /* held */IOMemoryMap * map; /* the whole of it in the client's address space */+ IOMemoryMap * pixelMap; /* ... the pixels, where those are another client's memory */uint32_t use[2]; /* what this client added to the use counts */bool detached; /* the client let go of it and will release shortly */};@@ -770,6 +803,9 @@ IOSurfaceRootUserClient::addRef(MiniIOSurface * surface)bzero(ref, sizeof(*ref));ref->surface = surface;ref->map = map;+ if (surface->client != NULL && surface->clientTask != fTask) {+ ref->pixelMap = surface->client->createMappingInTask(fTask, 0, kIOMapAnywhere);+ }surfaceHold(surface);return ref;}@@ -784,6 +820,7 @@ IOSurfaceRootUserClient::dropRef(struct minisurf_client_ref * ref)}s->shared()->useCount = (uint64_t) s->useCount[1] << 32 | s->useCount[0];OSSafeReleaseNULL(ref->map);+ OSSafeReleaseNULL(ref->pixelMap);*ref = fRefs[--fRefCount];surfaceUnhold(s);}@@ -812,7 +849,8 @@ IOSurfaceRootUserClient::returnSurface(struct minisurf_client_ref * ref, bool knmemcpy(r + 1, s->planes, size - sizeof(*r));r->shared = address;r->aux = address + MINISURF_AUX_OFFSET;- r->base = known ? ~0ULL : address + PAGE_SIZE;+ r->base = known ? ~0ULL : s->client == NULL ? address + PAGE_SIZE :+ ref->pixelMap != NULL ? ref->pixelMap->getAddress() : s->clientAddress;args->structureOutputSize = size;return kIOReturnSuccess;}diff --git a/xnu/files/iokit/Kernel/MiniIOSurface.h b/xnu/files/iokit/Kernel/MiniIOSurface.hindex 79890ac..2109fbb 100644--- a/xnu/files/iokit/Kernel/MiniIOSurface.h+++ b/xnu/files/iokit/Kernel/MiniIOSurface.h@@ -107,6 +107,16 @@ public:uint32_t holders; /* clients, send rights, kernel users */IOBufferMemoryDescriptor * memory; /* one page of shared state, then the pixels */uint8_t * kbase; /* the same, in the kernel */+ /*+ * ... unless the surface was made around memory its maker has already (IOSurfaceAddress):+ * then the pixels are that memory, held in place and seen from the kernel as well, and+ * `memory` is the page of shared state alone.+ */+ IOMemoryDescriptor * client;+ IOMemoryMap * clientMap;+ task_t clientTask; /* whose memory it is (not held: only compared) */+ mach_vm_address_t clientAddress; /* ... and where it is there */+ uint8_t * kpixels; /* where the pixel memory starts in the kernel, either way */struct minisurf_result info; /* addresses not filled in */struct minisurf_plane planes[MINISURF_MAX_PLANES];OSDictionary * values;@@ -134,7 +144,7 @@ public:uint8_t *pixels(void) const{- return kbase + PAGE_SIZE + info.offset;+ return kpixels + info.offset;}void updateValuesState(void);};diff --git a/xnu/files/iokit/Kernel/RedwoodGPU.cpp b/xnu/files/iokit/Kernel/RedwoodGPU.cppindex e782030..a36d526 100644--- a/xnu/files/iokit/Kernel/RedwoodGPU.cpp+++ b/xnu/files/iokit/Kernel/RedwoodGPU.cpp@@ -55,7 +55,7 @@ void redwood_gpu_to_memory(void * start, uint64_t size);uint32_t redwood_gpu_submit(const uint64_t * addresses, const uint32_t * words, uint32_t count);bool redwood_gpu_done(uint32_t fence);uint32_t redwood_gpu_last(void);-void redwood_gpu_worked(uint64_t * microseconds, uint32_t * submissions);+void redwood_gpu_worked(uint64_t * microseconds, uint32_t * submissions, uint32_t by[7]);void redwood_gpu_report(const char * why);void redwood_gpu_stop(void);@@ -140,7 +140,7 @@ struct drm_gem_close {};struct redwood_surface {uint32_t surface; /* the IOSurface's ID */- uint32_t handle; /* back: the buffer, */+ uint32_t handle; /* kRedwoodDescribe: no buffer is wanted; else, back: the buffer, */uint64_t size; /* how big it is, */uint32_t offset, rowBytes; /* where the pixels start in it and how long a row of them is, */uint32_t width, height, pixelFormat, planes; /* and what they are */@@ -150,6 +150,8 @@ struct redwood_show {uint32_t width, height, rowBytes, pad;};+#define kRedwoodDescribe 0xffffffffu+#define MSM_INFO_GET_OFFSET 0#define MSM_INFO_GET_IOVA 1#define MSM_INFO_SET_NAME 2@@ -165,6 +167,7 @@ struct redwood_show {#define GPU_SUBMIT_BOS 8192#define GPU_PATIENCE_MS 5000 /* for a buffer or a submission that somebody waits for without a limit */#define GPU_ASK_AGAIN_US 200 /* while waiting for the core: how long between two looks at how far it is */+#define GPU_PUSH_AGAIN_MS 4 /* a surface is pushed out of the cache again for a submission this long after the last */static IOLock * gGPULock; /* around everything that is the GPU's (redwood_gpu_...) */static bool gGPUReported;@@ -174,7 +177,7 @@ static bool gGPUCached; /* boot-arg gpu_cached=1 */static struct {uint64_t since, flushing, waiting, taking; /* times of the processor's clock */uint64_t flushed; /* bytes sent out of the cache */- uint32_t submissions, commands, buffers, surfaces, waits;+ uint32_t submissions, commands, buffers, surfaces, waits, over;} gGPUStat;/* Wait for the 3D core to be done with submission `fence`; with the lock held, which is let go of while waiting. */@@ -192,8 +195,8 @@ gpuWait(uint32_t fence, uint64_t milliseconds)}return true;}- if (mach_absolute_time() - began >= patience) {- break;+ if (mach_absolute_time() - began >= patience || gGPUReported) {+ break; /* (a GPU that has stopped answering is not waited for again) */}IOLockUnlock(gGPULock);delay_for_interval(GPU_ASK_AGAIN_US, NSEC_PER_USEC);@@ -221,6 +224,26 @@ millisecondsUntil(const struct msm_timespec * when)return (uint64_t) left / NSEC_PER_MSEC > GPU_PATIENCE_MS ? GPU_PATIENCE_MS : (uint64_t) left / NSEC_PER_MSEC;}+/*+ * A wait that was to end at a time that had passed when it was asked for, and the GPU was not+ * done: whoever asked goes on as if it were (a picture is shown half drawn). That is what comes+ * of a caller whose clock is not this one, so the first few are told.+ */+static void+waitOver(const struct msm_timespec * when, uint32_t fence)+{+ static uint32_t told;+ uint64_t now;++ gGPUStat.over++;+ if (told < 4) {+ told++;+ absolutetime_to_nanoseconds(mach_continuous_time(), &now);+ IOLog("RedwoodGPU: a wait for submission %u, which is not done, until %lld.%09lld s, asked for at %llu.%09llu s: "+ "over before it began\n", fence, when->seconds, when->nanoseconds, now / NSEC_PER_SEC, now % NSEC_PER_SEC);+ }+}+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */class RedwoodGPU : public IOService@@ -285,7 +308,8 @@ struct gpu_buffer {uint64_t address; /* the GPU's for it */uint64_t size;uint32_t flags;- uint32_t seed; /* the surface's when it was last pushed out of the cache */+ uint32_t seed; /* the surface's when it was last pushed out of the cache, */+ uint64_t pushed; /* ... and when that was */bool seen; /* by the program: mapped, and from then on written by it */};@@ -296,13 +320,15 @@ class RedwoodGPUUserClient : public IOUserClienttask_t fTask;struct gpu_buffer * fBuffers; /* handle n is fBuffers[n - 1] */uint32_t fNextQueue;+ uint32_t fNextBuffer; /* where the search for a free handle starts: one given back is not the next given */uint32_t fBufferCount;uint64_t fBufferBytes;uint32_t fSubmits;+ uint32_t fStrangers; /* requests that named a buffer the client does not have */struct gpu_buffer * buffer(uint32_t handle);struct gpu_buffer * newBuffer(IOMemoryDescriptor * memory, uint64_t from, uint64_t size, uint32_t flags, uint32_t * handle);- void freeBuffer(struct gpu_buffer * b);+ void freeBuffer(struct gpu_buffer * b, bool idle);IOReturn gemNew(struct msm_gem_new * request);IOReturn surfaceBuffer(struct redwood_surface * request);IOReturn gemInfo(struct msm_gem_info * request);@@ -341,11 +367,26 @@ RedwoodGPUUserClient::start(IOService * provider)return fBuffers != NULL;}-/* With the lock held. */+/*+ * With the lock held, and the GPU done with everything it has been given (`idle`). If it is+ * not -- it has stopped answering -- the buffer's memory stays the GPU's for good: it may yet be+ * written, and must not be somebody else's by then.+ */void-RedwoodGPUUserClient::freeBuffer(struct gpu_buffer * b)+RedwoodGPUUserClient::freeBuffer(struct gpu_buffer * b, bool idle){- if (redwood_gpu_ready()) {+ if (!idle) {+ static uint32_t said;++ if (b->surface == NULL) {+ b->memory->retain();+ } else {+ b->surface->memory->retain();+ }+ if (said++ < 8) {+ IOLog("RedwoodGPU: the GPU is not done with what it was given: a buffer of %llu bytes stays its own\n", b->size);+ }+ } else if (redwood_gpu_ready()) {redwood_gpu_unmap(b->address, b->size);redwood_gpu_address_free(b->address, b->size);}@@ -365,12 +406,11 @@ RedwoodGPUUserClient::clientClose(void)if (fBuffers != NULL) {IOLockLock(gGPULock);/* what the client had the GPU do last may still be going on, in these buffers */- if (redwood_gpu_ready()) {- gpuWait(redwood_gpu_last(), 1000);- }+ bool idle = !redwood_gpu_ready() || gpuWait(redwood_gpu_last(), 1000);+for (uint32_t i = 0; i < GPU_BUFFERS; i++) {if (fBuffers[i].memory != NULL) {- freeBuffer(&fBuffers[i]);+ freeBuffer(&fBuffers[i], idle);}}IOLockUnlock(gGPULock);@@ -395,6 +435,10 @@ struct gpu_buffer *RedwoodGPUUserClient::buffer(uint32_t handle){if (handle == 0 || handle > GPU_BUFFERS || fBuffers[handle - 1].memory == NULL) {+ /* (a program that names a buffer it has given back draws into somebody else's next: worth knowing) */+ if (handle != 0 && fStrangers++ < 16) {+ IOLog("RedwoodGPU: buffer %u is asked for and is not there\n", handle);+ }return NULL;}return &fBuffers[handle - 1];@@ -410,10 +454,13 @@ RedwoodGPUUserClient::newBuffer(IOMemoryDescriptor * memory, uint64_t from, uintstruct gpu_buffer * b = NULL;bool mapped = true;- for (uint32_t i = 0; i < GPU_BUFFERS; i++) {+ for (uint32_t n = 0; n < GPU_BUFFERS; n++) {+ uint32_t i = (fNextBuffer + n) % GPU_BUFFERS;+if (fBuffers[i].memory == NULL) {b = &fBuffers[i];*handle = i + 1;+ fNextBuffer = i + 1;break;}}@@ -437,6 +484,7 @@ RedwoodGPUUserClient::newBuffer(IOMemoryDescriptor * memory, uint64_t from, uintb->size = size;b->flags = flags;b->seed = 0;+ b->pushed = 0;b->seen = false;b->surface = NULL;fBufferCount++;@@ -483,7 +531,9 @@ RedwoodGPUUserClient::gemNew(struct msm_gem_new * request)/** The memory of an IOSurface as a buffer: its pixels, from the page after the surface's shared* state on (MiniIOSurface.h). The surface is held for as long as the buffer is there. A surface- * that is a buffer of this client's already is that buffer again, as with Linux's handles.+ * that is a buffer of this client's already is that buffer again, as with Linux's handles: so+ * the client must not be giving that buffer back at the same moment (freedreno asks under the+ * lock it gives buffers back under). Asked only to describe (kRedwoodDescribe), no buffer is made.*/IOReturnRedwoodGPUUserClient::surfaceBuffer(struct redwood_surface * request)@@ -496,6 +546,11 @@ RedwoodGPUUserClient::surfaceBuffer(struct redwood_surface * request)if (s == NULL) {return kIOReturnNotFound;}+ if (s->client != NULL) {+ /* (memory of a program's own with a surface made around it: not in whole pages, and seldom in rows the GPU takes) */+ MiniIOSurfaceDrop(s);+ return kIOReturnUnsupported;+ }size = round_page_64(s->info.allocSize);memory = IOSubMemoryDescriptor::withSubRange(s->memory, PAGE_SIZE, size, kIODirectionInOut);if (memory == NULL || size == 0 || size > GPU_BUFFER_MAX) {@@ -510,6 +565,12 @@ RedwoodGPUUserClient::surfaceBuffer(struct redwood_surface * request)request->height = s->info.height;request->pixelFormat = s->info.pixelFormat;request->planes = s->info.planeCount;+ if (request->handle == kRedwoodDescribe) {+ request->handle = 0;+ memory->release();+ MiniIOSurfaceDrop(s);+ return kIOReturnSuccess;+ }IOLockLock(gGPULock);for (uint32_t i = 0; i < GPU_BUFFERS; i++) {@@ -568,7 +629,7 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)struct msm_gem_submit_cmd * cmds = NULL;uint64_t addresses[GPU_SUBMIT_CMDS];uint32_t words[GPU_SUBMIT_CMDS], count = 0;- uint64_t began, nanos;+ uint64_t began, nanos, again;IOReturn ret = kIOReturnBadArgument;if (request->nr_cmds == 0 || request->nr_cmds > GPU_SUBMIT_CMDS || request->nr_bos == 0 || request->nr_bos > GPU_SUBMIT_BOS) {@@ -590,12 +651,16 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)/** The GPU reads memory and writes memory. What this processor's cache holds of an IOSurface* (somebody has written it, or has read it, as the display's driver does, and the GPU is to- * draw into it again) goes out of the cache now, once for every time the surface is said to- * have changed (its seed: whoever writes one locks it, and so does a program that has had- * the GPU draw into one: userland/src/eagl.c). A surface this program has mapped, every time.- * The buffers made here need none of that, unless they are ordinary memory (gpu_cached=1).+ * draw into it again) goes out of the cache now: when the surface is said to have changed+ * (its seed: who writes one is to lock it, and so does a program that has had the GPU draw+ * into one: userland/src/eagl.c), and in any case once in a few milliseconds, which is once+ * a frame and not once a submission: not everybody who draws into a surface says so (an+ * application drawing a picture that is on the screen meanwhile). A surface this program+ * has mapped, every time. The buffers made here need none of that, unless they are+ * ordinary memory (gpu_cached=1).*/began = mach_absolute_time();+ nanoseconds_to_absolutetime(GPU_PUSH_AGAIN_MS * NSEC_PER_MSEC, &again);for (uint32_t i = 0; i < request->nr_bos; i++) {struct gpu_buffer * b = buffer(bos[i].handle);@@ -605,10 +670,11 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)if (b->surface != NULL) {uint32_t seed = b->surface->shared()->seed;- if (seed == b->seed && !b->seen) {+ if (seed == b->seed && !b->seen && began - b->pushed < again) {continue;}b->seed = seed;+ b->pushed = began;} else if (!b->seen || !gGPUCached) {continue;}@@ -655,18 +721,20 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)absolutetime_to_nanoseconds(mach_absolute_time() - gGPUStat.since, &nanos);if (nanos >= 5 * NSEC_PER_SEC) {uint64_t flushing, waiting, taking, worked;- uint32_t timed;+ uint32_t timed, by[7];absolutetime_to_nanoseconds(gGPUStat.flushing, &flushing);absolutetime_to_nanoseconds(gGPUStat.waiting, &waiting);absolutetime_to_nanoseconds(gGPUStat.taking, &taking);- redwood_gpu_worked(&worked, &timed);+ redwood_gpu_worked(&worked, &timed, by);IOLog("RedwoodGPU: %u submissions in %llu ms: %u command buffers and %u buffers named, %llu KB out of the cache "- "in %llu us a submission; the GPU worked %llu us on each of %u; %u waits of %llu us; %u IOSurfaces taken as "- "buffers in %llu us each; %u buffers, %llu MB\n",+ "in %llu us a submission; the GPU worked %llu us on each of %u (%u under 0.1 ms, %u under 0.25, %u under 0.5, "+ "%u under 1, %u under 2, %u under 4, %u longer; %llu ms in all); %u waits of %llu us, %u over before they "+ "began; %u IOSurfaces taken as buffers in %llu us each; %u buffers, %llu MB\n",gGPUStat.submissions, nanos / NSEC_PER_MSEC, gGPUStat.commands, gGPUStat.buffers,(gGPUStat.flushed >> 10) / gGPUStat.submissions, flushing / NSEC_PER_USEC / gGPUStat.submissions,- timed ? worked / timed : 0, timed, gGPUStat.waits, gGPUStat.waits ? waiting / NSEC_PER_USEC / gGPUStat.waits : 0,+ timed ? worked / timed : 0, timed, by[0], by[1], by[2], by[3], by[4], by[5], by[6], worked / 1000,+ gGPUStat.waits, gGPUStat.waits ? waiting / NSEC_PER_USEC / gGPUStat.waits : 0, gGPUStat.over,gGPUStat.surfaces, gGPUStat.surfaces ? taking / NSEC_PER_USEC / gGPUStat.surfaces : 0, fBufferCount,fBufferBytes >> 20);bzero(&gGPUStat, sizeof(gGPUStat));@@ -748,7 +816,7 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentsize_t size = args->structureInputSize;struct gpu_buffer * b;IOReturn ret = kIOReturnSuccess;- uint64_t zero = 0;+ uint64_t zero = 0, patience;if (args->structureInput == NULL || size > sizeof(r) || fBuffers == NULL) {return kIOReturnBadArgument;@@ -775,8 +843,12 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentbreak;case kMsmGemCpuPrep: /* is the GPU done with a buffer: with all it has been given, here */IOLockLock(gGPULock);- if (!gpuWait(redwood_gpu_last(), (r.cpuPrep.op & MSM_PREP_NOSYNC) ? 0 : millisecondsUntil(&r.cpuPrep.timeout))) {+ patience = (r.cpuPrep.op & MSM_PREP_NOSYNC) ? 0 : millisecondsUntil(&r.cpuPrep.timeout);+ if (!gpuWait(redwood_gpu_last(), patience)) {ret = kIOReturnBusy;+ if (patience == 0 && !(r.cpuPrep.op & MSM_PREP_NOSYNC)) {+ waitOver(&r.cpuPrep.timeout, redwood_gpu_last());+ }}IOLockUnlock(gGPULock);break;@@ -785,8 +857,12 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentbreak;case kMsmWaitFence:IOLockLock(gGPULock);- if (!gpuWait(r.waitFence.fence, millisecondsUntil(&r.waitFence.timeout))) {+ patience = millisecondsUntil(&r.waitFence.timeout);+ if (!gpuWait(r.waitFence.fence, patience)) {ret = kIOReturnTimeout;+ if (patience == 0) {+ waitOver(&r.waitFence.timeout, r.waitFence.fence);+ }}IOLockUnlock(gGPULock);break;@@ -807,8 +883,7 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentif (b == NULL) {ret = kIOReturnBadArgument;} else {- gpuWait(redwood_gpu_last(), 1000);- freeBuffer(b);+ freeBuffer(b, gpuWait(redwood_gpu_last(), 1000));}IOLockUnlock(gGPULock);break;diff --git a/xnu/files/osfmk/arm64/loader_return.c b/xnu/files/osfmk/arm64/loader_return.cindex 74a1006..0825a12 100644--- a/xnu/files/osfmk/arm64/loader_return.c+++ b/xnu/files/osfmk/arm64/loader_return.c@@ -46,6 +46,7 @@ void loader_log_putc(char c);void loader_return(void);extern void redwood_adreno_quiesce(void);extern void redwood_adreno_stop(void);+extern void redwood_adreno_minute(void);void loader_checkpoint(uint64_t n);/* Also referenced from the early exception vector in start.s. */@@ -474,6 +475,7 @@ memory_report(void)MB(memorystatus_available_pages),MB(memorystatus_available_pages_critical), MB(memorystatus_available_pages_critical_base));#undef MB+ redwood_adreno_minute(); /* (and how warm the GPU is, if it is in use) */}/* Every HV_WDT_PET_SECONDS: feed the watchdog if there is one, and show any watching loader a sign of life. */