Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
GPU path without copies: surfaces are the textures' memory, 550 MHz
vcvkk4cfd94da70a5
diff --git a/README.md b/README.mdindex 9b8c0f0..6b4d23e 100644--- a/README.md+++ b/README.md@@ -174,16 +174,21 @@ XL_SECS=900 XL_CMDLINE="touchscreen=1 cpufreq=1 bootcpu=7 cpus=4" ./xl run #```shtools/mk-mesa.shGL=1 REST=1 userland/build.sh-XL_CMDLINE="touchscreen=1 cpufreq=1 bootcpu=7 cpus=4 adreno=21 adreno_skip=0x1c00 adreno_pause=0 adreno_keep=1" ./xl run+XL_CMDLINE="touchscreen=1 cpufreq=1 bootcpu=7 cpus=4 adreno=21 adreno_skip=0x1c00 adreno_pause=0 adreno_keep=1 adreno_speed=3" ./xl run```-При втором и следующих запусках за одну загрузку телефона добавлять `adreno_zap=0`. Без параметров `adreno…`+`adreno_speed=3` — частота GPU 550 МГц (без него 315). При втором и следующих запусках за одну загрузку телефона+добавлять `adreno_zap=0`. Без параметров `adreno…`тот же образ рисует программным драйвером Mesa (в QEMU — всегда так).-Состояние — первый рабочий вариант, не режим по умолчанию: проверены экран блокировки и домашний экран; кадр-рисуется за 16–17 мс, но 14 из них — копирование готовой картинки (память кадра пока не общая с GPU); прерываний-нет (ожидание опросом); сеанс, оборванный паникой или `./xl stop`, оставляет GPU включённым — после него-`./xl reboot`. Приложения, использующие Metal сами, от этого ничего не получают.+Состояние — опытный режим, не по умолчанию. GPU рисует прямо в память поверхностей (IOSurface экрана и+полноэкранных слоёв — общая с GPU, без копий); на коротком сценарии (разблокировка, прокрутка Настроек, Часы)+это 1277–1319 кадров против 929 у программного рендера на четырёх ядрах, лучшие пять секунд — 52–57 кадров/с+против 35 (`adreno_speed=3`, 550 МГц; без него 315 МГц и 46 кадров/с). Кадр при прокрутке стоит около 19 мс, до+ровных 60 кадров/с ещё не дотянуто. Известные дефекты: приложение «Часы» рисуется чёрным, на домашнем экране под+некоторыми значками бывают лишние треугольники. Прерываний нет (ожидание опросом). GPU выключается при любом+штатном конце сеанса (таймер, `./xl stop`, пять пальцев); после паники — `./xl reboot`. Приложения, использующие+Metal сами, от этого ничего не получают.## Сборка из исходниковdiff --git a/prebuilt/SHA256SUMS b/prebuilt/SHA256SUMSindex 38e69ae..ac7b6ae 100644--- a/prebuilt/SHA256SUMS+++ b/prebuilt/SHA256SUMS@@ -1,6 +1,6 @@-Built 2026-10-12 23:49 from this tree.+Built 2026-10-13 02:02 from this tree.-b79c92cb557d7b995cee74fb3aa6a895d07b191c3c69d8e40ea19ae148e36a84 prebuilt/mach.development.bcm2837+74611fb6b6e0d7351eb6e14c79c6f97400f093d9c050d1f95d916d0d4882337d prebuilt/mach.development.bcm28371332a598d2dec00a1d8e930044e4f00a12ad44c6e5e50eecf4b7fe7fae803bf2 prebuilt/loader.efid2a35a4a1d6658ffb3696914f727661b52c9333de8a3af8a31642b8023ce03ac prebuilt/userland/child1a413befd511384052e1962c1394adce20f718e1f2657ce63ab09828ebc4bd1b prebuilt/userland/cpu_testdiff --git a/prebuilt/mach.development.bcm2837 b/prebuilt/mach.development.bcm2837index e268a2c..33c0c7a 100755Binary files a/prebuilt/mach.development.bcm2837 and b/prebuilt/mach.development.bcm2837 differdiff --git a/xnu/README.md b/xnu/README.mdindex 56778a4..10c32f0 100644--- a/xnu/README.md+++ b/xnu/README.md@@ -2,8 +2,8 @@Наши правки XNU `xnu-8796.121.2` (ядро iOS 16.5, Darwin 22.5.0):-- `redwood.patch` — изменения в файлах Apple (56 файлов);-- `files/` — новые файлы по их путям в дереве XNU (31 файл);+- `redwood.patch` — изменения в файлах Apple (60 файлов);+- `files/` — новые файлы по их путям в дереве XNU (37 файлов);- `apply.sh` — берёт исходники у Apple и накладывает всё это; `export.sh` — обратное действие.Собирается на Linux стоковым clang, без macOS и Xcode.@@ -62,7 +62,7 @@ xnu-deps/setup.sh # ~30 с: исходники зависимостей,| `bsd/kern/kern_minimac.c`, `libkern/amfi/` | заглушки AMFI и Sandbox (MAC-политики), права из подписи программы || `bsd/dev/redwood_input.c`, `redwood_goodix.c`, `redwood_virtio_input.c` | ввод: `/dev/redwood-touch`, драйвер Goodix GT9916 (SPI бит-бангом по GPIO), virtio-input для QEMU || `bsd/dev/redwood_cpufreq.c` | частота процессора (Qualcomm EPSS) |-| `bsd/dev/redwood_adreno.c` | GPU телефона (Adreno 642L): пока только пошаговое включение питания и тактов и чтение первых регистров (`adreno=N`) |+| `bsd/dev/redwood_adreno.c`, `iokit/Kernel/RedwoodGPU.cpp` | GPU телефона (Adreno 642L): пошаговое включение (`adreno=N`: питание, GMU, zap-шейдер, микрокод), буферы и очередь команд; клиент IOKit с подмножеством msm DRM для Mesa || `bsd/pthread/pthread_kext_static.c` | pthread kext, слинкованный в ядро |Патч, помимо врезок под всё это, чинит то, что мешает собрать и запустить открытый XNU вне Apple: makefile'ы подdiff --git a/xnu/files/bsd/dev/redwood_adreno.c b/xnu/files/bsd/dev/redwood_adreno.cindex 0593e8d..9a6a54d 100644--- a/xnu/files/bsd/dev/redwood_adreno.c+++ b/xnu/files/bsd/dev/redwood_adreno.c@@ -63,8 +63,10 @@* adreno_fine=1 the newest step waits like that before every single access* adreno_keep=1 with adreno=21: the GPU is not stopped again but kept for the system to draw* with, through the service RedwoodGPU (see "the GPU kept on" below); it is- * given back five seconds before a run's time is up+ * given back when the run ends (adreno_dirty=1: left as a panic leaves it)* adreno_zap=0 step 19 has been done since the phone started: not again+ * adreno_speed=N step 17 asks the GMU for the 3D core's Nth speed instead of the first:+ * 1 315 MHz, 2 450 MHz, 3 550 MHz (the most this phone's GPU is rated for)* adreno_blind=1 step 20 starts the command processor with its microcode nowhere, and* adreno_stream1=N step 18 leaves the core's second stream out (0) or gives it an empty* context of its own (2): what the SMMU then records shows what the core@@ -72,6 +74,8 @@* adreno_watch=S after the last step, say once a second for S seconds (default 20) that* the machine is still alive, and what was written last* adreno_return=1 when all is over, go back to the loader instead of starting the system+ * display_probe=1 before any step: read what the firmware left the display controller+ * doing (display_probe() below; nothing is written)* The system does not go on starting until all of this is over, so that the end of the log is* ours; the touch screen driver's start would talk across it (leave touchscreen=1 out).*@@ -84,7 +88,6 @@* vendor kernel agrees on every offset used here, and is the only source for the frequency counter.*/#include <kern/clock.h>-#include <kern/thread_call.h>#include <libkern/libkern.h>#include <mach/mach_types.h>#include <mach/vm_param.h>@@ -1218,6 +1221,7 @@ struct gpu_space {bool changed; /* since it was last written out of this processor's cache (space_commit()) */} tables[SPACE_TABLES];uint64_t *last_entry, *last_table; /* the level 2 entry looked up last, and its table */+ uint32_t last_changed; /* the table an entry was written in last */};static uint64_t *@@ -1248,9 +1252,14 @@ space_new_table(struct gpu_space *space)static voidspace_changed(struct gpu_space *space, const uint64_t *entry){+ if ((uintptr_t)entry - (uintptr_t)space->tables[space->last_changed].table < 4096) {+ space->tables[space->last_changed].changed = true; /* page after page of one buffer */+ return;+ }for (uint32_t i = space->table_count; i-- > 0;) {if ((uintptr_t)entry - (uintptr_t)space->tables[i].table < 4096) {space->tables[i].changed = true;+ space->last_changed = i;return;}}@@ -2278,6 +2287,8 @@ static struct block gpu = { "gpu", 0x03d00000, 0x40000 };#define GMU_CORE_REQUEST (1u << 30) /* of host2gmu_intr_set: keep the 3D core up for us; bit 31: no more */#define GMU_CORE_GRANTED (1u << 31) /* of gmu2host_intr_info */+static uint32_t core_khz; /* the 3D core's speed, as asked of the GMU */+/** What the vendor's driver sends at this point (a6xx_hfi_start; Linux the same with fewer* levels): the table of speeds of the 3D core with the supply level each needs -- the phone's@@ -2294,7 +2305,7 @@ step_core_power(void)} speeds[] = { { 0, 0 }, { 315000, 64 }, { 450000, 128 }, { 550000, 192 } };static struct hfi_perf_table perf;static struct hfi_bw_table bw;- uint32_t core_start[2] = { 0, 0 }, start[1] = { 0 }, vote[4] = { 0, 1, 1, HFI_NO_INDEX }, granted = 0;+ uint32_t core_start[2] = { 0, 0 }, start[1] = { 0 }, vote[4] = { 0, 1, 1, HFI_NO_INDEX }, granted = 0, speed = 1;uint16_t gfx[16], cx[16], mx[16];unsigned gfx_levels, cx_levels, mx_levels;@@ -2342,7 +2353,12 @@ step_core_power(void)return false;}- announce("send the GMU its tables (speeds and supply levels, bus votes), \"start\", and the speed wanted: 315 MHz; "+ if (PE_parse_boot_argn("adreno_speed", &speed, sizeof(speed)) && speed >= 1 && speed < perf.gpu_levels) {+ vote[2] = speed;+ }+ core_khz = speeds[vote[2]].khz;+ printf("adreno: %u: the speed wanted: %u MHz\n", step, core_khz / 1000);+ announce("send the GMU its tables (speeds and supply levels, bus votes), \"start\", and the speed wanted; ""from here on it casts votes in RPMh by itself");if (!hfi_send("the table of speeds", HFI_PERF_TABLE, &perf.header, sizeof(perf) / 4) ||!hfi_send("the table of bus votes", HFI_BW_TABLE, &bw.header, sizeof(bw) / 4) ||@@ -2373,8 +2389,8 @@ step_core_power(void)if (!hfi_send("\"start\"", HFI_START, start, 1)) {return false;}- printf("adreno: %u: measured: the 3D core's clock runs at %u Hz (315 MHz was asked for), the GMU's in it at %u Hz\n", step,- measure_quietly(GPUCC_DEBUG_GX_GFX3D), measure_quietly(GPUCC_DEBUG_GX_GMU));+ printf("adreno: %u: measured: the 3D core's clock runs at %u Hz (%u MHz was asked for), the GMU's in it at %u Hz\n", step,+ measure_quietly(GPUCC_DEBUG_GX_GFX3D), core_khz / 1000, measure_quietly(GPUCC_DEBUG_GX_GMU));announce("read registers of the 3D core itself (0x3d00000): the first time it has power");get(&gpu, "rbbm_status", RBBM_STATUS);@@ -2410,6 +2426,7 @@ step_core_power(void)#define GBIF_QSB_SIDE0 (4 * 0x03) /* of the bus interface (step 6) */#define RBBM_GBIF_CLIENT_QOS_CNTL GPU_WORD(0x0011)#define RBBM_GBIF_HALT GPU_WORD(0x0016)+#define RBBM_GBIF_HALT_ACK GPU_WORD(0x0017)#define RBBM_INTERFACE_HANG_INT_CNTL GPU_WORD(0x001f)#define RBBM_PERFCTR_CNTL GPU_WORD(0x0500)#define RBBM_PERFCTR_GPU_BUSY_MASKED GPU_WORD(0x050b)@@ -3061,6 +3078,24 @@ redwood_adreno_quiesce(void)wr(gmu.base + GMU_CM3_SYSRESET, 1);gmu_runs = false;}+ if (core_runs) {+ /*+ * The 3D core may be in the middle of a frame. Its bus interface is told to let what+ * is under way finish and to start nothing more (Linux's+ * a6xx_bus_clear_pending_transactions), and stays so: nothing of it reaches memory+ * after this, whatever becomes of its tables.+ */+ wr(gpu.base + RBBM_GBIF_HALT, 1);+ for (int i = 0; i < 100000 && !(rd(gpu.base + RBBM_GBIF_HALT_ACK) & 1); i++) {+ }+ wr(gbif.base + GBIF_HALT, 1);+ for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 1); i++) {+ }+ wr(gbif.base + GBIF_HALT, 2);+ for (int i = 0; i < 100000 && !(rd(gbif.base + GBIF_HALT_ACK) & 2); i++) {+ }+ core_runs = false;+ }if (gmu_space_is_on) {wr(smmu.base + smmu_context(&gmu_space) + SMMU_CB_SCTLR, 0);gmu_space_is_on = false;@@ -3090,6 +3125,10 @@ redwood_adreno_quiesce(void)#define GPU_ADDRESSES_START 0x20000000ull /* above everything the steps use; below 4 GB, as Linux keeps them */#define GPU_ADDRESSES_SIZE 0xe0000000ull#define GPU_FENCE_AT 0x40 /* in the scratch page */+#define GPU_TIMES_AT 0x400 /* ... and when the core began and ended each of its last submissions */+#define GPU_TIMES 64+#define CP_ALWAYS_ON_COUNTER 0x0980 /* ticks of the 19.2 MHz crystal, 64 bits: for the core to read itself */+#define CP_REG_TO_MEM 0x3e#define CP_SCRATCH_REG2 0x0885#define CP_IB1_BASE GPU_WORD(0x0928)#define CP_IB1_REM_SIZE GPU_WORD(0x092a)@@ -3101,6 +3140,10 @@ redwood_adreno_quiesce(void)static bool gpu_kept;static uint32_t gpu_fence; /* the number of the last submission */+static uint32_t gpu_timed; /* ... of the last whose times have been added up */+static uint64_t gpu_ticks; /* how long the core has worked on the submissions added up */+static uint32_t gpu_timed_count;+static uint32_t gpu_exact = 1; /* boot-arg gpu_exact=0: the core does not wait to be idle before it notes the time */static struct {uint64_t start, size;} gpu_free[1024]; /* the addresses not in use, in order */@@ -3111,13 +3154,14 @@ extern void RedwoodGPUStop(void); /* takes the clients' lock, then calls rstatic void gpu_block_stop(void);/*- * A run that the loader ends after so many seconds (boot-arg loader_return_after) gives the GPU- * back a few seconds before that, in order and with time to do it: the next kernel then finds- * the GPU block as the phone's own start leaves it. A run that ends otherwise (a panic, xl stop)- * leaves it running, and the next kernel's step 4 has to deal with that.+ * A run that ends in order -- its time is up, the host asks for the end, five fingers on the+ * screen, a halt: whoever is about to call loader_return() for one of these calls+ * redwood_adreno_stop() first, from a thread that may wait -- gives the GPU back as the steps'+ * own ending does, and the next kernel finds the GPU block as the phone's start leaves it. A+ * panic cannot: it leaves the GPU block running but quiet (redwood_adreno_quiesce()), and the+ * next kernel's step 4 takes it down (stale_stop()). Boot-arg adreno_dirty=1 makes every ending+ * a panic's in this respect, to see that work.*/-#define GPU_STOP_AHEAD 5-voidredwood_gpu_stop(void){@@ -3129,10 +3173,14 @@ redwood_gpu_stop(void)}}-static void-gpu_stop_timer(__unused thread_call_param_t p0, __unused thread_call_param_t p1)+void+redwood_adreno_stop(void){- RedwoodGPUStop();+ uint32_t dirty = 0;++ if (gpu_kept && !(PE_parse_boot_argn("adreno_dirty", &dirty, sizeof(dirty)) && dirty)) {+ RedwoodGPUStop();+ }}bool@@ -3161,7 +3209,7 @@ redwood_gpu_parameter(uint32_t which, uint64_t *value)*value = GPU_CHIP_ID | (uint64_t)speed_bin << 32;return true;case 0x04: /* MAX_FREQ */- *value = 315000000;+ *value = core_khz * 1000ull;return true;case 0x05: /* TIMESTAMP */*value = mach_absolute_time();@@ -3278,6 +3326,49 @@ redwood_gpu_last(void)return gpu_fence;}+/* The core is to write down what time it is: at one end of submission `fence`. */+static void+ring_time(uint32_t fence, uint32_t end)+{+ ring_command(CP_REG_TO_MEM, 3);+ ring(CP_ALWAYS_ON_COUNTER | 2u << 18 | 1u << 30); /* two words, a 64-bit register */+ ring(CORE_SCRATCH_ADDRESS + GPU_TIMES_AT + 16 * (fence % GPU_TIMES) + 8 * end);+ ring(0);+}++/* Add up the times of the submissions the core has finished since this was last done. */+static void+gpu_times(void)+{+ volatile uint64_t *times = (volatile uint64_t *)(core_scratch.memory + GPU_TIMES_AT);+ uint32_t done;++ to_memory(core_scratch.memory, 4096);+ done = *(volatile uint32_t *)(core_scratch.memory + GPU_FENCE_AT);+ if (done - gpu_timed > GPU_TIMES) {+ gpu_timed = done - GPU_TIMES; /* (the older ones have been written over) */+ }+ while (gpu_timed != done) {+ uint32_t slot = ++gpu_timed % GPU_TIMES;++ if (gpu_timed != 0 && times[2 * slot + 1] > times[2 * slot]) {+ gpu_ticks += times[2 * slot + 1] - times[2 * slot];+ gpu_timed_count++;+ }+ }+}++/* How long the 3D core has worked since this was last asked, and on how many submissions. */+void+redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions)+{+ gpu_times();+ *microseconds = gpu_ticks * 10 / 192;+ *submissions = gpu_timed_count;+ gpu_ticks = 0;+ gpu_timed_count = 0;+}+/** Command buffers for the 3D core to run, in order: each an address and a count of words. As* Linux's a6xx_submit puts them in the ring. Returns the submission's number; 0: the ring is@@ -3286,7 +3377,7 @@ redwood_gpu_last(void)uint32_tredwood_gpu_submit(const uint64_t *addresses, const uint32_t *words, uint32_t count){- uint32_t wanted = 16 + 4 * count, room = 0;+ uint32_t wanted = 26 + 4 * count, room = 0;for (int i = 0; i < 20000; i++) {room = (rd(gpu.base + CP_RB_RPTR) + RING_WORDS - ring_at - 1) % RING_WORDS;@@ -3298,6 +3389,11 @@ redwood_gpu_submit(const uint64_t *addresses, const uint32_t *words, uint32_t coif (room < wanted) {return 0;}+ gpu_times();+ if (++gpu_fence == 0) {+ gpu_fence = 1;+ }+ ring_time(gpu_fence, 0);ring_command(CP_EVENT_WRITE, 1);ring(EVENT_INVALIDATE_DEPTH);ring_command(CP_EVENT_WRITE, 1);@@ -3308,9 +3404,10 @@ redwood_gpu_submit(const uint64_t *addresses, const uint32_t *words, uint32_t coring((uint32_t)(addresses[i] >> 32));ring(words[i]);}- if (++gpu_fence == 0) {- gpu_fence = 1;+ if (gpu_exact) {+ ring_command(CP_WAIT_FOR_IDLE, 0); /* the time is the pipeline's, not the command processor's, which is ahead of it */}+ ring_time(gpu_fence, 1);ring_register(CP_SCRATCH_REG2, gpu_fence);ring_command(CP_EVENT_WRITE, 4);ring(EVENT_CACHE_FLUSH);@@ -3356,6 +3453,81 @@ static bool(*const steps[])(void) = {* as long as the steps and the watch after them last, nothing else writes to the log, so that* its end -- all that the loader's notes keep -- is about what is being done here.*/+/* ---- beside the steps: what the firmware left the display doing (boot-arg display_probe=1) ---------------- */++/*+ * The panel is a DSI video panel (dt: qcom,mdss_dsi_m20_36_02_0a_dsc_vid: 1080 x 2400 at 120+ * frames a second, 2432 lines with the porches; or at 60 with a longer vertical front porch):+ * the display controller sends it the framebuffer over and over at the rate of its timing+ * registers, which nobody has touched since the firmware. This reads them, and nothing is+ * written. Each read is behind a check that what it reads has its clock: the bus clock of the+ * display's clock controller (by GCC, which answers always), in that controller the display+ * block's power switch and bus clock, then the timing of the interface the panel hangs on+ * (Linux's dpu_hw_intf.c has the registers' meaning). Every read is announced and waited for+ * (adreno_pause), as in the newest step.+ */+static struct block dispcc = { "dispcc", 0x0af00000, 0x2000 }; /* dt: clock-controller@af00000 */+static struct block disp_intf = { "intf1", 0x0ae36000, 0x300 }; /* dt: mdss_dsi0's mdp_intf_base */+#define GCC_DISP_AHB_CBCR 0x27004+#define DISPCC_MDSS_CORE_GDSCR 0x1004 /* dt: disp_cc_mdss_core_gdsc */+#define DISPCC_MDSS_PCLK0_CBCR 0x1010+#define DISPCC_MDSS_MDP_CBCR 0x1014+#define DISPCC_MDSS_AHB_CBCR 0x1050+#define INTF_TIMING_ENGINE_EN 0x000+#define INTF_HSYNC_CTL 0x008 /* clocks a line << 16 | of which the pulse */+#define INTF_VSYNC_PERIOD 0x00c /* clocks a frame */+#define INTF_DISPLAY_V_START 0x01c /* clocks before the first line shown, and */+#define INTF_DISPLAY_V_END 0x024 /* ... up to the last */+#define INTF_DISPLAY_HCTL 0x03c+#define INTF_FRAME_LINE_COUNT_EN 0x0a8+#define INTF_FRAME_COUNT 0x0ac+#define INTF_LINE_COUNT 0x0b0++static void+display_probe(void)+{+ uint32_t saved_fine = fine, line, frame, shown, counting, frames[2], lines[2];++ step = newest_step = 0;+ fine = 1;+ announce("the display: has its clock controller its bus clock (gcc 0x127004; GCC answers always)");+ if (get(&gcc, "disp_ahb_cbcr", GCC_DISP_AHB_CBCR) & CBCR_OFF) {+ printf("adreno: 0: the display's clock controller has no bus clock: nothing more is read\n");+ goto out;+ }+ announce("the display block's power switch and clocks, in its clock controller (0xaf00000: NOT READ BEFORE)");+ if (!(get(&dispcc, "mdss_core_gdscr", DISPCC_MDSS_CORE_GDSCR) & GDSC_POWER_ON) ||+ (get(&dispcc, "mdss_ahb_cbcr", DISPCC_MDSS_AHB_CBCR) & CBCR_OFF)) {+ printf("adreno: 0: the display block is without power or without its bus clock: nothing more is read\n");+ goto out;+ }+ get(&dispcc, "mdss_mdp_cbcr", DISPCC_MDSS_MDP_CBCR);+ get(&dispcc, "mdss_pclk0_cbcr", DISPCC_MDSS_PCLK0_CBCR);+ announce("the timing of the interface the panel hangs on (0xae36000: NOT READ BEFORE)");+ get(&disp_intf, "timing_engine_en", INTF_TIMING_ENGINE_EN);+ line = get(&disp_intf, "hsync_ctl", INTF_HSYNC_CTL) >> 16;+ frame = get(&disp_intf, "vsync_period", INTF_VSYNC_PERIOD);+ shown = get(&disp_intf, "display_v_end", INTF_DISPLAY_V_END) - get(&disp_intf, "display_v_start", INTF_DISPLAY_V_START);+ get(&disp_intf, "display_hctl", INTF_DISPLAY_HCTL);+ counting = get(&disp_intf, "frame_line_count_en", INTF_FRAME_LINE_COUNT_EN);+ frames[0] = get(&disp_intf, "frame_count", INTF_FRAME_COUNT);+ lines[0] = get(&disp_intf, "line_count", INTF_LINE_COUNT);+ quiet = true;+ sleep_ms(1000);+ frames[1] = get(&disp_intf, "frame_count", INTF_FRAME_COUNT);+ lines[1] = get(&disp_intf, "line_count", INTF_LINE_COUNT);+ quiet = false;+ if (line != 0) {+ printf("adreno: 0: THE DISPLAY: %u lines a frame, of which %u shown, %u clocks a line (the device tree's timing "+ "for 120 frames a second has 2432 lines; for 60, about twice that)\n", frame / line, (shown + 1) / line, line);+ }+ printf("adreno: 0: the interface's counters (%s, frame_line_count_en %#x): frames %u then %u a second later, "+ "lines %u then %u\n", (counting & 1) ? "counting" : "not counting", counting, frames[0], frames[1], lines[0], lines[1]);+out:+ fine = saved_fine;+ newest_step = 0;+}+voidredwood_adreno_init(void){@@ -3367,14 +3539,17 @@ redwood_adreno_init(void)last_step = only;skip_steps = ~(1u << only);}+ PE_parse_boot_argn("adreno_pause", &pause_ms, sizeof(pause_ms));+ PE_parse_boot_argn("adreno_fine", &fine, sizeof(fine));+ if (PE_parse_boot_argn("display_probe", &only, sizeof(only)) && only) {+ display_probe();+ }if (last_step == 0) {return;}if (last_step > STEPS) {last_step = STEPS;}- PE_parse_boot_argn("adreno_pause", &pause_ms, sizeof(pause_ms));- PE_parse_boot_argn("adreno_fine", &fine, sizeof(fine));PE_parse_boot_argn("adreno_watch", &watch_seconds, sizeof(watch_seconds));for (newest_step = last_step; newest_step > 0 && (skip_steps & 1u << newest_step); newest_step--) {}@@ -3397,18 +3572,13 @@ redwood_adreno_init(void)gpu_free[0].start = GPU_ADDRESSES_START;gpu_free[0].size = GPU_ADDRESSES_SIZE;gpu_free_count = 1;- *(volatile uint32_t *)(core_scratch.memory + GPU_FENCE_AT) = 0;+ bzero((void *)core_scratch.memory, 4096);to_memory(core_scratch.memory, 4096);+ PE_parse_boot_argn("gpu_exact", &gpu_exact, sizeof(gpu_exact));gpu_kept = true;quiet = true;printf("adreno: THE GPU STAYS ON for the system to draw with (adreno_keep): the service RedwoodGPU\n");RedwoodGPUStart();- if (PE_parse_boot_argn("loader_return_after", &only, sizeof(only)) && only > GPU_STOP_AHEAD + 5) {- uint64_t deadline;-- clock_interval_to_deadline(only - GPU_STOP_AHEAD, NSEC_PER_SEC, &deadline);- thread_call_enter_delayed(thread_call_allocate(gpu_stop_timer, NULL), deadline);- }return;}printf("adreno: not kept: the steps did not get as far as a 3D core that takes commands\n");diff --git a/xnu/files/bsd/dev/redwood_input.c b/xnu/files/bsd/dev/redwood_input.cindex dc665bb..69d5f75 100644--- a/xnu/files/bsd/dev/redwood_input.c+++ b/xnu/files/bsd/dev/redwood_input.c@@ -37,6 +37,7 @@ static voidexit_hold_elapsed(__unused thread_call_param_t p0, __unused thread_call_param_t p1){printf("\n--- %d fingers held on the screen for %d ms: back to the loader ---\n", EXIT_FINGERS, EXIT_HOLD_MS);+ redwood_adreno_stop();loader_return();}diff --git a/xnu/files/bsd/sys/redwood_input.h b/xnu/files/bsd/sys/redwood_input.hindex a12e979..42cbdc1 100644--- a/xnu/files/bsd/sys/redwood_input.h+++ b/xnu/files/bsd/sys/redwood_input.h@@ -44,6 +44,7 @@ void redwood_goodix_init(void); /* boot-arg touchscreen=1: the phone's tvoid redwood_cpufreq_init(void); /* boot-arg cpufreq=1: the phone's processor at full speed */void redwood_adreno_init(void); /* boot-arg adreno=N: the first N steps of waking the phone's GPU */void redwood_adreno_quiesce(void); /* on the way back to the loader: the GPU block stopped, out of our memory */+void redwood_adreno_stop(void); /* before that, where the ending is no panic's: the GPU given back in order *//** The GPU kept on (boot-arg adreno_keep=1), for iokit/Kernel/RedwoodGPU.cpp: see "the GPU kept on"@@ -60,6 +61,7 @@ void redwood_gpu_to_memory(void *start, uint64_t size);uint32_t redwood_gpu_submit(const uint64_t *addresses, const uint32_t *words, uint32_t count);bool redwood_gpu_done(uint32_t fence);uint32_t redwood_gpu_last(void);+void redwood_gpu_worked(uint64_t *microseconds, uint32_t *submissions);void redwood_gpu_report(const char *why);void redwood_gpu_stop(void);#endifdiff --git a/xnu/files/iokit/Kernel/MiniIOMobileFramebuffer.cpp b/xnu/files/iokit/Kernel/MiniIOMobileFramebuffer.cppindex e829074..773cf66 100644--- a/xnu/files/iokit/Kernel/MiniIOMobileFramebuffer.cpp+++ b/xnu/files/iokit/Kernel/MiniIOMobileFramebuffer.cpp@@ -73,7 +73,10 @@ enum {#define kFBNotifyVSync 5#define kFBNotifyTypes 8-#define kFBRefreshHz 60+#define kFBRefreshHz 60 /* unless boot-arg fb_hz says otherwise */++#define FRAME_TIMES 64 /* milliseconds told apart between two frames */+#define FRAME_PAUSE 90 /* from so many on, nothing was moving: not a slow frame */#define FOURCC(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (uint32_t)(d))@@ -120,6 +123,14 @@ static TUNABLE(uint32_t, gFBTrace, "fbtrace", 3);static TUNABLE(uint32_t, gRecSeconds, "rec_secs", 0);static TUNABLE(uint32_t, gRecStart, "rec_start", 0);static TUNABLE(uint32_t, gRecMegabytes, "rec_mb", 480);+static TUNABLE(uint32_t, gDiff, "fb_diff", 0);+static TUNABLE(uint32_t, gRefreshHz, "fb_hz", kFBRefreshHz);++static inline uint32_t+refreshHz(void)+{+ return gRefreshHz >= 1 && gRefreshHz <= 240 ? gRefreshHz : kFBRefreshHz;+}#define REC_STEP 3struct rec_header {@@ -157,10 +168,18 @@ public:MiniIOSurface * fDefaultSurface;uint32_t fSwapID;uint32_t fSwapsShown;+ uint32_t fLayersShown; /* which layers the last swap had a surface in */bool fGraphics; /* the text console has been told to stop *//* how the display is doing: frames and time spent drawing them here since fStatSince */uint32_t fStatFrames;uint64_t fStatSince, fStatDrawing;+ /* ... and how evenly: the time from each frame to the next, counted by the millisecond (the last count: longer) */+ uint64_t fLastFrame;+ uint32_t fFrameTimes[FRAME_TIMES + 1], fFramePauses, fFrameLongest;+ /* boot-arg fb_diff=1: which pixels of the panel each frame changed (it costs: the panel's memory is slow to read) */+ bool fDiff;+ uint64_t fDiffPixels;+ uint32_t fDiffLeft, fDiffTop, fDiffRight, fDiffBottom;/* the screen recording, if one was asked for */uint32_t * fColumn; /* drawLayer(): source column for every panel column */uint8_t * fRecBase;@@ -172,6 +191,7 @@ public:void publishModes(void);MiniIOSurface * defaultSurface(void);void present(const uint8_t * swap);+ void reportFrameTimes(void);void drawLayer(MiniIOSurface * s, const int32_t * src, const int32_t * dst);void record(MiniIOSurface * s);};@@ -192,7 +212,7 @@ setNumber(OSDictionary * dict, const char * key, uint64_t value)/** The modes of the display, the way CoreAnimation reads them (the properties TimingElements* and ColorElements; without a mode it draws nothing). One timing, the panel's size at- * 60 Hz, with one colour mode, 8-bit RGB. The identifiers are what "the current mode"+ * 60 Hz (boot-arg fb_hz=N: at N), with one colour mode, 8-bit RGB. The identifiers are what "the current mode"* (block 0x40) answers with.*/#define kFBTimingID 1@@ -207,7 +227,7 @@ IOMobileFramebuffer::publishModes(void)OSDictionary * color = OSDictionary::withCapacity(8);OSArray * timings = OSArray::withCapacity(1);OSArray * colors = OSArray::withCapacity(1);- const uint32_t rate = kFBRefreshHz << 16; /* 16.16 fixed point */+ const uint32_t rate = refreshHz() << 16; /* 16.16 fixed point */if (timing == NULL || horizontal == NULL || vertical == NULL || color == NULL || timings == NULL || colors == NULL) {return;@@ -284,6 +304,8 @@ IOMobileFramebuffer::start(IOService * provider)}}fColumn = IONewZero(uint32_t, fPanelWidth);+ fDiff = gDiff != 0;+ fDiffLeft = fDiffTop = ~0u;IOLog("MiniIOMobileFramebuffer: display %ux%u on a panel of %ux%u (%u bytes per row)\n", fWidth, fHeight,fPanelWidth, fPanelHeight, fRowBytes);@@ -432,20 +454,81 @@ IOMobileFramebuffer::drawLayer(MiniIOSurface * s, const int32_t * src, const intmemcpy(to + px0, from + sx, (size_t) sw * 4);continue;}+ if (fDiff) {+ for (int32_t x = cx0; x < cx1; x++) {+ if (to[x] != from[fColumn[x]]) {+ fDiffPixels++;+ fDiffLeft = (uint32_t) x < fDiffLeft ? (uint32_t) x : fDiffLeft;+ fDiffRight = (uint32_t) x > fDiffRight ? (uint32_t) x : fDiffRight;+ fDiffTop = (uint32_t) y < fDiffTop ? (uint32_t) y : fDiffTop;+ fDiffBottom = (uint32_t) y > fDiffBottom ? (uint32_t) y : fDiffBottom;+ }+ }+ }for (int32_t x = cx0; x < cx1; x++) {to[x] = from[fColumn[x]];}}}+/*+ * How evenly the frames of the last seconds came: how many followed the one before within the+ * time of one refresh at 120, 60, 40 and 30 a second, and the times half and 95 in 100 of them+ * stayed under. With the lock held.+ */+void+IOMobileFramebuffer::reportFrameTimes(void)+{+ static const uint32_t limits[] = { 10, 18, 26, 35, FRAME_TIMES + 1 };+ uint32_t within[sizeof(limits) / sizeof(limits[0])] = { 0 }, total = 0, sum = 0, half = 0, most = 0;++ for (uint32_t ms = 0; ms <= FRAME_TIMES; ms++) {+ for (uint32_t i = 0; i < sizeof(limits) / sizeof(limits[0]); i++) {+ if (ms < limits[i]) {+ within[i] += fFrameTimes[ms];+ break;+ }+ }+ total += fFrameTimes[ms];+ }+ for (uint32_t ms = 0; ms <= FRAME_TIMES && total != 0; ms++) {+ sum += fFrameTimes[ms];+ if (half == 0 && 2 * sum >= total) {+ half = ms + 1;+ }+ if (most == 0 && 100 * sum >= 95 * total) {+ most = ms + 1;+ }+ }+ if (total != 0) {+ IOLog("MiniIOMobileFramebuffer: from one frame to the next: %u times under 10 ms, %u under 18, %u under 26, %u under 35, "+ "%u longer (the longest %u ms); half under %u ms, 95 in 100 under %u ms; %u pauses\n", within[0], within[1],+ within[2], within[3], within[4], fFrameLongest, half, most, fFramePauses);+ }+ bzero(fFrameTimes, sizeof(fFrameTimes));+ fFramePauses = fFrameLongest = 0;+}+voidIOMobileFramebuffer::present(const uint8_t * swap){uint32_t set = swapWord(swap, SWAP_LAYERS_SET);uint64_t began = mach_absolute_time(), nanos;+ uint32_t layers = 0;bool report;IOLockLock(fLock);+ if (fLastFrame != 0) {+ absolutetime_to_nanoseconds(began - fLastFrame, &nanos);+ nanos /= NSEC_PER_MSEC;+ if (nanos >= FRAME_PAUSE) {+ fFramePauses++;+ } else {+ fFrameTimes[nanos < FRAME_TIMES ? nanos : FRAME_TIMES]++;+ fFrameLongest = nanos > fFrameLongest ? (uint32_t) nanos : fFrameLongest;+ }+ }+ fLastFrame = began;report = fSwapsShown < 8;if (report) {IOLog("MiniIOMobileFramebuffer: swap %u (#%u): surfaces %u %u %u %u, layers set 0x%x cleared 0x%x\n",@@ -457,6 +540,7 @@ IOMobileFramebuffer::present(const uint8_t * swap)int32_t src[4], dst[4];MiniIOSurface * s;+ layers |= sid != 0 ? 1u << layer : 0;if (sid == 0) {/* the bottom layer taken away: nothing is shown, the screen is black */if (layer == 0 && (set & 1) && fGraphics) {@@ -494,6 +578,12 @@ IOMobileFramebuffer::present(const uint8_t * swap)MiniIOSurfaceDrop(s);}fSwapsShown++;+ if (layers != fLayersShown) {+ IOLog("MiniIOMobileFramebuffer: swap #%u: surfaces in layers 0x%x now (0x%x before): %u %u %u %u\n", fSwapsShown, layers,+ fLayersShown, swapWord(swap, SWAP_SURFACE(0)), swapWord(swap, SWAP_SURFACE(1)), swapWord(swap, SWAP_SURFACE(2)),+ swapWord(swap, SWAP_SURFACE(3)));+ fLayersShown = layers;+ }/* every five seconds in which something was drawn: how many frames, and what each cost here */fStatFrames++;@@ -508,6 +598,14 @@ IOMobileFramebuffer::present(const uint8_t * swap)absolutetime_to_nanoseconds(fStatDrawing, &drawing);IOLog("MiniIOMobileFramebuffer: %u frames in %llu ms (%llu a second), %llu us each in the driver\n", fStatFrames,nanos / NSEC_PER_MSEC, fStatFrames * NSEC_PER_SEC / nanos, drawing / fStatFrames / NSEC_PER_USEC);+ reportFrameTimes();+ if (fDiff) {+ IOLog("MiniIOMobileFramebuffer: %llu pixels of the panel changed a frame, all within %u,%u to %u,%u\n",+ fDiffPixels / fStatFrames, fDiffLeft, fDiffTop, fDiffRight, fDiffBottom);+ fDiffPixels = 0;+ fDiffLeft = fDiffTop = ~0u;+ fDiffRight = fDiffBottom = 0;+ }fStatFrames = 0;fStatDrawing = 0;fStatSince = 0;@@ -529,6 +627,7 @@ class IOMobileFramebufferUserClient : public IOUserClientio_user_reference_t fRefcons[kFBNotifyTypes];thread_call_t fVSyncCall;uint64_t fVSyncCount;+ uint64_t fVSyncTime; /* when the refresh being waited for is due */bool fVSyncRunning;bool fClosed;@@ -637,10 +736,15 @@ IOMobileFramebufferUserClient::vsyncFired(thread_call_param_t self, thread_call_voidIOMobileFramebufferUserClient::armVSync(void){- uint64_t deadline;+ uint64_t period, now = mach_absolute_time();- clock_interval_to_deadline(1000000 / kFBRefreshHz, NSEC_PER_USEC, &deadline);- thread_call_enter_delayed(fVSyncCall, deadline);+ /* one refresh after the last, not after whenever we got round to this: the rate does not drift */+ nanoseconds_to_absolutetime(NSEC_PER_SEC / refreshHz(), &period);+ fVSyncTime += period;+ if (fVSyncTime <= now || fVSyncTime > now + 2 * period) {+ fVSyncTime = now + period;+ }+ thread_call_enter_delayed(fVSyncCall, fVSyncTime);}/*@@ -652,10 +756,11 @@ IOMobileFramebufferUserClient::vsync(void){OSAsyncReference64 ref;io_user_reference_t args[6];- uint64_t now = mach_absolute_time();+ uint64_t now;bool send = false;IOLockLock(fLock);+ now = fVSyncTime;if (fClosed || fCallbacks[kFBNotifyVSync] == 0 || fPorts[kFBNotifyVSync] == NULL) {fVSyncRunning = false;} else {diff --git a/xnu/files/iokit/Kernel/RedwoodGPU.cpp b/xnu/files/iokit/Kernel/RedwoodGPU.cppindex 99b6b4b..e782030 100644--- a/xnu/files/iokit/Kernel/RedwoodGPU.cpp+++ b/xnu/files/iokit/Kernel/RedwoodGPU.cpp@@ -11,12 +11,19 @@* What there is of the interface is what freedreno uses to draw: parameters, buffers (made,* told their address in the GPU's address space, mapped, given back), submission of command* buffers, and waiting for one to be done. A buffer is wired kernel memory entered into the 3D- * core's SMMU context; it is zero when made. The program has this processor's cache between- * itself and the memory, the GPU has not: see submit().+ * core's SMMU context; it is zero when made, and write-combined: what the program writes goes to+ * memory, where the GPU reads it, and not into this processor's cache (as Linux maps them for+ * freedreno; boot-arg gpu_cached=1 makes them ordinary memory, pushed out of the cache whenever+ * the GPU is given something to do: see submit()).*- * Not here yet: interrupts (waiting is asking again every millisecond), more than one program's- * worth of care (every client's buffers are in the one address space; a client that goes takes- * its own away), recovery of a GPU that has stopped answering (it is reported, see+ * One request is not Linux's: a buffer that is the memory of an IOSurface (surfaceBuffer()), for+ * drawing into what the display shows and for reading pictures where other programs have drawn+ * them. Linux hands such memory around as file descriptors; here it has a name already. That+ * memory is ordinary, cached memory, which everybody else reads and writes as such.+ *+ * Not here yet: interrupts (waiting is asking again five times a millisecond), more than one+ * program's worth of care (every client's buffers are in the one address space; a client that+ * goes takes its own away), recovery of a GPU that has stopped answering (it is reported, see* redwood_gpu_report(), and stays stopped), fences and buffers as file descriptors.*/@@ -27,11 +34,14 @@#include <IOKit/IOService.h>#include <IOKit/IOUserClient.h>#include <IOKit/IOBufferMemoryDescriptor.h>+#include <IOKit/IOSubMemoryDescriptor.h>#include <kern/clock.h>#include <mach/mach_time.h>#include <mach/vm_param.h>#include <sys/systm.h>+#include "MiniIOSurface.h"+extern "C" {/* bsd/dev/redwood_adreno.c ("the GPU kept on"; declared in bsd/sys/redwood_input.h, which is not seen from here) */bool redwood_gpu_ready(void);@@ -45,6 +55,7 @@ void redwood_gpu_to_memory(void * start, uint64_t size);uint32_t redwood_gpu_submit(const uint64_t * addresses, const uint32_t * words, uint32_t count);bool redwood_gpu_done(uint32_t fence);uint32_t redwood_gpu_last(void);+void redwood_gpu_worked(uint64_t * microseconds, uint32_t * submissions);void redwood_gpu_report(const char * why);void redwood_gpu_stop(void);@@ -68,6 +79,7 @@ enum {kMsmSubmitQueueNew = 0x4a,kMsmSubmitQueueClose = 0x4b,kMsmSubmitQueueQuery = 0x4c,+ kRedwoodSurface = 0x7e, /* ours: an IOSurface's memory as a buffer */kRedwoodShow = 0x7f, /* ours: pixels of the caller's onto the panel, to see a test's result */};@@ -126,6 +138,13 @@ struct msm_submitqueue_query {struct drm_gem_close {uint32_t handle, pad;};+struct redwood_surface {+ uint32_t surface; /* the IOSurface's ID */+ uint32_t handle; /* back: the buffer, */+ uint64_t size; /* how big it is, */+ uint32_t offset, rowBytes; /* where the pixels start in it and how long a row of them is, */+ uint32_t width, height, pixelFormat, planes; /* and what they are */+};struct redwood_show {uint64_t pixels;uint32_t width, height, rowBytes, pad;@@ -145,23 +164,39 @@ struct redwood_show {#define GPU_SUBMIT_CMDS 128#define GPU_SUBMIT_BOS 8192#define GPU_PATIENCE_MS 5000 /* for a buffer or a submission that somebody waits for without a limit */+#define GPU_ASK_AGAIN_US 200 /* while waiting for the core: how long between two looks at how far it is */static IOLock * gGPULock; /* around everything that is the GPU's (redwood_gpu_...) */static bool gGPUReported;+static bool gGPUCached; /* boot-arg gpu_cached=1 */++/* What the last seconds' submissions were and cost here, for the log. */+static struct {+ uint64_t since, flushing, waiting, taking; /* times of the processor's clock */+ uint64_t flushed; /* bytes sent out of the cache */+ uint32_t submissions, commands, buffers, surfaces, waits;+} gGPUStat;/* Wait for the 3D core to be done with submission `fence`; with the lock held, which is let go of while waiting. */static boolgpuWait(uint32_t fence, uint64_t milliseconds){- for (uint64_t waited = 0;; waited++) {+ uint64_t began = mach_absolute_time(), patience;++ nanoseconds_to_absolutetime(milliseconds * NSEC_PER_MSEC, &patience);+ for (bool waited = false;; waited = true) {if (redwood_gpu_done(fence)) {+ if (waited) {+ gGPUStat.waits++;+ gGPUStat.waiting += mach_absolute_time() - began;+ }return true;}- if (waited >= milliseconds) {+ if (mach_absolute_time() - began >= patience) {break;}IOLockUnlock(gGPULock);- IOSleep(1);+ delay_for_interval(GPU_ASK_AGAIN_US, NSEC_PER_USEC);IOLockLock(gGPULock);}if (milliseconds >= 1000 && !gGPUReported) {@@ -215,7 +250,9 @@ RedwoodGPUStart(void){IOService * provider = IOService::getPlatform();RedwoodGPU * gpu;+ uint32_t cached = 0;+ gGPUCached = PE_parse_boot_argn("gpu_cached", &cached, sizeof(cached)) && cached != 0;gGPULock = IOLockAlloc();gpu = OSTypeAlloc(RedwoodGPU);if (gpu == NULL || provider == NULL || !gpu->init() || !gpu->attach(provider)) {@@ -242,10 +279,14 @@ RedwoodGPUStop(void)/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */struct gpu_buffer {- IOBufferMemoryDescriptor * memory; /* NULL: this handle is free */+ IOMemoryDescriptor * memory; /* NULL: this handle is free */+ uint8_t * kernel; /* where it is in the kernel */+ MiniIOSurface * surface; /* whose memory it is, when not its own */uint64_t address; /* the GPU's for it */uint64_t size;uint32_t flags;+ uint32_t seed; /* the surface's when it was last pushed out of the cache */+ bool seen; /* by the program: mapped, and from then on written by it */};class RedwoodGPUUserClient : public IOUserClient@@ -260,8 +301,10 @@ class RedwoodGPUUserClient : public IOUserClientuint32_t fSubmits;struct gpu_buffer * buffer(uint32_t handle);+ struct gpu_buffer * newBuffer(IOMemoryDescriptor * memory, uint64_t from, uint64_t size, uint32_t flags, uint32_t * handle);void freeBuffer(struct gpu_buffer * b);IOReturn gemNew(struct msm_gem_new * request);+ IOReturn surfaceBuffer(struct redwood_surface * request);IOReturn gemInfo(struct msm_gem_info * request);IOReturn submit(struct msm_gem_submit * request);IOReturn show(const struct redwood_show * request);@@ -308,6 +351,10 @@ RedwoodGPUUserClient::freeBuffer(struct gpu_buffer * b)}b->memory->release();b->memory = NULL;+ if (b->surface != NULL) {+ MiniIOSurfaceDrop(b->surface);+ b->surface = NULL;+ }fBufferCount--;fBufferBytes -= b->size;}@@ -353,62 +400,141 @@ RedwoodGPUUserClient::buffer(uint32_t handle)return &fBuffers[handle - 1];}-IOReturn-RedwoodGPUUserClient::gemNew(struct msm_gem_new * request)+/*+ * A handle and addresses of the GPU's for `size` bytes of a memory descriptor, from byte `from`+ * of it on (wired memory; both in whole pages). With the lock held; NULL: there is no room.+ */+struct gpu_buffer *+RedwoodGPUUserClient::newBuffer(IOMemoryDescriptor * memory, uint64_t from, uint64_t size, uint32_t flags, uint32_t * handle){- uint64_t size = round_page_64(request->size);struct gpu_buffer * b = NULL;- IOBufferMemoryDescriptor * memory;bool mapped = true;- if (size == 0 || size > GPU_BUFFER_MAX) {- return kIOReturnBadArgument;- }- memory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, kIODirectionInOut | kIOMemoryKernelUserShared,- (vm_size_t) size, PAGE_SIZE);- if (memory == NULL) {- IOLog("RedwoodGPU: no memory for a buffer of %llu bytes (%u buffers, %llu MB with this client)\n", size, fBufferCount,- fBufferBytes >> 20);- return kIOReturnNoMemory;- }- bzero(memory->getBytesNoCopy(), size);- redwood_gpu_to_memory(memory->getBytesNoCopy(), size);-- IOLockLock(gGPULock);for (uint32_t i = 0; i < GPU_BUFFERS; i++) {if (fBuffers[i].memory == NULL) {b = &fBuffers[i];- request->handle = i + 1;+ *handle = i + 1;break;}}if (b == NULL || (b->address = redwood_gpu_address(size)) == 0) {- IOLockUnlock(gGPULock);- memory->release();IOLog("RedwoodGPU: no %s left for a buffer of %llu bytes\n", b ? "addresses" : "handles", size);- return kIOReturnNoMemory;+ return NULL;}for (uint64_t offset = 0; offset < size && mapped; offset += 4096) {IOByteCount length = 0;- addr64_t phys = memory->getPhysicalSegment(offset, &length, kIOMemoryMapperNone);+ addr64_t phys = memory->getPhysicalSegment(from + offset, &length, kIOMemoryMapperNone);- mapped = phys != 0 && redwood_gpu_map(b->address + offset, phys, request->flags & MSM_BO_GPU_READONLY);+ mapped = phys != 0 && redwood_gpu_map(b->address + offset, phys, flags & MSM_BO_GPU_READONLY);}redwood_gpu_mapped();if (!mapped) {redwood_gpu_unmap(b->address, size);redwood_gpu_address_free(b->address, size);- IOLockUnlock(gGPULock);- memory->release();IOLog("RedwoodGPU: a buffer of %llu bytes cannot be entered into the GPU's address space\n", size);- return kIOReturnNoMemory;+ return NULL;}- b->memory = memory;b->size = size;- b->flags = request->flags;+ b->flags = flags;+ b->seed = 0;+ b->seen = false;+ b->surface = NULL;fBufferCount++;fBufferBytes += size;+ return b;+}++IOReturn+RedwoodGPUUserClient::gemNew(struct msm_gem_new * request)+{+ uint64_t size = round_page_64(request->size);+ struct gpu_buffer * b;+ IOBufferMemoryDescriptor * memory;++ if (size == 0 || size > GPU_BUFFER_MAX) {+ return kIOReturnBadArgument;+ }+ /* (the memory comes zeroed, and making it write-combined pushes the zeroes out of the cache) */+ memory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, kIODirectionInOut | kIOMemoryKernelUserShared |+ (gGPUCached ? 0 : kIOMapWriteCombineCache), (vm_size_t) size, PAGE_SIZE);+ if (memory == NULL) {+ IOLog("RedwoodGPU: no memory for a buffer of %llu bytes (%u buffers, %llu MB with this client)\n", size, fBufferCount,+ fBufferBytes >> 20);+ return kIOReturnNoMemory;+ }+ if (gGPUCached) {+ redwood_gpu_to_memory(memory->getBytesNoCopy(), size);+ }++ IOLockLock(gGPULock);+ b = newBuffer(memory, 0, size, request->flags, &request->handle);+ if (b != NULL) {+ b->memory = memory;+ b->kernel = (uint8_t *) memory->getBytesNoCopy();+ }+ IOLockUnlock(gGPULock);+ if (b == NULL) {+ memory->release();+ return kIOReturnNoMemory;+ }+ return kIOReturnSuccess;+}++/*+ * The memory of an IOSurface as a buffer: its pixels, from the page after the surface's shared+ * state on (MiniIOSurface.h). The surface is held for as long as the buffer is there. A surface+ * that is a buffer of this client's already is that buffer again, as with Linux's handles.+ */+IOReturn+RedwoodGPUUserClient::surfaceBuffer(struct redwood_surface * request)+{+ MiniIOSurface * s = MiniIOSurfaceCopy(request->surface);+ struct gpu_buffer * b = NULL;+ IOMemoryDescriptor * memory;+ uint64_t size, began = mach_absolute_time();++ if (s == NULL) {+ return kIOReturnNotFound;+ }+ size = round_page_64(s->info.allocSize);+ memory = IOSubMemoryDescriptor::withSubRange(s->memory, PAGE_SIZE, size, kIODirectionInOut);+ if (memory == NULL || size == 0 || size > GPU_BUFFER_MAX) {+ OSSafeReleaseNULL(memory);+ MiniIOSurfaceDrop(s);+ return kIOReturnNoMemory;+ }+ request->size = size;+ request->offset = s->info.offset;+ request->rowBytes = s->info.bytesPerRow;+ request->width = s->info.width;+ request->height = s->info.height;+ request->pixelFormat = s->info.pixelFormat;+ request->planes = s->info.planeCount;++ IOLockLock(gGPULock);+ for (uint32_t i = 0; i < GPU_BUFFERS; i++) {+ if (fBuffers[i].memory != NULL && fBuffers[i].surface == s) {+ b = &fBuffers[i];+ request->handle = i + 1;+ break;+ }+ }+ if (b != NULL) {+ memory->release();+ MiniIOSurfaceDrop(s);+ } else if ((b = newBuffer(s->memory, PAGE_SIZE, size, 0, &request->handle)) != NULL) {+ b->memory = memory;+ b->kernel = s->kbase + PAGE_SIZE;+ b->surface = s;+ gGPUStat.surfaces++;+ }+ gGPUStat.taking += mach_absolute_time() - began;IOLockUnlock(gGPULock);+ if (b == NULL) {+ memory->release();+ MiniIOSurfaceDrop(s);+ return kIOReturnNoMemory;+ }return kIOReturnSuccess;}@@ -442,6 +568,7 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)struct msm_gem_submit_cmd * cmds = NULL;uint64_t addresses[GPU_SUBMIT_CMDS];uint32_t words[GPU_SUBMIT_CMDS], count = 0;+ uint64_t began, nanos;IOReturn ret = kIOReturnBadArgument;if (request->nr_cmds == 0 || request->nr_cmds > GPU_SUBMIT_CMDS || request->nr_bos == 0 || request->nr_bos > GPU_SUBMIT_BOS) {@@ -461,18 +588,36 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)}IOLockLock(gGPULock);/*- * The program writes its buffers through this processor's cache and the GPU reads memory,- * and the other way round for what the GPU draws: everything the submission names goes out- * of the cache now. (Mapped write-combined, as on Linux, none of this would be needed: see- * redwood_drm_map() in tools/mesa/redwood-drm.)+ * The GPU reads memory and writes memory. What this processor's cache holds of an IOSurface+ * (somebody has written it, or has read it, as the display's driver does, and the GPU is to+ * draw into it again) goes out of the cache now, once for every time the surface is said to+ * have changed (its seed: whoever writes one locks it, and so does a program that has had+ * the GPU draw into one: userland/src/eagl.c). A surface this program has mapped, every time.+ * The buffers made here need none of that, unless they are ordinary memory (gpu_cached=1).*/+ began = mach_absolute_time();for (uint32_t i = 0; i < request->nr_bos; i++) {struct gpu_buffer * b = buffer(bos[i].handle);- if (b != NULL) {- redwood_gpu_to_memory(b->memory->getBytesNoCopy(), b->size);+ if (b == NULL) {+ continue;}+ if (b->surface != NULL) {+ uint32_t seed = b->surface->shared()->seed;++ if (seed == b->seed && !b->seen) {+ continue;+ }+ b->seed = seed;+ } else if (!b->seen || !gGPUCached) {+ continue;+ }+ redwood_gpu_to_memory(b->kernel, b->size);+ gGPUStat.flushed += b->size;}+ redwood_gpu_to_memory(NULL, 0); /* (and what was written last is in memory: a barrier) */+ gGPUStat.flushing += mach_absolute_time() - began;+ gGPUStat.buffers += request->nr_bos;for (uint32_t i = 0; i < request->nr_cmds; i++) {struct gpu_buffer * b = cmds[i].submit_idx < request->nr_bos ? buffer(bos[cmds[i].submit_idx].handle) : NULL;@@ -501,6 +646,31 @@ RedwoodGPUUserClient::submit(struct msm_gem_submit * request)IOLog("RedwoodGPU: submission %u: %u command buffers, the first at %#llx, %u words\n", request->fence, count,addresses[0], words[0]);}+ /* every five seconds in which something was submitted: how much, and what it cost here */+ gGPUStat.submissions++;+ gGPUStat.commands += count;+ if (gGPUStat.since == 0) {+ gGPUStat.since = began;+ }+ absolutetime_to_nanoseconds(mach_absolute_time() - gGPUStat.since, &nanos);+ if (nanos >= 5 * NSEC_PER_SEC) {+ uint64_t flushing, waiting, taking, worked;+ uint32_t timed;++ absolutetime_to_nanoseconds(gGPUStat.flushing, &flushing);+ absolutetime_to_nanoseconds(gGPUStat.waiting, &waiting);+ absolutetime_to_nanoseconds(gGPUStat.taking, &taking);+ redwood_gpu_worked(&worked, &timed);+ IOLog("RedwoodGPU: %u submissions in %llu ms: %u command buffers and %u buffers named, %llu KB out of the cache "+ "in %llu us a submission; the GPU worked %llu us on each of %u; %u waits of %llu us; %u IOSurfaces taken as "+ "buffers in %llu us each; %u buffers, %llu MB\n",+ gGPUStat.submissions, nanos / NSEC_PER_MSEC, gGPUStat.commands, gGPUStat.buffers,+ (gGPUStat.flushed >> 10) / gGPUStat.submissions, flushing / NSEC_PER_USEC / gGPUStat.submissions,+ timed ? worked / timed : 0, timed, gGPUStat.waits, gGPUStat.waits ? waiting / NSEC_PER_USEC / gGPUStat.waits : 0,+ gGPUStat.surfaces, gGPUStat.surfaces ? taking / NSEC_PER_USEC / gGPUStat.surfaces : 0, fBufferCount,+ fBufferBytes >> 20);+ bzero(&gGPUStat, sizeof(gGPUStat));+ }IOLockUnlock(gGPULock);ret = request->fence != 0 ? kIOReturnSuccess : kIOReturnTimeout;out:@@ -548,6 +718,7 @@ RedwoodGPUUserClient::clientMemoryForType(UInt32 type, IOOptionBits * options, Ib = buffer(type);if (b != NULL) {b->memory->retain();+ b->seen = true;*memory = b->memory;*options = 0;ret = kIOReturnSuccess;@@ -571,6 +742,7 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentstruct msm_submitqueue queue;struct msm_submitqueue_query query;struct drm_gem_close close;+ struct redwood_surface surface;struct redwood_show show;} r;size_t size = args->structureInputSize;@@ -640,6 +812,9 @@ RedwoodGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgument}IOLockUnlock(gGPULock);break;+ case kRedwoodSurface:+ ret = surfaceBuffer(&r.surface);+ break;case kRedwoodShow:ret = show(&r.show);break;diff --git a/xnu/files/osfmk/arm64/loader_return.c b/xnu/files/osfmk/arm64/loader_return.cindex a2810f2..49f09bb 100644--- a/xnu/files/osfmk/arm64/loader_return.c+++ b/xnu/files/osfmk/arm64/loader_return.c@@ -44,6 +44,7 @@ extern void gic_v3_irq_quiesce(void);void loader_log_putc(char c);void loader_return(void);extern void redwood_adreno_quiesce(void);+extern void redwood_adreno_stop(void);void loader_checkpoint(uint64_t n);/* Also referenced from the early exception vector in start.s. */@@ -358,6 +359,9 @@ loader_return_timer(__unused thread_call_param_t p0, __unused thread_call_param_loader_log_putc(*c);}PE_parse_boot_argn("return_panic", &as_panic, sizeof(as_panic));+ if (!as_panic) {+ redwood_adreno_stop(); /* a GPU kept for the system is given back while there is a thread to wait in */+ }if (PE_parse_boot_argn("return_cpu", &cpu, sizeof(cpu)) && cpu <= (uint32_t)ml_get_max_cpu_number() &&cpu_xcall((int)cpu, loader_return_test, (void *)(uintptr_t)(as_panic ? 2 : 1)) == KERN_SUCCESS) {for (;;) {@@ -399,6 +403,7 @@ loader_stop_poll(__unused thread_call_param_t p0, __unused thread_call_param_t pfor (const char *c = banner; *c; c++) {loader_log_putc(*c);}+ redwood_adreno_stop();loader_return();}clock_interval_to_deadline(1, NSEC_PER_SEC, &deadline);diff --git a/xnu/redwood.patch b/xnu/redwood.patchindex a197a43..c63247f 100644--- a/xnu/redwood.patch+++ b/xnu/redwood.patch@@ -1661,7 +1661,7 @@ index 39daaf8db949..64ceb1a5f201 100644}os_atomic_clear_exclusive();diff --git a/osfmk/arm/pmap/pmap.c b/osfmk/arm/pmap/pmap.c-index fc2096fa0010..d71b2640cb8c 100644+index fc2096fa0010..5011ab612305 100644--- a/osfmk/arm/pmap/pmap.c+++ b/osfmk/arm/pmap/pmap.c@@ -93,6 +93,8 @@@@ -1684,7 +1684,17 @@ index fc2096fa0010..d71b2640cb8c 100644}goto protect_skip_pve_pass1;}-@@ -11623,7 +11625,8 @@ pmap_cs_lockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)+@@ -6877,7 +6879,8 @@ void+ pmap_sync_page_attributes_phys(+ ppnum_t pp)+ {+- flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE);++ /* (redwood: the page number is 32 bits wide and memory goes on above 4 GB: widen before shifting) */++ flush_dcache((vm_offset_t) pp << PAGE_SHIFT, PAGE_SIZE, TRUE);+ }++ #if CONFIG_COREDUMP+@@ -11623,7 +11626,8 @@ pmap_cs_lockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)#if XNU_MONITORpmap_ppl_lockdown_pages(kva, size, PVH_FLAG_LOCKDOWN_CS, ppl_writable);#else@@ -1694,7 +1704,7 @@ index fc2096fa0010..d71b2640cb8c 100644#endif}-@@ -11633,7 +11636,8 @@ pmap_cs_unlockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)+@@ -11633,7 +11637,8 @@ pmap_cs_unlockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)#if XNU_MONITORpmap_ppl_unlockdown_pages(kva, size, PVH_FLAG_LOCKDOWN_CS, ppl_writable);#else@@ -1764,28 +1774,30 @@ index 94e91306b143..0ec1323086b7 100644#endifdiff --git a/osfmk/arm64/machine_routines.c b/osfmk/arm64/machine_routines.c-index e76e7e70ab9c..d702d171e3d1 100644+index e76e7e70ab9c..a1503e9d28ea 100644--- a/osfmk/arm64/machine_routines.c+++ b/osfmk/arm64/machine_routines.c-@@ -734,6 +734,8 @@ ml_get_machine_mem(void)+@@ -734,6 +734,9 @@ ml_get_machine_mem(void)return machine_info.memory_size;}+extern void loader_return(void);++extern void redwood_adreno_stop(void);+__attribute__((noreturn))voidhalt_all_cpus(boolean_t reboot)-@@ -745,6 +747,8 @@ halt_all_cpus(boolean_t reboot)+@@ -745,6 +748,9 @@ halt_all_cpus(boolean_t reboot)printf("CPU halted\n");PEHaltRestart(kPEHaltCPU);}+ /* No platform driver powers the machine off: hand control (and the log) back to the loader. */++ redwood_adreno_stop();+ loader_return();while (1) {;}-@@ -2237,8 +2241,16 @@ _enable_virtual_timer(void)+@@ -2237,8 +2243,16 @@ _enable_virtual_timer(void)uint64_t cntvctl = CNTV_CTL_EL0_ENABLE; /* One wants to use 32 bits, but "mrs" prefers it this way */__builtin_arm_wsr64("CNTV_CTL_EL0", cntvctl);