Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
UFS: the kernel reads the phone's storage, through Qualcomm's own driver on core 0
vcvkk64e8d53481af
diff --git a/README.md b/README.mdindex bcd9e6c..89f9a5f 100644--- a/README.md+++ b/README.md@@ -398,6 +398,34 @@ P3 поверхности кадров копировались, и этогопальцев); после паники следующий запуск сам останавливает его и запускает заново. Приложения, использующие Metalсами, от этого ничего не получают.+## Накопитель телефона (UFS)++**Ядро читает флеш телефона** — все 112 разделов GPT шести логических единиц UFS, только чтение, ~250 МБ/с.+Своего драйвера UFS в ядре нет, и он не нужен: загрузчик никогда не выходит из UEFI, драйвер Qualcomm+(`UFSDxe`) уже поднял контроллер, его PHY, такты и питание и остаётся жить на ядре 0, где загрузчик и так+обслуживает ядро порта (батарея). С `ufs=2` загрузчик выкладывает общую страницу (`ufs_shared_pa=`) с таблицей+разделов, запросом и ответом и буфер на 4 МБ; ядро пишет запрос — какой раздел, какие блоки, — а ядро 0+читает их через `ReadBlocks` этого драйвера (`xnuloader/loader.c`, `block_poll`; ядро —+`bsd/dev/redwood_ufs.c`). Каждый раздел — `/dev/ufs_ИМЯ` (блочное) и `/dev/rufs_ИМЯ` (символьное), имя — как в+таблице разделов. Нужен `bootcpu=7`: служба живёт в цикле, которым ядро 0 следит за ядром порта, а он есть,+только когда ядро порта работает на другом ядре.++Только чтение — по построению: открытие на запись отказывается, запись, дошедшая до драйвера, отказывается,+а в странице просто нет поля, которым можно было бы попросить записать, — загрузчик зовёт `ReadBlocks` и+больше ничего. Поля, которые пишет ядро, и поля, которые пишет загрузчик, лежат в разных строках кэша, а в+буфер данных ядро 0 само не пишет никогда — поэтому очистка кэша одной стороны не затирает записи другой.++Проверка — `ufs=1` (только загрузчик) или `ufs=2 ufs_check=1` (и ядро), затем `tools/ufs-survey.py`:+- заголовок GPT каждой логической единицы сам себя проверяет (CRC-32 заголовка и таблицы разделов) — у всех+ шести обе суммы верны: прочитанные байты — те, что на флеше, независимо от версии прошивки;+- шестнадцать разделов (`xbl`, `tz`, `hyp`, `abl`, `aop`, `bluetooth`, `dsp`, `modem`, оба слота) ядро+ прочитало через почтовый ящик — 568 запросов, 586 735 616 байт, 0 ошибок — и их хеши совпали с тем, что+ загрузчик прочитал сам до старта ядра. С образами из `fw/fw.zip` они **не** совпадают: на телефоне стоит+ другая версия прошивки (начала разделов при этом верные — ELF у загрузчиков, FAT у `modem` и `bluetooth`).++Загрузчик пишет на флеш с первого коммита проекта ровно одно — свой журнал, в служебный раздел `logdump`+(`write_logs`); к этому драйверу это не относится.+## Снимки экранаВсё снято с экрана телефона за один сеанс: образ `GL=1 REST=1`, запуск с `gpu=1`, касания подаёт сценарийdiff --git a/tools/ufs-survey.py b/tools/ufs-survey.pynew file mode 100644index 0000000..6202187--- /dev/null+++ b/tools/ufs-survey.py@@ -0,0 +1,75 @@+#!/usr/bin/env python3+"""Check what the loader read from the phone's storage against the ROM's own images.++usage: ufs-survey.py [LOG] [FW_ZIP]+ LOG the loader's log of a run with ufs=1 among XL_CMDLINE (default logs/last.log)+ FW_ZIP the ROM's firmware archive (default fw/fw.zip)++With ufs=1 the loader (xnuloader/loader.c, ufs_survey) lists the phone's logical units and GPT+partitions as Qualcomm's own UFS driver in UEFI sees them, and for the partitions there are images+of in the ROM's firmware-update/ it prints an FNV-1a hash of exactly as many bytes as the image+has. This computes the same hash from the images and says, partition by partition, whether the+two agree: where they do, reading through that driver gives the bytes that are on the flash.+With ufs=2 ufs_check=1 the kernel then reads the same partitions through the loader's page+(bsd/dev/redwood_ufs.c) and prints the same hashes: those are shown as kernel:NAME.+Nothing is read from the phone here; it only reads the log.+"""+import re+import sys+import zipfile+++def fnv(data, h=0xcbf29ce484222325):+ for b in data:+ h = ((h ^ b) * 0x100000001b3) & 0xffffffffffffffff+ return h+++def main():+ log_path = sys.argv[1] if len(sys.argv) > 1 else 'logs/last.log'+ zip_path = sys.argv[2] if len(sys.argv) > 2 else 'fw/fw.zip'+ text = open(log_path, encoding='utf-8', errors='replace').read()++ units = re.findall(r'^ufs: unit (\d+): .*$', text, re.M)+ parts = re.findall(r'^ufs: part (\S+): lba (\d+)\.\.(\d+), (\d+) KB$', text, re.M)+ hashes = re.findall(r'^ufs: (\S+): fnv ([0-9a-f]+) over (\d+) bytes, (\d+) ms$', text, re.M)+ # the same partitions read again by the kernel through the loader's page (ufs=2 ufs_check=1)+ checks = re.findall(r'ufs: check (\S+): fnv ([0-9a-f]+) over (\d+) bytes, (\d+) ms', text)+ hashes += [('kernel:' + name, value, size, ms) for name, value, size, ms in checks]+ errors = re.findall(r'^ufs: (\S+): (reading at block .*|.* the partition is smaller)$', text, re.M)+ if not parts:+ sys.exit(f'ufs-survey.py: no "ufs:" lines in {log_path} (was the run made with ufs=1?)')+ print(f'{len(units)} logical units, {len(parts)} partitions in the log')++ images = {}+ with zipfile.ZipFile(zip_path) as z:+ for info in z.infolist():+ m = re.match(r'firmware-update/(.+)\.img$', info.filename)+ if m:+ images[m.group(1)] = info.filename+ agreed = differed = 0+ for name, value, size, ms in hashes:+ base = re.sub(r'_[ab]$', '', name.removeprefix('kernel:'))+ if base not in images:+ print(f' {name}: no image named {base}.img in {zip_path}')+ continue+ data = z.read(images[base])+ if len(data) != int(size):+ print(f' {name}: the image has {len(data)} bytes, the loader hashed {size}')+ differed += 1+ continue+ mine = fnv(data)+ same = mine == int(value, 16)+ agreed += same+ differed += not same+ rate = int(size) / max(int(ms), 1) / 1000+ print(f' {name:<21} {"same" if same else "DIFFERENT":<9} {int(size):>11} bytes '+ f'in {ms:>5} ms ({rate:.0f} MB/s) flash {value} image {mine:x}')+ for name, what in errors:+ print(f' {name}: {what}')+ print(f'{agreed} agree, {differed} differ, {len(errors)} not read')+ return 0 if differed == 0 and not errors and agreed > 0 else 1+++if __name__ == '__main__':+ sys.exit(main())diff --git a/xnu/files/bsd/dev/redwood_input.c b/xnu/files/bsd/dev/redwood_input.cindex 6c584d6..8076cf0 100644--- a/xnu/files/bsd/dev/redwood_input.c+++ b/xnu/files/bsd/dev/redwood_input.c@@ -219,6 +219,7 @@ redwood_input_init(void)redwood_goodix_init();redwood_buttons_init();redwood_battery_init();+ redwood_ufs_init();redwood_net_init();redwood_adreno_init();redwood_wifi_init();diff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.cnew file mode 100644index 0000000..1ffbe36--- /dev/null+++ b/xnu/files/bsd/dev/redwood_ufs.c@@ -0,0 +1,480 @@+/*+ * The phone's storage, read only: one block device and one character device for every GPT+ * partition of its UFS (/dev/ufs_NAME and /dev/rufs_NAME, NAME as the partition table has it).+ *+ * There is no UFS driver here, and none is needed. UEFI never exits on this port: Qualcomm's own+ * UFS driver (UFSDxe) has brought up the controller, its PHY, its clocks and power, and stays+ * resident, and xnuloader keeps serving the kernel from core 0 while the kernel runs on another+ * core (it already does for the battery). So with ufs=2 on its command line the loader publishes a+ * page (ufs_shared_pa=) with the table of partitions, a request and an answer, and a buffer; this+ * driver writes a request -- which partition, which blocks -- and core 0 reads them with that+ * driver's ReadBlocks into the buffer (xnuloader/loader.c, block_poll). Nothing in this kernel+ * touches the storage controller.+ *+ * Read only by construction: opening for writing is refused, a write that reaches the strategy+ * routine is refused, and the page has no field in which a write could be asked for -- the+ * loader's side calls ReadBlocks and nothing else.+ *+ * One request at a time (the page has one), so requests are serialized by a mutex and split into+ * pieces of the buffer's size. The kernel writes only the request's cache lines and core 0 only+ * the answer's (see the loader), both through mappings that do not cache here, so neither side+ * ever overwrites the other's.+ *+ * ufs_check=1 reads, once at start, the partitions there are images of in the ROM's+ * firmware-update/ (bluetooth, xbl, tz, ...) through the same path and prints an FNV-1a hash of+ * as many bytes as each image has: tools/ufs-survey.py compares those with the images and with+ * what the loader read itself before the kernel started.+ */+#include <kern/clock.h>+#include <kern/locks.h>+#include <kern/thread.h>+#include <libkern/libkern.h>+#include <mach/mach_types.h>+#include <mach/vm_param.h>+#include <miscfs/devfs/devfs.h>+#include <pexpert/pexpert.h>+#include <sys/buf.h>+#include <sys/conf.h>+#include <sys/disk.h>+#include <sys/errno.h>+#include <sys/systm.h>+#include <sys/sysctl.h>+#include <sys/redwood_input.h>+#include <vm/memory_types.h>++extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);+extern void IOSleep(unsigned int milliseconds);++/* the page the loader publishes: the same layout as struct loader_block in xnuloader/loader.c */+#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */+#define BLOCK_VERSION 1+#define BLOCK_MAX_PARTS 128++struct loader_block_part {+ char name[36];+ uint32_t block_size;+ uint64_t blocks;+};++struct loader_block {+ uint64_t magic;+ uint32_t version, size;+ uint32_t parts, buffer_size;+ uint64_t buffer_pa;+ struct {+ volatile uint32_t request;+ uint32_t part;+ uint64_t lba, count;+ } ask __attribute__((aligned(128)));+ struct {+ volatile uint32_t answered;+ uint32_t reserved;+ int64_t status;+ uint64_t served, failed, bytes, ticks;+ } answer __attribute__((aligned(128)));+ struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));+};++#define UFS_PAGE 0x4000UL+#define UFS_TIMEOUT_MS 20000 /* core 0 answers in well under a second; this is for a dead loader */++static volatile struct loader_block *ufs_page;+static const uint8_t *ufs_buffer;+static uint32_t ufs_buffer_size, ufs_parts, ufs_seq;+static uint32_t ufs_bsize[BLOCK_MAX_PARTS]; /* the block size each device is counted in (spec_open sets 512) */+static int ufs_bmajor = -1, ufs_cmajor = -1;+static bool ufs_ready, ufs_dead;+static uint64_t ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us;++static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");+static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);++SYSCTL_DECL(_kern_redwood);+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_requests,+ "requests the phone's storage has answered through the loader");+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_errors,+ "requests the phone's storage has refused or not answered");+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_bytes,+ "bytes read from the phone's storage");+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_wait_us,+ "microseconds spent waiting for core 0's answers");++/*+ * Native blocks [lba, lba + count) of partition part, into dst: one request of at most the+ * buffer's size. Called with ufs_lock held.+ */+static int+ufs_request(uint32_t part, uint64_t lba, uint64_t count, uint8_t *dst)+{+ uint64_t bytes = count * ufs_page->part[part].block_size, started, waited_ms = 0;+ uint32_t seq = ++ufs_seq;+ int64_t status;++ if (ufs_dead) {+ return EIO;+ }+ ufs_page->ask.part = part;+ ufs_page->ask.lba = lba;+ ufs_page->ask.count = count;+ __asm__ volatile ("dmb sy" ::: "memory");+ ufs_page->ask.request = seq;+ __asm__ volatile ("dmb sy" ::: "memory");++ started = mach_absolute_time();+ while (ufs_page->answer.answered != seq) {+ /* spin briefly for a quick answer, then sleep a millisecond at a time */+ uint64_t us;++ absolutetime_to_nanoseconds(mach_absolute_time() - started, &us);+ us /= 1000;+ if (us < 2000) {+ delay(20);+ continue;+ }+ IOSleep(1);+ if (++waited_ms > UFS_TIMEOUT_MS) {+ /* no answer: the loader is gone or stuck; nothing more is asked of it */+ printf("ufs: no answer from the loader to request %u in %u s: the storage is off from now on\n",+ seq, UFS_TIMEOUT_MS / 1000);+ ufs_dead = true;+ ufs_errors++;+ return EIO;+ }+ }+ __asm__ volatile ("dmb sy" ::: "memory");+ {+ uint64_t ns;++ absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);+ ufs_wait_us += ns / 1000;+ }+ status = ufs_page->answer.status;+ if (status != 0) {+ printf("ufs: %s: blocks %llu..%llu: the loader's read failed (status %llx)\n",+ ufs_page->part[part].name, lba, lba + count - 1, (uint64_t)status);+ ufs_errors++;+ return EIO;+ }+ memcpy(dst, ufs_buffer, (size_t)bytes);+ ufs_requests++;+ ufs_bytes += bytes;+ return 0;+}++/*+ * Bytes [offset, offset + len) of partition part into dst, whatever their alignment: the native+ * blocks that cover them are read a buffer at a time and the wanted bytes copied out.+ */+static int+ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)+{+ uint64_t nbs = ufs_page->part[part].block_size, per = ufs_buffer_size / nbs;+ static uint8_t *bounce;+ int error = 0;++ if (bounce == NULL) {+ bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);+ if (bounce == NULL) {+ return ENOMEM;+ }+ }+ lck_mtx_lock(&ufs_lock);+ while (len > 0 && error == 0) {+ uint64_t first = offset / nbs, skip = offset % nbs;+ uint64_t count = (skip + len + nbs - 1) / nbs, take;++ if (count > per) {+ count = per;+ }+ take = count * nbs - skip;+ if (take > len) {+ take = len;+ }+ if (skip == 0 && take == count * nbs) {+ error = ufs_request(part, first, count, dst); /* whole blocks: straight in */+ } else {+ error = ufs_request(part, first, count, bounce);+ if (error == 0) {+ memcpy(dst, bounce + skip, (size_t)take);+ }+ }+ offset += take;+ dst += take;+ len -= take;+ }+ lck_mtx_unlock(&ufs_lock);+ return error;+}++static uint64_t+ufs_part_bytes(uint32_t part)+{+ return ufs_page->part[part].blocks * ufs_page->part[part].block_size;+}++/* ---- the devices ---- */++static int+ufs_open(dev_t dev, int flags, __unused int devtype, __unused proc_t p)+{+ if (!ufs_ready || (uint32_t)minor(dev) >= ufs_parts) {+ return ENXIO;+ }+ if (flags & FWRITE) {+ return EROFS;+ }+ return 0;+}++static int+ufs_close(__unused dev_t dev, __unused int flags, __unused int devtype, __unused proc_t p)+{+ return 0;+}++static void+ufs_strategy(buf_t bp)+{+ uint32_t part = (uint32_t)minor(buf_device(bp));+ uint64_t offset, len, size;+ int error;++ if (!ufs_ready || part >= ufs_parts) {+ buf_seterror(bp, ENXIO);+ buf_biodone(bp);+ return;+ }+ if (!(buf_flags(bp) & B_READ)) {+ buf_seterror(bp, EROFS); /* this device never writes */+ buf_biodone(bp);+ return;+ }+ size = ufs_part_bytes(part);+ offset = (uint64_t)buf_blkno(bp) * ufs_bsize[part];+ len = buf_count(bp);+ if (offset >= size) {+ buf_seterror(bp, offset == size ? 0 : EINVAL);+ buf_setresid(bp, buf_count(bp));+ buf_biodone(bp);+ return;+ }+ if (offset + len > size) {+ len = size - offset;+ }+ error = ufs_read(part, offset, (uint8_t *)buf_dataptr(bp), len);+ buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : len)));+ if (error) {+ buf_seterror(bp, error);+ }+ buf_biodone(bp);+}++static int+ufs_ioctl(dev_t dev, u_long cmd, caddr_t data, __unused int fflag, __unused proc_t p)+{+ uint32_t part = (uint32_t)minor(dev), native;++ if (!ufs_ready || part >= ufs_parts) {+ return ENXIO;+ }+ native = ufs_page->part[part].block_size;+ switch (cmd) {+ case DKIOCGETBLOCKSIZE:+ *(uint32_t *)data = ufs_bsize[part];+ return 0;+ case DKIOCSETBLOCKSIZE:+ /*+ * spec_open() sets the block size to 512 to count the device in 512-byte blocks and+ * then back, and treats a refusal as a broken device; reads are by byte here, so any+ * power of two from 512 to the native block size will do.+ */+ if (*(uint32_t *)data < 512 || *(uint32_t *)data > native ||+ (*(uint32_t *)data & (*(uint32_t *)data - 1)) != 0) {+ return EINVAL;+ }+ ufs_bsize[part] = *(uint32_t *)data;+ return 0;+ case DKIOCGETPHYSICALBLOCKSIZE:+ *(uint32_t *)data = native;+ return 0;+ case DKIOCGETBLOCKCOUNT:+ *(uint64_t *)data = ufs_part_bytes(part) / ufs_bsize[part];+ return 0;+ case DKIOCISWRITABLE:+ *(uint32_t *)data = 0;+ return 0;+ case DKIOCGETMAXBLOCKCOUNTREAD:+ *(uint64_t *)data = ufs_buffer_size / ufs_bsize[part];+ return 0;+ default:+ return ENOTTY;+ }+}++static int+ufs_psize(dev_t dev)+{+ uint32_t part = (uint32_t)minor(dev);++ return ufs_ready && part < ufs_parts ? (int)ufs_bsize[part] : -1;+}++static int+ufs_cread(dev_t dev, struct uio *uio, __unused int ioflag)+{+ return physio(ufs_strategy, NULL, dev, B_READ, (u_int (*)(buf_t))ufs_psize, uio,+ ufs_bsize[minor(dev) < BLOCK_MAX_PARTS ? minor(dev) : 0]);+}++static const struct bdevsw ufs_bdevsw = {+ .d_open = ufs_open,+ .d_close = ufs_close,+ .d_strategy = ufs_strategy,+ .d_ioctl = ufs_ioctl,+ .d_dump = eno_dump,+ .d_psize = ufs_psize,+ .d_type = D_DISK,+};++static const struct cdevsw ufs_cdevsw = {+ .d_open = ufs_open,+ .d_close = ufs_close,+ .d_read = ufs_cread,+ .d_write = eno_rdwrt,+ .d_ioctl = ufs_ioctl,+ .d_stop = eno_stop,+ .d_reset = eno_reset,+ .d_ttys = NULL,+ .d_select = eno_select,+ .d_mmap = eno_mmap,+ .d_strategy = eno_strat,+ .d_reserved_1 = eno_getc,+ .d_reserved_2 = eno_putc,+ .d_type = D_DISK,+};++/* ---- the check: the partitions there are images of, hashed as the loader hashes them ---- */++static const struct {+ const char *name;+ uint64_t bytes;+} ufs_references[] = {+ { "bluetooth", 503808 }, { "abl", 208896 }, { "aop", 208896 }, { "xbl", 3706880 },+ { "tz", 3710976 }, { "hyp", 3653632 }, { "dsp", 67108864 }, { "modem", 214265856 },+};++static bool+ufs_is_reference(const char *name, const char *ref)+{+ size_t n = strlen(ref);++ return strncmp(name, ref, n) == 0 &&+ (name[n] == 0 || (name[n] == '_' && (name[n + 1] == 'a' || name[n + 1] == 'b') && name[n + 2] == 0));+}++static void+ufs_check(__unused void *arg, __unused wait_result_t wr)+{+ uint8_t *chunk = kalloc_data(1 << 20, Z_WAITOK);++ /*+ * Not at once: what the kernel prints in its first minute or so does not come back with the+ * log on the phone (the first line that does is from long after this driver has started), so+ * the check waits until it will be read.+ */+ IOSleep(40000);+ printf("ufs: check: %u partitions of the phone's storage, read through the loader's page\n", ufs_parts);+ for (uint32_t part = 0; chunk != NULL && part < ufs_parts; part++) {+ for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {+ uint64_t h = 0xcbf29ce484222325ULL, done = 0, started = mach_absolute_time(), ns;+ uint64_t bytes = ufs_references[r].bytes;+ int error = 0;++ if (!ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {+ continue;+ }+ while (done < bytes && error == 0) {+ uint64_t n = bytes - done < (1 << 20) ? bytes - done : (1 << 20);++ error = ufs_read(part, done, chunk, n);+ for (uint64_t i = 0; error == 0 && i < n; i++) {+ h = (h ^ chunk[i]) * 0x100000001b3ULL;+ }+ done += n;+ }+ absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);+ if (error) {+ printf("ufs: check %s: read failed (%d)\n", ufs_page->part[part].name, error);+ } else {+ printf("ufs: check %s: fnv %llx over %llu bytes, %llu ms\n", ufs_page->part[part].name,+ h, bytes, ns / 1000000);+ }+ }+ }+ if (chunk != NULL) {+ kfree_data(chunk, 1 << 20);+ }+ printf("ufs: check done: %llu requests, %llu errors, %llu bytes, %llu ms waiting\n",+ ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us / 1000);+}++/* ---- bringing it up ---- */++void+redwood_ufs_init(void)+{+ uint64_t pa = 0, buffer_pa;+ uint32_t check = 0, made = 0;+ vm_offset_t base;++ if (!PE_parse_boot_argn("ufs_shared_pa", &pa, sizeof(pa)) || pa == 0) {+ return; /* the loader was not asked for ufs=2: there is nothing to read through */+ }+ base = ml_io_map_unmappable((vm_offset_t)(pa & ~(UFS_PAGE - 1)),+ ((pa & (UFS_PAGE - 1)) + sizeof(struct loader_block) + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_WCOMB);+ ufs_page = (volatile struct loader_block *)(base + (pa & (UFS_PAGE - 1)));+ if (ufs_page->magic != BLOCK_MAGIC || ufs_page->version != BLOCK_VERSION ||+ ufs_page->size != sizeof(struct loader_block) || ufs_page->parts > BLOCK_MAX_PARTS) {+ printf("ufs: the loader's page at %llx is not one this kernel knows (magic %llx, version %u, size %u)\n",+ pa, ufs_page->magic, ufs_page->version, ufs_page->size);+ return;+ }+ ufs_parts = ufs_page->parts;+ ufs_buffer_size = ufs_page->buffer_size;+ buffer_pa = ufs_page->buffer_pa;+ ufs_buffer = (const uint8_t *)(ml_io_map_unmappable((vm_offset_t)(buffer_pa & ~(UFS_PAGE - 1)),+ ((buffer_pa & (UFS_PAGE - 1)) + ufs_buffer_size + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_WCOMB) ++ (buffer_pa & (UFS_PAGE - 1)));+ ufs_seq = ufs_page->answer.answered; /* a loader that served an earlier kernel counts on from there */++ ufs_bmajor = bdevsw_add(-1, &ufs_bdevsw);+ ufs_cmajor = ufs_bmajor < 0 ? -1 : cdevsw_add_with_bdev(-1, &ufs_cdevsw, ufs_bmajor);+ if (ufs_cmajor < 0) {+ printf("ufs: no device numbers to be had\n");+ return;+ }+ for (uint32_t i = 0; i < ufs_parts; i++) {+ char name[48];++ ufs_bsize[i] = ufs_page->part[i].block_size;+ snprintf(name, sizeof(name), "ufs_%s", ufs_page->part[i].name);+ if (devfs_make_node(makedev(ufs_bmajor, i), DEVFS_BLOCK, UID_ROOT, GID_OPERATOR, 0640, "%s", name) == NULL) {+ printf("ufs: no node for %s (the same name twice?)\n", name);+ continue;+ }+ snprintf(name, sizeof(name), "rufs_%s", ufs_page->part[i].name);+ (void)devfs_make_node(makedev(ufs_cmajor, i), DEVFS_CHAR, UID_ROOT, GID_OPERATOR, 0640, "%s", name);+ made++;+ }+ ufs_ready = true;+ printf("ufs: %u partitions of the phone's storage as /dev/ufs_NAME and /dev/rufs_NAME, read only, "+ "through the loader's page at %llx and its %u KB buffer\n", made, pa, ufs_buffer_size >> 10);++ if (PE_parse_boot_argn("ufs_check", &check, sizeof(check)) && check != 0) {+ thread_t thread;++ if (kernel_thread_start(ufs_check, NULL, &thread) == KERN_SUCCESS) {+ thread_deallocate(thread);+ }+ }+}diff --git a/xnu/files/bsd/sys/redwood_input.h b/xnu/files/bsd/sys/redwood_input.hindex bdfb148..5edde86 100644--- a/xnu/files/bsd/sys/redwood_input.h+++ b/xnu/files/bsd/sys/redwood_input.h@@ -42,6 +42,7 @@ void redwood_input_init(void);void redwood_virtio_input_init(void); /* boot-arg virtio_input=1: QEMU's virtio pointer and keyboard */void redwood_goodix_init(void); /* boot-arg touchscreen=1: the phone's Goodix or FocalTech touch screen */void redwood_buttons_init(void); /* boot-arg buttons=1: the phone's power and volume keys */+void redwood_ufs_init(void); /* ufs=2 to xnuloader: the phone's storage, read only, through the loader (ufs_shared_pa=) */void redwood_battery_init(void); /* boot-arg battery=1: Qualcomm UEFI charger state shared by xnuloader */int redwood_battery_temperature(void); /* the battery's temperature in C, -273 while it is not known *//* one transfer at a time through the PMIC arbiter for this system (bsd/dev/redwood_buttons.c) */diff --git a/xnu/redwood.patch b/xnu/redwood.patchindex eada4c3..157db95 100644--- a/xnu/redwood.patch+++ b/xnu/redwood.patch@@ -46,10 +46,10 @@ index 222158a93918..68bb0f3e56bf 100755####################################################################### HISTORYdiff --git a/bsd/conf/files b/bsd/conf/files-index 44274016a4cf..9f1fe3efd685 100644+index 44274016a4cf..94a03cc34d2c 100644--- a/bsd/conf/files+++ b/bsd/conf/files-@@ -109,6 +109,31 @@ bsd/dev/munge.c optional config_requires_u32_munging+@@ -109,6 +109,32 @@ bsd/dev/munge.c optional config_requires_u32_mungingbsd/dev/unix_startup.c standardbsd/dev/monotonic.c optional monotonic@@ -58,6 +58,7 @@ index 44274016a4cf..9f1fe3efd685 100644+bsd/dev/redwood_goodix.c standard+bsd/dev/redwood_buttons.c standard+bsd/dev/redwood_battery.c standard++bsd/dev/redwood_ufs.c standard+bsd/dev/redwood_net.c standard+bsd/dev/redwood_virtio_net.c standard+bsd/dev/redwood_apple80211.c standard@@ -81,7 +82,7 @@ index 44274016a4cf..9f1fe3efd685 100644bsd/dev/dev_perfmon.c optional config_perfmonbsd/libkern/crc32.c standard-@@ -155,6 +180,10 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs+@@ -155,6 +181,10 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfsbsd/miscfs/mockfs/mockfs_fsnode.c optional mockfsbsd/miscfs/mockfs/mockfs_vfsops.c optional mockfsbsd/miscfs/mockfs/mockfs_vnops.c optional mockfs@@ -92,7 +93,7 @@ index 44274016a4cf..9f1fe3efd685 100644bsd/miscfs/routefs/routefs_ops.c optional routefs-@@ -361,6 +390,7 @@ bsd/security/audit/audit_syscalls.c standard+@@ -361,6 +391,7 @@ bsd/security/audit/audit_syscalls.c standardbsd/security/audit/audit_worker.c optional config_auditbsd/pthread/pthread_shims.c standard@@ -100,7 +101,7 @@ index 44274016a4cf..9f1fe3efd685 100644bsd/pthread/pthread_priority.c standardbsd/pthread/pthread_workqueue.c standard-@@ -383,6 +413,7 @@ bsd/kern/kern_debug.c standard+@@ -383,6 +414,7 @@ bsd/kern/kern_debug.c standardbsd/kern/kern_symfile.c standardbsd/kern/kern_descrip.c standardbsd/kern/kern_guarded.c standarddiff --git a/xnuloader/loader.c b/xnuloader/loader.cindex 741de39..29f6bf3 100644--- a/xnuloader/loader.c+++ b/xnuloader/loader.c@@ -1073,6 +1073,364 @@ charger_poll(void)(uint64_t)(uintptr_t)&charger_shared + sizeof(charger_shared));}+/* ------------------------------------------------------------------------ */+/* UFS survey (ufs=1): what the phone's storage holds, read only */++/*+ * The phone's storage as Qualcomm's own UFS driver in this UEFI sees it (UFSDxe): every raw+ * logical unit, every GPT partition with its name, place and size, and for the partitions there+ * are reference images of -- firmware-update/ in the ROM's fw.zip -- an FNV-1a hash of exactly+ * as many bytes as the image has, which tools/ufs-survey.py computes from the images on the host.+ * Where the two agree, reading through this driver gives the bytes that are on the flash, and that+ * is the ground a block device for the kernel is to stand on.+ *+ * Only read_blocks is ever called here: nothing is written, flushed or reset. It runs before the+ * kernel is started, with UEFI's boot services whole and interrupts on.+ */+static const struct {+ const char *name;+ uint64_t bytes;+} ufs_references[] = {+ { "bluetooth", 503808 }, { "abl", 208896 }, { "aop", 208896 }, { "xbl", 3706880 },+ { "tz", 3710976 }, { "hyp", 3653632 }, { "dsp", 67108864 }, { "modem", 214265856 },+};++static uint8_t ufs_buffer[1024 * 1024] __attribute__((aligned(4096)));++static uint64_t+ufs_fnv(uint64_t h, const uint8_t *p, uint64_t n)+{+ for (uint64_t i = 0; i < n; i++) {+ h = (h ^ p[i]) * 0x100000001b3ULL;+ }+ return h;+}++/* the partition's name in ASCII (GPT names are UTF-16; these are all plain letters) */+static void+ufs_name(const efi_partition_info *info, char *out, unsigned size)+{+ unsigned i = 0;++ for (; i + 1 < size && i < 36 && info->name[i] != 0; i++) {+ out[i] = info->name[i] < 0x80 ? (char)info->name[i] : '?';+ }+ out[i] = 0;+}++/* "bluetooth_a" and "bluetooth_b" are both "bluetooth": the image is the same for either slot */+static int+ufs_reference_of(const char *name)+{+ for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {+ const char *a = ufs_references[r].name, *b = name;++ while (*a && *a == *b) {+ a++;+ b++;+ }+ if (*a == 0 && (*b == 0 || ((b[0] == '_') && (b[1] == 'a' || b[1] == 'b') && b[2] == 0))) {+ return (int)r;+ }+ }+ return -1;+}++static void+ufs_hash_partition(efi_block_io *bio, const char *name, uint64_t bytes)+{+ uint64_t bs = bio->media->block_size, h = 0xcbf29ce484222325ULL, done = 0, started = READ_SYSREG(CNTVCT_EL0);+ uint64_t chunk = sizeof(ufs_buffer) / bs * bs, lba = 0;++ if (bs == 0 || bs > sizeof(ufs_buffer) || (bio->media->last_block + 1) * bs < bytes) {+ print("ufs: %s: %d bytes asked for, the partition is smaller\n", name, bytes);+ return;+ }+ while (done < bytes) {+ uint64_t want = bytes - done < chunk ? (bytes - done + bs - 1) / bs * bs : chunk;+ efi_status s = bio->read_blocks(bio, bio->media->media_id, lba, want, ufs_buffer);++ if (EFI_ERROR(s)) {+ print("ufs: %s: reading at block %d: error %x\n", name, lba, (uint64_t)s);+ return;+ }+ if (done == 0) {+ print("ufs: %s: begins", name);+ for (int k = 0; k < 16; k++) {+ print(" %x", (uint64_t)ufs_buffer[k]);+ }+ print("\n");+ }+ h = ufs_fnv(h, ufs_buffer, bytes - done < want ? bytes - done : want);+ done += bytes - done < want ? bytes - done : want;+ lba += want / bs;+ if (wdt_armed) {+ wdt_pet();+ }+ }+ print("ufs: %s: fnv %x over %d bytes, %d ms\n", name, h, bytes,+ (READ_SYSREG(CNTVCT_EL0) - started) * 1000 / READ_SYSREG(CNTFRQ_EL0));+}++/* CRC-32 as GPT uses it (IEEE 802.3, reflected, 0xedb88320) */+static uint32_t+ufs_crc32(const uint8_t *p, uint64_t n)+{+ uint32_t c = 0xffffffffu;++ for (uint64_t i = 0; i < n; i++) {+ c ^= p[i];+ for (int k = 0; k < 8; k++) {+ c = (c >> 1) ^ (0xedb88320u & (0u - (c & 1)));+ }+ }+ return ~c;+}++/*+ * A unit's GPT checks itself: its header carries a CRC-32 of the header and one of the table of+ * partitions. Whatever firmware is on the phone, the bytes read are the bytes on the flash only if+ * both come out right -- the one check here that does not depend on having the same images.+ */+static void+ufs_check_gpt(efi_block_io *bio, uint64_t unit)+{+ uint64_t bs = bio->media->block_size, entries_lba, count, size, bytes, read;+ uint32_t header_size, header_crc, entries_crc, got_header, got_entries;+ uint8_t header[512];++ if (bs < 512 || bs > sizeof(ufs_buffer) ||+ EFI_ERROR(bio->read_blocks(bio, bio->media->media_id, 1, bs, ufs_buffer))) {+ print("ufs: unit %d: the GPT header could not be read\n", unit);+ return;+ }+ if (ufs_buffer[0] != 'E' || ufs_buffer[1] != 'F' || ufs_buffer[2] != 'I' || ufs_buffer[3] != ' ' ||+ ufs_buffer[4] != 'P' || ufs_buffer[5] != 'A' || ufs_buffer[6] != 'R' || ufs_buffer[7] != 'T') {+ print("ufs: unit %d: no GPT header in block 1\n", unit);+ return;+ }+ memcpy(header, ufs_buffer, sizeof(header));+ header_size = *(uint32_t *)(header + 12);+ header_crc = *(uint32_t *)(header + 16);+ entries_lba = *(uint64_t *)(header + 72);+ count = *(uint32_t *)(header + 80);+ size = *(uint32_t *)(header + 84);+ entries_crc = *(uint32_t *)(header + 88);+ if (header_size < 92 || header_size > sizeof(header)) {+ print("ufs: unit %d: GPT header size %d\n", unit, (uint64_t)header_size);+ return;+ }+ *(uint32_t *)(header + 16) = 0;+ got_header = ufs_crc32(header, header_size);+ bytes = count * size;+ if (bytes == 0 || bytes > sizeof(ufs_buffer)) {+ print("ufs: unit %d: GPT header crc %s, a table of %d bytes not checked\n", unit,+ got_header == header_crc ? "right" : "WRONG", bytes);+ return;+ }+ read = (bytes + bs - 1) / bs * bs;+ if (EFI_ERROR(bio->read_blocks(bio, bio->media->media_id, entries_lba, read, ufs_buffer))) {+ print("ufs: unit %d: the GPT table could not be read\n", unit);+ return;+ }+ got_entries = ufs_crc32(ufs_buffer, bytes);+ print("ufs: unit %d: GPT header crc %s (%x), table of %d entries crc %s (%x)\n", unit,+ got_header == header_crc ? "right" : "WRONG", (uint64_t)got_header, count,+ got_entries == entries_crc ? "right" : "WRONG", (uint64_t)got_entries);+}++static void+ufs_survey(void)+{+ static const efi_guid part_guid = EFI_PARTITION_INFO_GUID;+ static const efi_guid blk_guid = EFI_BLOCK_IO_GUID;+ efi_handle *handles;+ uint64_t count, units = 0, partitions = 0;++ /* the raw logical units: the block devices that are not partitions of another */+ if (!EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &blk_guid, NULL, &count, &handles))) {+ for (uint64_t i = 0; i < count; i++) {+ efi_block_io *bio;++ if (EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio)) || bio->media->logical) {+ continue;+ }+ print("ufs: unit %d: media %d, %d blocks of %d (%d MB), %s, align %d\n", units++,+ (uint64_t)bio->media->media_id, bio->media->last_block + 1, (uint64_t)bio->media->block_size,+ (bio->media->last_block + 1) * bio->media->block_size >> 20,+ bio->media->read_only ? "read only" : "writable", (uint64_t)bio->media->io_align);+ ufs_check_gpt(bio, units - 1);+ }+ }+ /* the GPT partitions, with the hash of those there are images of */+ if (EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &part_guid, NULL, &count, &handles))) {+ print("ufs: no partition information in this UEFI\n");+ return;+ }+ for (uint64_t i = 0; i < count; i++) {+ efi_partition_info *info;+ efi_block_io *bio;+ char name[40];+ int r;++ if (EFI_ERROR(BS->handle_protocol(handles[i], &part_guid, (void **)&info)) || info->type != 2 ||+ EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio))) {+ continue;+ }+ ufs_name(info, name, sizeof(name));+ partitions++;+ print("ufs: part %s: lba %d..%d, %d KB\n", name, info->start_lba, info->end_lba,+ (bio->media->last_block + 1) * bio->media->block_size >> 10);+ if ((r = ufs_reference_of(name)) >= 0) {+ ufs_hash_partition(bio, name, ufs_references[r].bytes);+ }+ }+ print("ufs: %d raw units, %d partitions\n", units, partitions);+}++/* ------------------------------------------------------------------------ */+/* UFS block service (ufs=2): the kernel reads the phone's storage through */+/* this loader, and this loader through Qualcomm's own UFS driver */++/*+ * The kernel has no driver for the phone's storage, and does not need one: UEFI never exits here,+ * Qualcomm's UFS driver (UFSDxe) has the controller, its PHY, clocks and power up and stays+ * resident, and this loader already serves the kernel from core 0 while the kernel runs on another+ * (the battery). So the kernel asks for blocks through a page both can see, and core 0 reads them+ * with that driver's ReadBlocks into a buffer both can see. The kernel's side is+ * bsd/dev/redwood_ufs.c; it gets the page's address as ufs_shared_pa= on its command line.+ *+ * Read only, by construction: this service calls read_blocks and nothing else, the kernel's side+ * refuses writes before they are asked for, and there is no field in the page that could ask for+ * one.+ *+ * Each side writes its own cache lines only: the kernel the request, core 0 the answer (and the+ * table, once, before the kernel starts). A line one side cleans out of its cache therefore never+ * carries a stale copy of what the other side wrote into it. Core 0 never writes into the data+ * buffer itself, so after the controller's DMA there is nothing of its own in its cache to write+ * back over the data.+ */+#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */+#define BLOCK_VERSION 1+#define BLOCK_MAX_PARTS 128+#define BLOCK_BUFFER_SIZE (4u << 20)++struct loader_block_part {+ char name[36];+ uint32_t block_size;+ uint64_t blocks;+};++struct loader_block {+ /* written by the loader, once, before the kernel starts */+ uint64_t magic;+ uint32_t version, size;+ uint32_t parts, buffer_size;+ uint64_t buffer_pa;+ /* written by the kernel: what it wants, then request is advanced */+ struct {+ volatile uint32_t request;+ uint32_t part;+ uint64_t lba, count; /* in the partition's own blocks */+ } ask __attribute__((aligned(128)));+ /* written by the loader: answered becomes request when the data is in the buffer */+ struct {+ volatile uint32_t answered;+ uint32_t reserved;+ int64_t status; /* the EFI status of the read, 0 when it went well */+ uint64_t served, failed, bytes, ticks;+ } answer __attribute__((aligned(128)));+ struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));+};++/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */+static struct loader_block block_shared __attribute__((aligned(0x2000)));+static uint8_t block_buffer[BLOCK_BUFFER_SIZE] __attribute__((aligned(0x2000)));+static efi_block_io *block_bios[BLOCK_MAX_PARTS];+static int block_enabled;++static void+block_start(void)+{+ static const efi_guid part_guid = EFI_PARTITION_INFO_GUID;+ static const efi_guid blk_guid = EFI_BLOCK_IO_GUID;+ efi_handle *handles;+ uint64_t count;++ memset(&block_shared, 0, sizeof(block_shared));+ if (EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &part_guid, NULL, &count, &handles))) {+ print("ufs: no partition information: the block service is off\n");+ return;+ }+ for (uint64_t i = 0; i < count && block_shared.parts < BLOCK_MAX_PARTS; i++) {+ struct loader_block_part *bp = &block_shared.part[block_shared.parts];+ efi_partition_info *info;+ efi_block_io *bio;++ if (EFI_ERROR(BS->handle_protocol(handles[i], &part_guid, (void **)&info)) || info->type != 2 ||+ EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio)) ||+ bio->media->block_size == 0 || bio->media->block_size > BLOCK_BUFFER_SIZE) {+ continue;+ }+ ufs_name(info, bp->name, sizeof(bp->name));+ bp->block_size = bio->media->block_size;+ bp->blocks = bio->media->last_block + 1;+ block_bios[block_shared.parts++] = bio;+ }+ block_shared.magic = BLOCK_MAGIC;+ block_shared.version = BLOCK_VERSION;+ block_shared.size = sizeof(block_shared);+ block_shared.buffer_size = BLOCK_BUFFER_SIZE;+ block_shared.buffer_pa = (uint64_t)(uintptr_t)block_buffer;+ clean_dcache((uint64_t)(uintptr_t)&block_shared, (uint64_t)(uintptr_t)&block_shared + sizeof(block_shared));+ block_enabled = 1;+ print("ufs: block service: %d partitions, page %x, buffer %x of %d KB\n", (uint64_t)block_shared.parts,+ (uint64_t)(uintptr_t)&block_shared, block_shared.buffer_pa, (uint64_t)(BLOCK_BUFFER_SIZE >> 10));+}++/* one request, if there is one: called from core 0's watch loop while the kernel runs */+static void+block_poll(void)+{+ uint64_t ask = (uint64_t)(uintptr_t)&block_shared.ask;+ uint32_t request, part;+ uint64_t lba, count, bytes, started;+ efi_status s = 0x8000000000000002ULL; /* EFI_INVALID_PARAMETER: a request out of bounds */++ /* the kernel writes the request into memory past this core's cache: look at memory */+ flush_dcache(ask, ask + sizeof(block_shared.ask));+ request = block_shared.ask.request;+ if (request == block_shared.answer.answered) {+ return;+ }+ part = block_shared.ask.part;+ lba = block_shared.ask.lba;+ count = block_shared.ask.count;+ started = READ_SYSREG(CNTVCT_EL0);+ bytes = 0;+ if (part < block_shared.parts && count != 0 && count <= BLOCK_BUFFER_SIZE / block_shared.part[part].block_size &&+ lba < block_shared.part[part].blocks && count <= block_shared.part[part].blocks - lba) {+ efi_block_io *bio = block_bios[part];++ bytes = count * block_shared.part[part].block_size;+ s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);+ /* what the controller wrote is in memory; drop whatever this core held of the buffer */+ flush_dcache((uint64_t)(uintptr_t)block_buffer, (uint64_t)(uintptr_t)block_buffer + bytes);+ }+ block_shared.answer.status = (int64_t)s;+ block_shared.answer.ticks += READ_SYSREG(CNTVCT_EL0) - started;+ if (EFI_ERROR(s)) {+ block_shared.answer.failed++;+ } else {+ block_shared.answer.served++;+ block_shared.answer.bytes += bytes;+ }+ __asm__ volatile ("dmb sy");+ block_shared.answer.answered = request;+ clean_dcache((uint64_t)(uintptr_t)&block_shared.answer,+ (uint64_t)(uintptr_t)&block_shared.answer + sizeof(block_shared.answer));+}+static voidcharger_start(void){@@ -1255,6 +1613,9 @@ run_on_other_cpu(uint64_t mpidr, uint64_t entry, uint64_t boot_args_pa, const ststatus = OTHER_CPU_HUNG;}}+ if (block_enabled) {+ block_poll();+ }usb_poll_command();for (volatile int i = 0; i < 20000; i++) {}@@ -1734,7 +2095,7 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)/* "bootcpu=N" among the kernel's arguments: run it on core N (see run_on_other_cpu) */uint64_t this_mpidr = READ_SYSREG(MPIDR_EL1) & 0xFFFFFF, kernel_mpidr = this_mpidr;/* "cpus=N" or "cpumask=M" there too: the kernel may have more cores than the one it starts on */- int more_cores = 0, battery_requested = 0;+ int more_cores = 0, battery_requested = 0, ufs_requested = 0;/* "torch=N" (not 0) there too: the phone's flashlight is the kernel's (bsd/dev/redwood_torch.c) */int torch_requested = 0;/* "cam=5" there too: the phone's wide camera is iOS's (bsd/dev/redwood_camera.c) */@@ -1751,10 +2112,20 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)torch_requested |= starts_with(c, "torch=") && (c == pi.cmdline || c[-1] == ' ') && c[6] >= '1' && c[6] <= '9';camera_requested |= starts_with(c, "cam=5") && (c == pi.cmdline || c[-1] == ' ') && (c[5] == 0 || c[5] == ' ');gamut_p3 |= starts_with(c, "gamut=p3") && (c == pi.cmdline || c[-1] == ' ');+ if (starts_with(c, "ufs=") && (c == pi.cmdline || c[-1] == ' ') && c[4] >= '1' && c[4] <= '2' &&+ (c[5] == 0 || c[5] == ' ')) {+ ufs_requested = c[4] - '0';+ }}if (battery_requested) {charger_start();}+ if (ufs_requested >= 1) {+ ufs_survey();+ }+ if (ufs_requested >= 2) {+ block_start();+ }if (checkpoint == CHECKPOINT_SELFTEST) {kernel_mpidr = this_mpidr;}@@ -1857,6 +2228,19 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)ba->commandLine[n++] = *c;}}+ if (block_enabled) {+ static const char prefix[] = "ufs_shared_pa=0x";+ static const char hex[] = "0123456789abcdef";+ uint64_t address = (uint64_t)(uintptr_t)&block_shared;++ for (const char *c = prefix; *c && n < BOOT_LINE_LENGTH - 1; c++) {+ ba->commandLine[n++] = *c;+ }+ for (int shift = 60; shift >= 0 && n < BOOT_LINE_LENGTH - 1; shift -= 4) {+ ba->commandLine[n++] = hex[(address >> shift) & 0xf];+ }+ if (n < BOOT_LINE_LENGTH - 1) ba->commandLine[n++] = ' ';+ }if (battery_requested) {static const char prefix[] = "battery_shared_pa=0x";static const char hex[] = "0123456789abcdef";