Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

UFS: the kernel reads the phone's storage, through Qualcomm's own driver on core 0

vcvkk64e8d53481af
diff --git a/README.md b/README.md
index bcd9e6c..89f9a5f 100644
--- a/README.md
+++ b/README.md
@@ -398,6 +398,34 @@ P3 поверхности кадров копировались, и этого
пальцев); после паники следующий запуск сам останавливает его и запускает заново. Приложения, использующие Metal
сами, от этого ничего не получают.
+## Накопитель телефона (UFS)
+
+**Ядро читает флеш телефона** — все 112 разделов GPT шести логических единиц UFS, только чтение, ~250 МБ/с.
+Своего драйвера UFS в ядре нет, и он не нужен: загрузчик никогда не выходит из UEFI, драйвер Qualcomm
+(`UFSDxe`) уже поднял контроллер, его PHY, такты и питание и остаётся жить на ядре 0, где загрузчик и так
+обслуживает ядро порта (батарея). С `ufs=2` загрузчик выкладывает общую страницу (`ufs_shared_pa=`) с таблицей
+разделов, запросом и ответом и буфер на 4 МБ; ядро пишет запрос — какой раздел, какие блоки, — а ядро 0
+читает их через `ReadBlocks` этого драйвера (`xnuloader/loader.c`, `block_poll`; ядро —
+`bsd/dev/redwood_ufs.c`). Каждый раздел — `/dev/ufs_ИМЯ` (блочное) и `/dev/rufs_ИМЯ` (символьное), имя — как в
+таблице разделов. Нужен `bootcpu=7`: служба живёт в цикле, которым ядро 0 следит за ядром порта, а он есть,
+только когда ядро порта работает на другом ядре.
+
+Только чтение — по построению: открытие на запись отказывается, запись, дошедшая до драйвера, отказывается,
+а в странице просто нет поля, которым можно было бы попросить записать, — загрузчик зовёт `ReadBlocks` и
+больше ничего. Поля, которые пишет ядро, и поля, которые пишет загрузчик, лежат в разных строках кэша, а в
+буфер данных ядро 0 само не пишет никогда — поэтому очистка кэша одной стороны не затирает записи другой.
+
+Проверка — `ufs=1` (только загрузчик) или `ufs=2 ufs_check=1` (и ядро), затем `tools/ufs-survey.py`:
+- заголовок GPT каждой логической единицы сам себя проверяет (CRC-32 заголовка и таблицы разделов) — у всех
+ шести обе суммы верны: прочитанные байты — те, что на флеше, независимо от версии прошивки;
+- шестнадцать разделов (`xbl`, `tz`, `hyp`, `abl`, `aop`, `bluetooth`, `dsp`, `modem`, оба слота) ядро
+ прочитало через почтовый ящик — 568 запросов, 586 735 616 байт, 0 ошибок — и их хеши совпали с тем, что
+ загрузчик прочитал сам до старта ядра. С образами из `fw/fw.zip` они **не** совпадают: на телефоне стоит
+ другая версия прошивки (начала разделов при этом верные — ELF у загрузчиков, FAT у `modem` и `bluetooth`).
+
+Загрузчик пишет на флеш с первого коммита проекта ровно одно — свой журнал, в служебный раздел `logdump`
+(`write_logs`); к этому драйверу это не относится.
+
## Снимки экрана
Всё снято с экрана телефона за один сеанс: образ `GL=1 REST=1`, запуск с `gpu=1`, касания подаёт сценарий
diff --git a/tools/ufs-survey.py b/tools/ufs-survey.py
new file mode 100644
index 0000000..6202187
--- /dev/null
+++ b/tools/ufs-survey.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Check what the loader read from the phone's storage against the ROM's own images.
+
+usage: ufs-survey.py [LOG] [FW_ZIP]
+ LOG the loader's log of a run with ufs=1 among XL_CMDLINE (default logs/last.log)
+ FW_ZIP the ROM's firmware archive (default fw/fw.zip)
+
+With ufs=1 the loader (xnuloader/loader.c, ufs_survey) lists the phone's logical units and GPT
+partitions as Qualcomm's own UFS driver in UEFI sees them, and for the partitions there are images
+of in the ROM's firmware-update/ it prints an FNV-1a hash of exactly as many bytes as the image
+has. This computes the same hash from the images and says, partition by partition, whether the
+two agree: where they do, reading through that driver gives the bytes that are on the flash.
+With ufs=2 ufs_check=1 the kernel then reads the same partitions through the loader's page
+(bsd/dev/redwood_ufs.c) and prints the same hashes: those are shown as kernel:NAME.
+Nothing is read from the phone here; it only reads the log.
+"""
+import re
+import sys
+import zipfile
+
+
+def fnv(data, h=0xcbf29ce484222325):
+ for b in data:
+ h = ((h ^ b) * 0x100000001b3) & 0xffffffffffffffff
+ return h
+
+
+def main():
+ log_path = sys.argv[1] if len(sys.argv) > 1 else 'logs/last.log'
+ zip_path = sys.argv[2] if len(sys.argv) > 2 else 'fw/fw.zip'
+ text = open(log_path, encoding='utf-8', errors='replace').read()
+
+ units = re.findall(r'^ufs: unit (\d+): .*$', text, re.M)
+ parts = re.findall(r'^ufs: part (\S+): lba (\d+)\.\.(\d+), (\d+) KB$', text, re.M)
+ hashes = re.findall(r'^ufs: (\S+): fnv ([0-9a-f]+) over (\d+) bytes, (\d+) ms$', text, re.M)
+ # the same partitions read again by the kernel through the loader's page (ufs=2 ufs_check=1)
+ checks = re.findall(r'ufs: check (\S+): fnv ([0-9a-f]+) over (\d+) bytes, (\d+) ms', text)
+ hashes += [('kernel:' + name, value, size, ms) for name, value, size, ms in checks]
+ errors = re.findall(r'^ufs: (\S+): (reading at block .*|.* the partition is smaller)$', text, re.M)
+ if not parts:
+ sys.exit(f'ufs-survey.py: no "ufs:" lines in {log_path} (was the run made with ufs=1?)')
+ print(f'{len(units)} logical units, {len(parts)} partitions in the log')
+
+ images = {}
+ with zipfile.ZipFile(zip_path) as z:
+ for info in z.infolist():
+ m = re.match(r'firmware-update/(.+)\.img$', info.filename)
+ if m:
+ images[m.group(1)] = info.filename
+ agreed = differed = 0
+ for name, value, size, ms in hashes:
+ base = re.sub(r'_[ab]$', '', name.removeprefix('kernel:'))
+ if base not in images:
+ print(f' {name}: no image named {base}.img in {zip_path}')
+ continue
+ data = z.read(images[base])
+ if len(data) != int(size):
+ print(f' {name}: the image has {len(data)} bytes, the loader hashed {size}')
+ differed += 1
+ continue
+ mine = fnv(data)
+ same = mine == int(value, 16)
+ agreed += same
+ differed += not same
+ rate = int(size) / max(int(ms), 1) / 1000
+ print(f' {name:<21} {"same" if same else "DIFFERENT":<9} {int(size):>11} bytes '
+ f'in {ms:>5} ms ({rate:.0f} MB/s) flash {value} image {mine:x}')
+ for name, what in errors:
+ print(f' {name}: {what}')
+ print(f'{agreed} agree, {differed} differ, {len(errors)} not read')
+ return 0 if differed == 0 and not errors and agreed > 0 else 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/xnu/files/bsd/dev/redwood_input.c b/xnu/files/bsd/dev/redwood_input.c
index 6c584d6..8076cf0 100644
--- a/xnu/files/bsd/dev/redwood_input.c
+++ b/xnu/files/bsd/dev/redwood_input.c
@@ -219,6 +219,7 @@ redwood_input_init(void)
redwood_goodix_init();
redwood_buttons_init();
redwood_battery_init();
+ redwood_ufs_init();
redwood_net_init();
redwood_adreno_init();
redwood_wifi_init();
diff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.c
new file mode 100644
index 0000000..1ffbe36
--- /dev/null
+++ b/xnu/files/bsd/dev/redwood_ufs.c
@@ -0,0 +1,480 @@
+/*
+ * The phone's storage, read only: one block device and one character device for every GPT
+ * partition of its UFS (/dev/ufs_NAME and /dev/rufs_NAME, NAME as the partition table has it).
+ *
+ * There is no UFS driver here, and none is needed. UEFI never exits on this port: Qualcomm's own
+ * UFS driver (UFSDxe) has brought up the controller, its PHY, its clocks and power, and stays
+ * resident, and xnuloader keeps serving the kernel from core 0 while the kernel runs on another
+ * core (it already does for the battery). So with ufs=2 on its command line the loader publishes a
+ * page (ufs_shared_pa=) with the table of partitions, a request and an answer, and a buffer; this
+ * driver writes a request -- which partition, which blocks -- and core 0 reads them with that
+ * driver's ReadBlocks into the buffer (xnuloader/loader.c, block_poll). Nothing in this kernel
+ * touches the storage controller.
+ *
+ * Read only by construction: opening for writing is refused, a write that reaches the strategy
+ * routine is refused, and the page has no field in which a write could be asked for -- the
+ * loader's side calls ReadBlocks and nothing else.
+ *
+ * One request at a time (the page has one), so requests are serialized by a mutex and split into
+ * pieces of the buffer's size. The kernel writes only the request's cache lines and core 0 only
+ * the answer's (see the loader), both through mappings that do not cache here, so neither side
+ * ever overwrites the other's.
+ *
+ * ufs_check=1 reads, once at start, the partitions there are images of in the ROM's
+ * firmware-update/ (bluetooth, xbl, tz, ...) through the same path and prints an FNV-1a hash of
+ * as many bytes as each image has: tools/ufs-survey.py compares those with the images and with
+ * what the loader read itself before the kernel started.
+ */
+#include <kern/clock.h>
+#include <kern/locks.h>
+#include <kern/thread.h>
+#include <libkern/libkern.h>
+#include <mach/mach_types.h>
+#include <mach/vm_param.h>
+#include <miscfs/devfs/devfs.h>
+#include <pexpert/pexpert.h>
+#include <sys/buf.h>
+#include <sys/conf.h>
+#include <sys/disk.h>
+#include <sys/errno.h>
+#include <sys/systm.h>
+#include <sys/sysctl.h>
+#include <sys/redwood_input.h>
+#include <vm/memory_types.h>
+
+extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
+extern void IOSleep(unsigned int milliseconds);
+
+/* the page the loader publishes: the same layout as struct loader_block in xnuloader/loader.c */
+#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */
+#define BLOCK_VERSION 1
+#define BLOCK_MAX_PARTS 128
+
+struct loader_block_part {
+ char name[36];
+ uint32_t block_size;
+ uint64_t blocks;
+};
+
+struct loader_block {
+ uint64_t magic;
+ uint32_t version, size;
+ uint32_t parts, buffer_size;
+ uint64_t buffer_pa;
+ struct {
+ volatile uint32_t request;
+ uint32_t part;
+ uint64_t lba, count;
+ } ask __attribute__((aligned(128)));
+ struct {
+ volatile uint32_t answered;
+ uint32_t reserved;
+ int64_t status;
+ uint64_t served, failed, bytes, ticks;
+ } answer __attribute__((aligned(128)));
+ struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));
+};
+
+#define UFS_PAGE 0x4000UL
+#define UFS_TIMEOUT_MS 20000 /* core 0 answers in well under a second; this is for a dead loader */
+
+static volatile struct loader_block *ufs_page;
+static const uint8_t *ufs_buffer;
+static uint32_t ufs_buffer_size, ufs_parts, ufs_seq;
+static uint32_t ufs_bsize[BLOCK_MAX_PARTS]; /* the block size each device is counted in (spec_open sets 512) */
+static int ufs_bmajor = -1, ufs_cmajor = -1;
+static bool ufs_ready, ufs_dead;
+static uint64_t ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us;
+
+static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");
+static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);
+
+SYSCTL_DECL(_kern_redwood);
+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_requests,
+ "requests the phone's storage has answered through the loader");
+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_errors,
+ "requests the phone's storage has refused or not answered");
+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_bytes,
+ "bytes read from the phone's storage");
+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_wait_us,
+ "microseconds spent waiting for core 0's answers");
+
+/*
+ * Native blocks [lba, lba + count) of partition part, into dst: one request of at most the
+ * buffer's size. Called with ufs_lock held.
+ */
+static int
+ufs_request(uint32_t part, uint64_t lba, uint64_t count, uint8_t *dst)
+{
+ uint64_t bytes = count * ufs_page->part[part].block_size, started, waited_ms = 0;
+ uint32_t seq = ++ufs_seq;
+ int64_t status;
+
+ if (ufs_dead) {
+ return EIO;
+ }
+ ufs_page->ask.part = part;
+ ufs_page->ask.lba = lba;
+ ufs_page->ask.count = count;
+ __asm__ volatile ("dmb sy" ::: "memory");
+ ufs_page->ask.request = seq;
+ __asm__ volatile ("dmb sy" ::: "memory");
+
+ started = mach_absolute_time();
+ while (ufs_page->answer.answered != seq) {
+ /* spin briefly for a quick answer, then sleep a millisecond at a time */
+ uint64_t us;
+
+ absolutetime_to_nanoseconds(mach_absolute_time() - started, &us);
+ us /= 1000;
+ if (us < 2000) {
+ delay(20);
+ continue;
+ }
+ IOSleep(1);
+ if (++waited_ms > UFS_TIMEOUT_MS) {
+ /* no answer: the loader is gone or stuck; nothing more is asked of it */
+ printf("ufs: no answer from the loader to request %u in %u s: the storage is off from now on\n",
+ seq, UFS_TIMEOUT_MS / 1000);
+ ufs_dead = true;
+ ufs_errors++;
+ return EIO;
+ }
+ }
+ __asm__ volatile ("dmb sy" ::: "memory");
+ {
+ uint64_t ns;
+
+ absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
+ ufs_wait_us += ns / 1000;
+ }
+ status = ufs_page->answer.status;
+ if (status != 0) {
+ printf("ufs: %s: blocks %llu..%llu: the loader's read failed (status %llx)\n",
+ ufs_page->part[part].name, lba, lba + count - 1, (uint64_t)status);
+ ufs_errors++;
+ return EIO;
+ }
+ memcpy(dst, ufs_buffer, (size_t)bytes);
+ ufs_requests++;
+ ufs_bytes += bytes;
+ return 0;
+}
+
+/*
+ * Bytes [offset, offset + len) of partition part into dst, whatever their alignment: the native
+ * blocks that cover them are read a buffer at a time and the wanted bytes copied out.
+ */
+static int
+ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
+{
+ uint64_t nbs = ufs_page->part[part].block_size, per = ufs_buffer_size / nbs;
+ static uint8_t *bounce;
+ int error = 0;
+
+ if (bounce == NULL) {
+ bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);
+ if (bounce == NULL) {
+ return ENOMEM;
+ }
+ }
+ lck_mtx_lock(&ufs_lock);
+ while (len > 0 && error == 0) {
+ uint64_t first = offset / nbs, skip = offset % nbs;
+ uint64_t count = (skip + len + nbs - 1) / nbs, take;
+
+ if (count > per) {
+ count = per;
+ }
+ take = count * nbs - skip;
+ if (take > len) {
+ take = len;
+ }
+ if (skip == 0 && take == count * nbs) {
+ error = ufs_request(part, first, count, dst); /* whole blocks: straight in */
+ } else {
+ error = ufs_request(part, first, count, bounce);
+ if (error == 0) {
+ memcpy(dst, bounce + skip, (size_t)take);
+ }
+ }
+ offset += take;
+ dst += take;
+ len -= take;
+ }
+ lck_mtx_unlock(&ufs_lock);
+ return error;
+}
+
+static uint64_t
+ufs_part_bytes(uint32_t part)
+{
+ return ufs_page->part[part].blocks * ufs_page->part[part].block_size;
+}
+
+/* ---- the devices ---- */
+
+static int
+ufs_open(dev_t dev, int flags, __unused int devtype, __unused proc_t p)
+{
+ if (!ufs_ready || (uint32_t)minor(dev) >= ufs_parts) {
+ return ENXIO;
+ }
+ if (flags & FWRITE) {
+ return EROFS;
+ }
+ return 0;
+}
+
+static int
+ufs_close(__unused dev_t dev, __unused int flags, __unused int devtype, __unused proc_t p)
+{
+ return 0;
+}
+
+static void
+ufs_strategy(buf_t bp)
+{
+ uint32_t part = (uint32_t)minor(buf_device(bp));
+ uint64_t offset, len, size;
+ int error;
+
+ if (!ufs_ready || part >= ufs_parts) {
+ buf_seterror(bp, ENXIO);
+ buf_biodone(bp);
+ return;
+ }
+ if (!(buf_flags(bp) & B_READ)) {
+ buf_seterror(bp, EROFS); /* this device never writes */
+ buf_biodone(bp);
+ return;
+ }
+ size = ufs_part_bytes(part);
+ offset = (uint64_t)buf_blkno(bp) * ufs_bsize[part];
+ len = buf_count(bp);
+ if (offset >= size) {
+ buf_seterror(bp, offset == size ? 0 : EINVAL);
+ buf_setresid(bp, buf_count(bp));
+ buf_biodone(bp);
+ return;
+ }
+ if (offset + len > size) {
+ len = size - offset;
+ }
+ error = ufs_read(part, offset, (uint8_t *)buf_dataptr(bp), len);
+ buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : len)));
+ if (error) {
+ buf_seterror(bp, error);
+ }
+ buf_biodone(bp);
+}
+
+static int
+ufs_ioctl(dev_t dev, u_long cmd, caddr_t data, __unused int fflag, __unused proc_t p)
+{
+ uint32_t part = (uint32_t)minor(dev), native;
+
+ if (!ufs_ready || part >= ufs_parts) {
+ return ENXIO;
+ }
+ native = ufs_page->part[part].block_size;
+ switch (cmd) {
+ case DKIOCGETBLOCKSIZE:
+ *(uint32_t *)data = ufs_bsize[part];
+ return 0;
+ case DKIOCSETBLOCKSIZE:
+ /*
+ * spec_open() sets the block size to 512 to count the device in 512-byte blocks and
+ * then back, and treats a refusal as a broken device; reads are by byte here, so any
+ * power of two from 512 to the native block size will do.
+ */
+ if (*(uint32_t *)data < 512 || *(uint32_t *)data > native ||
+ (*(uint32_t *)data & (*(uint32_t *)data - 1)) != 0) {
+ return EINVAL;
+ }
+ ufs_bsize[part] = *(uint32_t *)data;
+ return 0;
+ case DKIOCGETPHYSICALBLOCKSIZE:
+ *(uint32_t *)data = native;
+ return 0;
+ case DKIOCGETBLOCKCOUNT:
+ *(uint64_t *)data = ufs_part_bytes(part) / ufs_bsize[part];
+ return 0;
+ case DKIOCISWRITABLE:
+ *(uint32_t *)data = 0;
+ return 0;
+ case DKIOCGETMAXBLOCKCOUNTREAD:
+ *(uint64_t *)data = ufs_buffer_size / ufs_bsize[part];
+ return 0;
+ default:
+ return ENOTTY;
+ }
+}
+
+static int
+ufs_psize(dev_t dev)
+{
+ uint32_t part = (uint32_t)minor(dev);
+
+ return ufs_ready && part < ufs_parts ? (int)ufs_bsize[part] : -1;
+}
+
+static int
+ufs_cread(dev_t dev, struct uio *uio, __unused int ioflag)
+{
+ return physio(ufs_strategy, NULL, dev, B_READ, (u_int (*)(buf_t))ufs_psize, uio,
+ ufs_bsize[minor(dev) < BLOCK_MAX_PARTS ? minor(dev) : 0]);
+}
+
+static const struct bdevsw ufs_bdevsw = {
+ .d_open = ufs_open,
+ .d_close = ufs_close,
+ .d_strategy = ufs_strategy,
+ .d_ioctl = ufs_ioctl,
+ .d_dump = eno_dump,
+ .d_psize = ufs_psize,
+ .d_type = D_DISK,
+};
+
+static const struct cdevsw ufs_cdevsw = {
+ .d_open = ufs_open,
+ .d_close = ufs_close,
+ .d_read = ufs_cread,
+ .d_write = eno_rdwrt,
+ .d_ioctl = ufs_ioctl,
+ .d_stop = eno_stop,
+ .d_reset = eno_reset,
+ .d_ttys = NULL,
+ .d_select = eno_select,
+ .d_mmap = eno_mmap,
+ .d_strategy = eno_strat,
+ .d_reserved_1 = eno_getc,
+ .d_reserved_2 = eno_putc,
+ .d_type = D_DISK,
+};
+
+/* ---- the check: the partitions there are images of, hashed as the loader hashes them ---- */
+
+static const struct {
+ const char *name;
+ uint64_t bytes;
+} ufs_references[] = {
+ { "bluetooth", 503808 }, { "abl", 208896 }, { "aop", 208896 }, { "xbl", 3706880 },
+ { "tz", 3710976 }, { "hyp", 3653632 }, { "dsp", 67108864 }, { "modem", 214265856 },
+};
+
+static bool
+ufs_is_reference(const char *name, const char *ref)
+{
+ size_t n = strlen(ref);
+
+ return strncmp(name, ref, n) == 0 &&
+ (name[n] == 0 || (name[n] == '_' && (name[n + 1] == 'a' || name[n + 1] == 'b') && name[n + 2] == 0));
+}
+
+static void
+ufs_check(__unused void *arg, __unused wait_result_t wr)
+{
+ uint8_t *chunk = kalloc_data(1 << 20, Z_WAITOK);
+
+ /*
+ * Not at once: what the kernel prints in its first minute or so does not come back with the
+ * log on the phone (the first line that does is from long after this driver has started), so
+ * the check waits until it will be read.
+ */
+ IOSleep(40000);
+ printf("ufs: check: %u partitions of the phone's storage, read through the loader's page\n", ufs_parts);
+ for (uint32_t part = 0; chunk != NULL && part < ufs_parts; part++) {
+ for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
+ uint64_t h = 0xcbf29ce484222325ULL, done = 0, started = mach_absolute_time(), ns;
+ uint64_t bytes = ufs_references[r].bytes;
+ int error = 0;
+
+ if (!ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {
+ continue;
+ }
+ while (done < bytes && error == 0) {
+ uint64_t n = bytes - done < (1 << 20) ? bytes - done : (1 << 20);
+
+ error = ufs_read(part, done, chunk, n);
+ for (uint64_t i = 0; error == 0 && i < n; i++) {
+ h = (h ^ chunk[i]) * 0x100000001b3ULL;
+ }
+ done += n;
+ }
+ absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
+ if (error) {
+ printf("ufs: check %s: read failed (%d)\n", ufs_page->part[part].name, error);
+ } else {
+ printf("ufs: check %s: fnv %llx over %llu bytes, %llu ms\n", ufs_page->part[part].name,
+ h, bytes, ns / 1000000);
+ }
+ }
+ }
+ if (chunk != NULL) {
+ kfree_data(chunk, 1 << 20);
+ }
+ printf("ufs: check done: %llu requests, %llu errors, %llu bytes, %llu ms waiting\n",
+ ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us / 1000);
+}
+
+/* ---- bringing it up ---- */
+
+void
+redwood_ufs_init(void)
+{
+ uint64_t pa = 0, buffer_pa;
+ uint32_t check = 0, made = 0;
+ vm_offset_t base;
+
+ if (!PE_parse_boot_argn("ufs_shared_pa", &pa, sizeof(pa)) || pa == 0) {
+ return; /* the loader was not asked for ufs=2: there is nothing to read through */
+ }
+ base = ml_io_map_unmappable((vm_offset_t)(pa & ~(UFS_PAGE - 1)),
+ ((pa & (UFS_PAGE - 1)) + sizeof(struct loader_block) + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_WCOMB);
+ ufs_page = (volatile struct loader_block *)(base + (pa & (UFS_PAGE - 1)));
+ if (ufs_page->magic != BLOCK_MAGIC || ufs_page->version != BLOCK_VERSION ||
+ ufs_page->size != sizeof(struct loader_block) || ufs_page->parts > BLOCK_MAX_PARTS) {
+ printf("ufs: the loader's page at %llx is not one this kernel knows (magic %llx, version %u, size %u)\n",
+ pa, ufs_page->magic, ufs_page->version, ufs_page->size);
+ return;
+ }
+ ufs_parts = ufs_page->parts;
+ ufs_buffer_size = ufs_page->buffer_size;
+ buffer_pa = ufs_page->buffer_pa;
+ ufs_buffer = (const uint8_t *)(ml_io_map_unmappable((vm_offset_t)(buffer_pa & ~(UFS_PAGE - 1)),
+ ((buffer_pa & (UFS_PAGE - 1)) + ufs_buffer_size + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_WCOMB) +
+ (buffer_pa & (UFS_PAGE - 1)));
+ ufs_seq = ufs_page->answer.answered; /* a loader that served an earlier kernel counts on from there */
+
+ ufs_bmajor = bdevsw_add(-1, &ufs_bdevsw);
+ ufs_cmajor = ufs_bmajor < 0 ? -1 : cdevsw_add_with_bdev(-1, &ufs_cdevsw, ufs_bmajor);
+ if (ufs_cmajor < 0) {
+ printf("ufs: no device numbers to be had\n");
+ return;
+ }
+ for (uint32_t i = 0; i < ufs_parts; i++) {
+ char name[48];
+
+ ufs_bsize[i] = ufs_page->part[i].block_size;
+ snprintf(name, sizeof(name), "ufs_%s", ufs_page->part[i].name);
+ if (devfs_make_node(makedev(ufs_bmajor, i), DEVFS_BLOCK, UID_ROOT, GID_OPERATOR, 0640, "%s", name) == NULL) {
+ printf("ufs: no node for %s (the same name twice?)\n", name);
+ continue;
+ }
+ snprintf(name, sizeof(name), "rufs_%s", ufs_page->part[i].name);
+ (void)devfs_make_node(makedev(ufs_cmajor, i), DEVFS_CHAR, UID_ROOT, GID_OPERATOR, 0640, "%s", name);
+ made++;
+ }
+ ufs_ready = true;
+ printf("ufs: %u partitions of the phone's storage as /dev/ufs_NAME and /dev/rufs_NAME, read only, "
+ "through the loader's page at %llx and its %u KB buffer\n", made, pa, ufs_buffer_size >> 10);
+
+ if (PE_parse_boot_argn("ufs_check", &check, sizeof(check)) && check != 0) {
+ thread_t thread;
+
+ if (kernel_thread_start(ufs_check, NULL, &thread) == KERN_SUCCESS) {
+ thread_deallocate(thread);
+ }
+ }
+}
diff --git a/xnu/files/bsd/sys/redwood_input.h b/xnu/files/bsd/sys/redwood_input.h
index bdfb148..5edde86 100644
--- a/xnu/files/bsd/sys/redwood_input.h
+++ b/xnu/files/bsd/sys/redwood_input.h
@@ -42,6 +42,7 @@ void redwood_input_init(void);
void redwood_virtio_input_init(void); /* boot-arg virtio_input=1: QEMU's virtio pointer and keyboard */
void redwood_goodix_init(void); /* boot-arg touchscreen=1: the phone's Goodix or FocalTech touch screen */
void redwood_buttons_init(void); /* boot-arg buttons=1: the phone's power and volume keys */
+void redwood_ufs_init(void); /* ufs=2 to xnuloader: the phone's storage, read only, through the loader (ufs_shared_pa=) */
void redwood_battery_init(void); /* boot-arg battery=1: Qualcomm UEFI charger state shared by xnuloader */
int redwood_battery_temperature(void); /* the battery's temperature in C, -273 while it is not known */
/* one transfer at a time through the PMIC arbiter for this system (bsd/dev/redwood_buttons.c) */
diff --git a/xnu/redwood.patch b/xnu/redwood.patch
index eada4c3..157db95 100644
--- a/xnu/redwood.patch
+++ b/xnu/redwood.patch
@@ -46,10 +46,10 @@ index 222158a93918..68bb0f3e56bf 100755
######################################################################
# HISTORY
diff --git a/bsd/conf/files b/bsd/conf/files
-index 44274016a4cf..9f1fe3efd685 100644
+index 44274016a4cf..94a03cc34d2c 100644
--- a/bsd/conf/files
+++ b/bsd/conf/files
-@@ -109,6 +109,31 @@ bsd/dev/munge.c optional config_requires_u32_munging
+@@ -109,6 +109,32 @@ bsd/dev/munge.c optional config_requires_u32_munging
bsd/dev/unix_startup.c standard
bsd/dev/monotonic.c optional monotonic
@@ -58,6 +58,7 @@ index 44274016a4cf..9f1fe3efd685 100644
+bsd/dev/redwood_goodix.c standard
+bsd/dev/redwood_buttons.c standard
+bsd/dev/redwood_battery.c standard
++bsd/dev/redwood_ufs.c standard
+bsd/dev/redwood_net.c standard
+bsd/dev/redwood_virtio_net.c standard
+bsd/dev/redwood_apple80211.c standard
@@ -81,7 +82,7 @@ index 44274016a4cf..9f1fe3efd685 100644
bsd/dev/dev_perfmon.c optional config_perfmon
bsd/libkern/crc32.c standard
-@@ -155,6 +180,10 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs
+@@ -155,6 +181,10 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs
bsd/miscfs/mockfs/mockfs_fsnode.c optional mockfs
bsd/miscfs/mockfs/mockfs_vfsops.c optional mockfs
bsd/miscfs/mockfs/mockfs_vnops.c optional mockfs
@@ -92,7 +93,7 @@ index 44274016a4cf..9f1fe3efd685 100644
bsd/miscfs/routefs/routefs_ops.c optional routefs
-@@ -361,6 +390,7 @@ bsd/security/audit/audit_syscalls.c standard
+@@ -361,6 +391,7 @@ bsd/security/audit/audit_syscalls.c standard
bsd/security/audit/audit_worker.c optional config_audit
bsd/pthread/pthread_shims.c standard
@@ -100,7 +101,7 @@ index 44274016a4cf..9f1fe3efd685 100644
bsd/pthread/pthread_priority.c standard
bsd/pthread/pthread_workqueue.c standard
-@@ -383,6 +413,7 @@ bsd/kern/kern_debug.c standard
+@@ -383,6 +414,7 @@ bsd/kern/kern_debug.c standard
bsd/kern/kern_symfile.c standard
bsd/kern/kern_descrip.c standard
bsd/kern/kern_guarded.c standard
diff --git a/xnuloader/loader.c b/xnuloader/loader.c
index 741de39..29f6bf3 100644
--- a/xnuloader/loader.c
+++ b/xnuloader/loader.c
@@ -1073,6 +1073,364 @@ charger_poll(void)
(uint64_t)(uintptr_t)&charger_shared + sizeof(charger_shared));
}
+/* ------------------------------------------------------------------------ */
+/* UFS survey (ufs=1): what the phone's storage holds, read only */
+
+/*
+ * The phone's storage as Qualcomm's own UFS driver in this UEFI sees it (UFSDxe): every raw
+ * logical unit, every GPT partition with its name, place and size, and for the partitions there
+ * are reference images of -- firmware-update/ in the ROM's fw.zip -- an FNV-1a hash of exactly
+ * as many bytes as the image has, which tools/ufs-survey.py computes from the images on the host.
+ * Where the two agree, reading through this driver gives the bytes that are on the flash, and that
+ * is the ground a block device for the kernel is to stand on.
+ *
+ * Only read_blocks is ever called here: nothing is written, flushed or reset. It runs before the
+ * kernel is started, with UEFI's boot services whole and interrupts on.
+ */
+static const struct {
+ const char *name;
+ uint64_t bytes;
+} ufs_references[] = {
+ { "bluetooth", 503808 }, { "abl", 208896 }, { "aop", 208896 }, { "xbl", 3706880 },
+ { "tz", 3710976 }, { "hyp", 3653632 }, { "dsp", 67108864 }, { "modem", 214265856 },
+};
+
+static uint8_t ufs_buffer[1024 * 1024] __attribute__((aligned(4096)));
+
+static uint64_t
+ufs_fnv(uint64_t h, const uint8_t *p, uint64_t n)
+{
+ for (uint64_t i = 0; i < n; i++) {
+ h = (h ^ p[i]) * 0x100000001b3ULL;
+ }
+ return h;
+}
+
+/* the partition's name in ASCII (GPT names are UTF-16; these are all plain letters) */
+static void
+ufs_name(const efi_partition_info *info, char *out, unsigned size)
+{
+ unsigned i = 0;
+
+ for (; i + 1 < size && i < 36 && info->name[i] != 0; i++) {
+ out[i] = info->name[i] < 0x80 ? (char)info->name[i] : '?';
+ }
+ out[i] = 0;
+}
+
+/* "bluetooth_a" and "bluetooth_b" are both "bluetooth": the image is the same for either slot */
+static int
+ufs_reference_of(const char *name)
+{
+ for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
+ const char *a = ufs_references[r].name, *b = name;
+
+ while (*a && *a == *b) {
+ a++;
+ b++;
+ }
+ if (*a == 0 && (*b == 0 || ((b[0] == '_') && (b[1] == 'a' || b[1] == 'b') && b[2] == 0))) {
+ return (int)r;
+ }
+ }
+ return -1;
+}
+
+static void
+ufs_hash_partition(efi_block_io *bio, const char *name, uint64_t bytes)
+{
+ uint64_t bs = bio->media->block_size, h = 0xcbf29ce484222325ULL, done = 0, started = READ_SYSREG(CNTVCT_EL0);
+ uint64_t chunk = sizeof(ufs_buffer) / bs * bs, lba = 0;
+
+ if (bs == 0 || bs > sizeof(ufs_buffer) || (bio->media->last_block + 1) * bs < bytes) {
+ print("ufs: %s: %d bytes asked for, the partition is smaller\n", name, bytes);
+ return;
+ }
+ while (done < bytes) {
+ uint64_t want = bytes - done < chunk ? (bytes - done + bs - 1) / bs * bs : chunk;
+ efi_status s = bio->read_blocks(bio, bio->media->media_id, lba, want, ufs_buffer);
+
+ if (EFI_ERROR(s)) {
+ print("ufs: %s: reading at block %d: error %x\n", name, lba, (uint64_t)s);
+ return;
+ }
+ if (done == 0) {
+ print("ufs: %s: begins", name);
+ for (int k = 0; k < 16; k++) {
+ print(" %x", (uint64_t)ufs_buffer[k]);
+ }
+ print("\n");
+ }
+ h = ufs_fnv(h, ufs_buffer, bytes - done < want ? bytes - done : want);
+ done += bytes - done < want ? bytes - done : want;
+ lba += want / bs;
+ if (wdt_armed) {
+ wdt_pet();
+ }
+ }
+ print("ufs: %s: fnv %x over %d bytes, %d ms\n", name, h, bytes,
+ (READ_SYSREG(CNTVCT_EL0) - started) * 1000 / READ_SYSREG(CNTFRQ_EL0));
+}
+
+/* CRC-32 as GPT uses it (IEEE 802.3, reflected, 0xedb88320) */
+static uint32_t
+ufs_crc32(const uint8_t *p, uint64_t n)
+{
+ uint32_t c = 0xffffffffu;
+
+ for (uint64_t i = 0; i < n; i++) {
+ c ^= p[i];
+ for (int k = 0; k < 8; k++) {
+ c = (c >> 1) ^ (0xedb88320u & (0u - (c & 1)));
+ }
+ }
+ return ~c;
+}
+
+/*
+ * A unit's GPT checks itself: its header carries a CRC-32 of the header and one of the table of
+ * partitions. Whatever firmware is on the phone, the bytes read are the bytes on the flash only if
+ * both come out right -- the one check here that does not depend on having the same images.
+ */
+static void
+ufs_check_gpt(efi_block_io *bio, uint64_t unit)
+{
+ uint64_t bs = bio->media->block_size, entries_lba, count, size, bytes, read;
+ uint32_t header_size, header_crc, entries_crc, got_header, got_entries;
+ uint8_t header[512];
+
+ if (bs < 512 || bs > sizeof(ufs_buffer) ||
+ EFI_ERROR(bio->read_blocks(bio, bio->media->media_id, 1, bs, ufs_buffer))) {
+ print("ufs: unit %d: the GPT header could not be read\n", unit);
+ return;
+ }
+ if (ufs_buffer[0] != 'E' || ufs_buffer[1] != 'F' || ufs_buffer[2] != 'I' || ufs_buffer[3] != ' ' ||
+ ufs_buffer[4] != 'P' || ufs_buffer[5] != 'A' || ufs_buffer[6] != 'R' || ufs_buffer[7] != 'T') {
+ print("ufs: unit %d: no GPT header in block 1\n", unit);
+ return;
+ }
+ memcpy(header, ufs_buffer, sizeof(header));
+ header_size = *(uint32_t *)(header + 12);
+ header_crc = *(uint32_t *)(header + 16);
+ entries_lba = *(uint64_t *)(header + 72);
+ count = *(uint32_t *)(header + 80);
+ size = *(uint32_t *)(header + 84);
+ entries_crc = *(uint32_t *)(header + 88);
+ if (header_size < 92 || header_size > sizeof(header)) {
+ print("ufs: unit %d: GPT header size %d\n", unit, (uint64_t)header_size);
+ return;
+ }
+ *(uint32_t *)(header + 16) = 0;
+ got_header = ufs_crc32(header, header_size);
+ bytes = count * size;
+ if (bytes == 0 || bytes > sizeof(ufs_buffer)) {
+ print("ufs: unit %d: GPT header crc %s, a table of %d bytes not checked\n", unit,
+ got_header == header_crc ? "right" : "WRONG", bytes);
+ return;
+ }
+ read = (bytes + bs - 1) / bs * bs;
+ if (EFI_ERROR(bio->read_blocks(bio, bio->media->media_id, entries_lba, read, ufs_buffer))) {
+ print("ufs: unit %d: the GPT table could not be read\n", unit);
+ return;
+ }
+ got_entries = ufs_crc32(ufs_buffer, bytes);
+ print("ufs: unit %d: GPT header crc %s (%x), table of %d entries crc %s (%x)\n", unit,
+ got_header == header_crc ? "right" : "WRONG", (uint64_t)got_header, count,
+ got_entries == entries_crc ? "right" : "WRONG", (uint64_t)got_entries);
+}
+
+static void
+ufs_survey(void)
+{
+ static const efi_guid part_guid = EFI_PARTITION_INFO_GUID;
+ static const efi_guid blk_guid = EFI_BLOCK_IO_GUID;
+ efi_handle *handles;
+ uint64_t count, units = 0, partitions = 0;
+
+ /* the raw logical units: the block devices that are not partitions of another */
+ if (!EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &blk_guid, NULL, &count, &handles))) {
+ for (uint64_t i = 0; i < count; i++) {
+ efi_block_io *bio;
+
+ if (EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio)) || bio->media->logical) {
+ continue;
+ }
+ print("ufs: unit %d: media %d, %d blocks of %d (%d MB), %s, align %d\n", units++,
+ (uint64_t)bio->media->media_id, bio->media->last_block + 1, (uint64_t)bio->media->block_size,
+ (bio->media->last_block + 1) * bio->media->block_size >> 20,
+ bio->media->read_only ? "read only" : "writable", (uint64_t)bio->media->io_align);
+ ufs_check_gpt(bio, units - 1);
+ }
+ }
+ /* the GPT partitions, with the hash of those there are images of */
+ if (EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &part_guid, NULL, &count, &handles))) {
+ print("ufs: no partition information in this UEFI\n");
+ return;
+ }
+ for (uint64_t i = 0; i < count; i++) {
+ efi_partition_info *info;
+ efi_block_io *bio;
+ char name[40];
+ int r;
+
+ if (EFI_ERROR(BS->handle_protocol(handles[i], &part_guid, (void **)&info)) || info->type != 2 ||
+ EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio))) {
+ continue;
+ }
+ ufs_name(info, name, sizeof(name));
+ partitions++;
+ print("ufs: part %s: lba %d..%d, %d KB\n", name, info->start_lba, info->end_lba,
+ (bio->media->last_block + 1) * bio->media->block_size >> 10);
+ if ((r = ufs_reference_of(name)) >= 0) {
+ ufs_hash_partition(bio, name, ufs_references[r].bytes);
+ }
+ }
+ print("ufs: %d raw units, %d partitions\n", units, partitions);
+}
+
+/* ------------------------------------------------------------------------ */
+/* UFS block service (ufs=2): the kernel reads the phone's storage through */
+/* this loader, and this loader through Qualcomm's own UFS driver */
+
+/*
+ * The kernel has no driver for the phone's storage, and does not need one: UEFI never exits here,
+ * Qualcomm's UFS driver (UFSDxe) has the controller, its PHY, clocks and power up and stays
+ * resident, and this loader already serves the kernel from core 0 while the kernel runs on another
+ * (the battery). So the kernel asks for blocks through a page both can see, and core 0 reads them
+ * with that driver's ReadBlocks into a buffer both can see. The kernel's side is
+ * bsd/dev/redwood_ufs.c; it gets the page's address as ufs_shared_pa= on its command line.
+ *
+ * Read only, by construction: this service calls read_blocks and nothing else, the kernel's side
+ * refuses writes before they are asked for, and there is no field in the page that could ask for
+ * one.
+ *
+ * Each side writes its own cache lines only: the kernel the request, core 0 the answer (and the
+ * table, once, before the kernel starts). A line one side cleans out of its cache therefore never
+ * carries a stale copy of what the other side wrote into it. Core 0 never writes into the data
+ * buffer itself, so after the controller's DMA there is nothing of its own in its cache to write
+ * back over the data.
+ */
+#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */
+#define BLOCK_VERSION 1
+#define BLOCK_MAX_PARTS 128
+#define BLOCK_BUFFER_SIZE (4u << 20)
+
+struct loader_block_part {
+ char name[36];
+ uint32_t block_size;
+ uint64_t blocks;
+};
+
+struct loader_block {
+ /* written by the loader, once, before the kernel starts */
+ uint64_t magic;
+ uint32_t version, size;
+ uint32_t parts, buffer_size;
+ uint64_t buffer_pa;
+ /* written by the kernel: what it wants, then request is advanced */
+ struct {
+ volatile uint32_t request;
+ uint32_t part;
+ uint64_t lba, count; /* in the partition's own blocks */
+ } ask __attribute__((aligned(128)));
+ /* written by the loader: answered becomes request when the data is in the buffer */
+ struct {
+ volatile uint32_t answered;
+ uint32_t reserved;
+ int64_t status; /* the EFI status of the read, 0 when it went well */
+ uint64_t served, failed, bytes, ticks;
+ } answer __attribute__((aligned(128)));
+ struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));
+};
+
+/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */
+static struct loader_block block_shared __attribute__((aligned(0x2000)));
+static uint8_t block_buffer[BLOCK_BUFFER_SIZE] __attribute__((aligned(0x2000)));
+static efi_block_io *block_bios[BLOCK_MAX_PARTS];
+static int block_enabled;
+
+static void
+block_start(void)
+{
+ static const efi_guid part_guid = EFI_PARTITION_INFO_GUID;
+ static const efi_guid blk_guid = EFI_BLOCK_IO_GUID;
+ efi_handle *handles;
+ uint64_t count;
+
+ memset(&block_shared, 0, sizeof(block_shared));
+ if (EFI_ERROR(BS->locate_handle_buffer(ByProtocol, &part_guid, NULL, &count, &handles))) {
+ print("ufs: no partition information: the block service is off\n");
+ return;
+ }
+ for (uint64_t i = 0; i < count && block_shared.parts < BLOCK_MAX_PARTS; i++) {
+ struct loader_block_part *bp = &block_shared.part[block_shared.parts];
+ efi_partition_info *info;
+ efi_block_io *bio;
+
+ if (EFI_ERROR(BS->handle_protocol(handles[i], &part_guid, (void **)&info)) || info->type != 2 ||
+ EFI_ERROR(BS->handle_protocol(handles[i], &blk_guid, (void **)&bio)) ||
+ bio->media->block_size == 0 || bio->media->block_size > BLOCK_BUFFER_SIZE) {
+ continue;
+ }
+ ufs_name(info, bp->name, sizeof(bp->name));
+ bp->block_size = bio->media->block_size;
+ bp->blocks = bio->media->last_block + 1;
+ block_bios[block_shared.parts++] = bio;
+ }
+ block_shared.magic = BLOCK_MAGIC;
+ block_shared.version = BLOCK_VERSION;
+ block_shared.size = sizeof(block_shared);
+ block_shared.buffer_size = BLOCK_BUFFER_SIZE;
+ block_shared.buffer_pa = (uint64_t)(uintptr_t)block_buffer;
+ clean_dcache((uint64_t)(uintptr_t)&block_shared, (uint64_t)(uintptr_t)&block_shared + sizeof(block_shared));
+ block_enabled = 1;
+ print("ufs: block service: %d partitions, page %x, buffer %x of %d KB\n", (uint64_t)block_shared.parts,
+ (uint64_t)(uintptr_t)&block_shared, block_shared.buffer_pa, (uint64_t)(BLOCK_BUFFER_SIZE >> 10));
+}
+
+/* one request, if there is one: called from core 0's watch loop while the kernel runs */
+static void
+block_poll(void)
+{
+ uint64_t ask = (uint64_t)(uintptr_t)&block_shared.ask;
+ uint32_t request, part;
+ uint64_t lba, count, bytes, started;
+ efi_status s = 0x8000000000000002ULL; /* EFI_INVALID_PARAMETER: a request out of bounds */
+
+ /* the kernel writes the request into memory past this core's cache: look at memory */
+ flush_dcache(ask, ask + sizeof(block_shared.ask));
+ request = block_shared.ask.request;
+ if (request == block_shared.answer.answered) {
+ return;
+ }
+ part = block_shared.ask.part;
+ lba = block_shared.ask.lba;
+ count = block_shared.ask.count;
+ started = READ_SYSREG(CNTVCT_EL0);
+ bytes = 0;
+ if (part < block_shared.parts && count != 0 && count <= BLOCK_BUFFER_SIZE / block_shared.part[part].block_size &&
+ lba < block_shared.part[part].blocks && count <= block_shared.part[part].blocks - lba) {
+ efi_block_io *bio = block_bios[part];
+
+ bytes = count * block_shared.part[part].block_size;
+ s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);
+ /* what the controller wrote is in memory; drop whatever this core held of the buffer */
+ flush_dcache((uint64_t)(uintptr_t)block_buffer, (uint64_t)(uintptr_t)block_buffer + bytes);
+ }
+ block_shared.answer.status = (int64_t)s;
+ block_shared.answer.ticks += READ_SYSREG(CNTVCT_EL0) - started;
+ if (EFI_ERROR(s)) {
+ block_shared.answer.failed++;
+ } else {
+ block_shared.answer.served++;
+ block_shared.answer.bytes += bytes;
+ }
+ __asm__ volatile ("dmb sy");
+ block_shared.answer.answered = request;
+ clean_dcache((uint64_t)(uintptr_t)&block_shared.answer,
+ (uint64_t)(uintptr_t)&block_shared.answer + sizeof(block_shared.answer));
+}
+
static void
charger_start(void)
{
@@ -1255,6 +1613,9 @@ run_on_other_cpu(uint64_t mpidr, uint64_t entry, uint64_t boot_args_pa, const st
status = OTHER_CPU_HUNG;
}
}
+ if (block_enabled) {
+ block_poll();
+ }
usb_poll_command();
for (volatile int i = 0; i < 20000; i++) {
}
@@ -1734,7 +2095,7 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
/* "bootcpu=N" among the kernel's arguments: run it on core N (see run_on_other_cpu) */
uint64_t this_mpidr = READ_SYSREG(MPIDR_EL1) & 0xFFFFFF, kernel_mpidr = this_mpidr;
/* "cpus=N" or "cpumask=M" there too: the kernel may have more cores than the one it starts on */
- int more_cores = 0, battery_requested = 0;
+ int more_cores = 0, battery_requested = 0, ufs_requested = 0;
/* "torch=N" (not 0) there too: the phone's flashlight is the kernel's (bsd/dev/redwood_torch.c) */
int torch_requested = 0;
/* "cam=5" there too: the phone's wide camera is iOS's (bsd/dev/redwood_camera.c) */
@@ -1751,10 +2112,20 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
torch_requested |= starts_with(c, "torch=") && (c == pi.cmdline || c[-1] == ' ') && c[6] >= '1' && c[6] <= '9';
camera_requested |= starts_with(c, "cam=5") && (c == pi.cmdline || c[-1] == ' ') && (c[5] == 0 || c[5] == ' ');
gamut_p3 |= starts_with(c, "gamut=p3") && (c == pi.cmdline || c[-1] == ' ');
+ if (starts_with(c, "ufs=") && (c == pi.cmdline || c[-1] == ' ') && c[4] >= '1' && c[4] <= '2' &&
+ (c[5] == 0 || c[5] == ' ')) {
+ ufs_requested = c[4] - '0';
+ }
}
if (battery_requested) {
charger_start();
}
+ if (ufs_requested >= 1) {
+ ufs_survey();
+ }
+ if (ufs_requested >= 2) {
+ block_start();
+ }
if (checkpoint == CHECKPOINT_SELFTEST) {
kernel_mpidr = this_mpidr;
}
@@ -1857,6 +2228,19 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
ba->commandLine[n++] = *c;
}
}
+ if (block_enabled) {
+ static const char prefix[] = "ufs_shared_pa=0x";
+ static const char hex[] = "0123456789abcdef";
+ uint64_t address = (uint64_t)(uintptr_t)&block_shared;
+
+ for (const char *c = prefix; *c && n < BOOT_LINE_LENGTH - 1; c++) {
+ ba->commandLine[n++] = *c;
+ }
+ for (int shift = 60; shift >= 0 && n < BOOT_LINE_LENGTH - 1; shift -= 4) {
+ ba->commandLine[n++] = hex[(address >> shift) & 0xf];
+ }
+ if (n < BOOT_LINE_LENGTH - 1) ba->commandLine[n++] = ' ';
+ }
if (battery_requested) {
static const char prefix[] = "battery_shared_pa=0x";
static const char hex[] = "0123456789abcdef";