Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
UFS: the kernel's own transfer list, in the loader's memory, put in for a run and taken out again
vcvkkf25e6a6f25d6
diff --git a/tools/flash-fingerprint.sh b/tools/flash-fingerprint.shnew file mode 100755index 0000000..2bdd23a--- /dev/null+++ b/tools/flash-fingerprint.sh@@ -0,0 +1,63 @@+#!/bin/bash+# A read-only fingerprint of the phone's flash, to tell whether anything has changed it.+#+# flash-fingerprint.sh android DIR from Android (adb, root): for each logical unit sda..sdf its+# size, sha256 of its first and last 8 blocks of 4 KB and+# `sgdisk --android-dump`; sha256 of userdata's first MB, of+# darwin's first 5628755968 bytes (the iOS volume image), the+# number of files under /sdcard, and whether Android is up+# (sys.boot_completed, /data mounted) -> DIR/android.txt+# flash-fingerprint.sh survey LOG DIR the loader's own survey (ufs=1 or 2) out of a run's log:+# each unit's GPT header and table CRCs, every partition's+# place, the hashes of the partitions there are images of,+# super's metadata, userdata's superblock (timings dropped)+# -> DIR/survey.txt+# flash-fingerprint.sh diff DIR1 DIR2 what differs+#+# Nothing here writes to the phone: dd only reads (if=), sgdisk only prints (--android-dump), and the+# script goes to the phone's shell on its standard input, not as a file.+set -e+case "$1" in+android)+ mkdir -p "$2"+ adb shell 'su -c sh' > "$2/android.txt.new" <<'PHONE'+for d in sda sdb sdc sdd sde sdf; do+ size=$(blockdev --getsize64 /dev/block/$d)+ n=$(($(blockdev --getsz /dev/block/$d) / 8)) # (the phone's shell counts in 32 bits: sectors, not bytes)+ first=$(dd if=/dev/block/$d bs=4096 count=8 2>/dev/null | sha256sum | cut -d' ' -f1)+ last=$(dd if=/dev/block/$d bs=4096 skip=$((n - 8)) count=8 2>/dev/null | sha256sum | cut -d' ' -f1)+ echo "$d: $size bytes, first 8 blocks $first, last 8 blocks $last"+ sgdisk --android-dump /dev/block/$d 2>&1 | sed "s/^/$d sgdisk: /"+done+echo "userdata ($(readlink /dev/block/by-name/userdata)) first MB: $(dd if=/dev/block/by-name/userdata bs=1048576 count=1 2>/dev/null | sha256sum | cut -d' ' -f1)"+echo "darwin ($(readlink /dev/block/by-name/darwin)) first 5628755968 bytes: $(dd if=/dev/block/by-name/darwin bs=4194304 count=1342 2>/dev/null | sha256sum | cut -d' ' -f1)"+echo "files under /sdcard: $(find /sdcard/ -type f 2>/dev/null | wc -l)"+echo "android: boot_completed=$(getprop sys.boot_completed), /data: $(mount | grep ' /data ' | cut -d' ' -f1,5)"+PHONE+ mv "$2/android.txt.new" "$2/android.txt"+ grep -E '^(sd[a-f]:|userdata|darwin|files|android)' "$2/android.txt"+ ;;+survey)+ mkdir -p "$3"+ awk '/^== KERNEL LOG ==/{exit} {print}' "$2" | tr -d '\r' | grep -a '^ufs: ' |+ grep -E '^ufs: (unit [0-9]+: (media|GPT|path)|part |[A-Za-z0-9_]+: fnv |super |userdata sb |no partition)' |+ sed -E 's/, [0-9]+ ms$//' > "$3/survey.txt"+ echo "$(wc -l < "$3/survey.txt") survey lines from $2"+ grep -E 'GPT header|fnv' "$3/survey.txt" | head -40+ ;;+diff)+ for f in android.txt survey.txt; do+ if [ -f "$2/$f" ] && [ -f "$3/$f" ]; then+ if diff -u "$2/$f" "$3/$f"; then+ echo "$f: the same"+ fi+ else+ echo "$f: not in both"+ fi+ done+ ;;+*)+ sed -n '2,20p' "$0"+ exit 1+ ;;+esacdiff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.cindex 8912d87..6a5403e 100644--- a/xnu/files/bsd/dev/redwood_ufs.c+++ b/xnu/files/bsd/dev/redwood_ufs.c@@ -43,6 +43,7 @@#include <sys/redwood_input.h>#include <libkern/crc.h>#include <vm/memory_types.h>+#include "redwood_ufshc.h"extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);extern void IOSleep(unsigned int milliseconds);@@ -51,7 +52,7 @@ extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length); /* osfmk/arm/* the page the loader publishes: the same layout as struct loader_block in xnuloader/loader.c */#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */-#define BLOCK_VERSION 2+#define BLOCK_VERSION 3#define BLOCK_MAX_PARTS 160#define BLOCK_MAX_UNITS 16#define BLOCK_PART_UNIT 1u /* a whole logical unit, not a partition */@@ -84,6 +85,20 @@ struct loader_block {uint64_t served, failed, bytes, ticks;} answer __attribute__((aligned(128)));struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));+ /* the loader's offer of an area for this kernel's own transfer request list, and UEFI's list */+ struct {+ uint64_t hc_pa;+ uint64_t area_pa;+ uint32_t area_size, ucd_offset, ucd_size, slots;+ uint32_t uefi_utrlba, uefi_utrlbau;+ } native __attribute__((aligned(128)));+ struct {+ volatile uint32_t take_back; /* the loader's: it takes the controller back, send nothing more */+ } loader_native __attribute__((aligned(128)));+ struct {+ volatile uint32_t list_in; /* this kernel's: its list is in */+ uint32_t switched_in, switched_out, reserved;+ } kernel_native __attribute__((aligned(128)));};#define UFS_PAGE 0x4000UL@@ -105,17 +120,13 @@ static uint64_t ufs_last_end;/** This kernel's own UFS driver (redwood_ufshc.c), with ufs_native=1: each partition's logical* unit and where it begins there, in bytes; reads go that way unless it fails, and then through- * the loader again.+ * the loader again. While its list is in the controller the loader is asked for nothing.*/-extern bool ufshc_probe(bool only);-extern bool ufshc_available(void);-extern int ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len);-extern void ufshc_report(const char *when);static uint8_t ufs_lun[BLOCK_MAX_PARTS]; /* 0xff: not known, read through the loader */static uint64_t ufs_lun_offset[BLOCK_MAX_PARTS];static bool ufs_native_mapped;static volatile bool ufs_native_on;-static uint64_t ufs_native_fallbacks;+static uint64_t ufs_native_fallbacks, ufs_mailbox_refused;static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);@@ -268,6 +279,15 @@ ufs_read_mailbox(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)}}lck_mtx_lock(&ufs_lock);+ if (ufshc_list_in()) {+ /* this kernel's own transfer list is in the controller: UEFI's driver must not be asked to ring */+ if (ufs_mailbox_refused++ < 4) {+ printf("ufs: %s: not asked of the loader while this kernel's own transfer list is in\n",+ ufs_page->part[part].name);+ }+ lck_mtx_unlock(&ufs_lock);+ return EIO;+ }while (len > 0 && error == 0) {uint64_t first = offset / nbs, skip = offset % nbs;uint64_t count = (skip + len + nbs - 1) / nbs, take;@@ -305,6 +325,30 @@ ufs_read_native(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)return ufshc_read(ufs_lun[part], ufs_lun_offset[part] + offset, dst, len);}+/* The switches of lists, with the loader's mailbox held: nothing is asked of UEFI meanwhile */+static int+ufs_native_in(bool prove)+{+ int error;++ lck_mtx_lock(&ufs_lock);+ /* (a loader that did not answer may still be inside UEFI's driver) */+ error = ufs_dead ? EIO : ufshc_switch_in(prove);+ lck_mtx_unlock(&ufs_lock);+ return error;+}++static int+ufs_native_out(const char *why)+{+ int error;++ lck_mtx_lock(&ufs_lock);+ error = ufshc_switch_out(why);+ lck_mtx_unlock(&ufs_lock);+ return error;+}+/* The way chosen (ufs_native=, kern.redwood.ufs_native); if this kernel's own fails, the loader's */static intufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)@@ -315,10 +359,13 @@ ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)if (error == 0) {return 0;}- ufs_native_on = false;- ufs_native_fallbacks++;- printf("ufs: %s: bytes %llu..%llu: this kernel's own read failed (%d); through the loader from now on\n",- ufs_page->part[part].name, offset, offset + len - 1, error);+ if (ufs_native_on) {+ ufs_native_on = false;+ ufs_native_fallbacks++;+ printf("ufs: %s: bytes %llu..%llu: this kernel's own read failed (%d); through the loader from now on\n",+ ufs_page->part[part].name, offset, offset + len - 1, error);+ }+ ufs_native_out("a read failed");}return ufs_read_mailbox(part, offset, dst, len);}@@ -594,234 +641,278 @@ ufs_fnv(uint64_t h, const uint8_t *p, uint64_t n)return h;}+static uint32_t+ufs_entry(const char *name)+{+ for (uint32_t i = 0; i < ufs_parts; i++) {+ if (strcmp((const char *)ufs_page->part[i].name, name) == 0) {+ return i;+ }+ }+ return BLOCK_MAX_PARTS;+}+/*- * Each unit's logical unit number: the one its device path gives, if it gives one, else the first- * of 0..7 not yet taken whose block 1 (the GPT header, with the unit's own GUID in it) is what the- * loader reads as block 1 of that unit. Either way it is so only if the two are the same.+ * Which logical unit each unit the loader lists is (their device paths do not say): unit k is+ * taken to be LUN k -- with ufs_native=1 only the unit darwin is on, which must be LUN 0 -- and it+ * is so only if block 1 (the GPT header, with the unit's own GUID in it) read through the loader+ * is the same as block 1 of that LUN read here; no other LUN is read. The loader's reads are made+ * first, with UEFI's list in; then this kernel's list goes in (with its proof) and stays in.*/static bool-ufs_native_map(void)+ufs_native_map(bool all_units){- uint8_t *a = kalloc_data(2 * 4096, Z_WAITOK), *b = a + 4096;- uint8_t unit_lun[BLOCK_MAX_UNITS];- uint32_t taken = 0, units = 0;- bool all = true;+ uint8_t *mine = kalloc_data(4096, Z_WAITOK), *theirs = kalloc_data(BLOCK_MAX_UNITS * 4096, Z_WAITOK);+ uint32_t units[BLOCK_MAX_UNITS], n = 0, mask = 0, darwin = ufs_entry("darwin");+ bool good = mine != NULL && theirs != NULL;- memset(unit_lun, 0xff, sizeof(unit_lun));memset(ufs_lun, 0xff, sizeof(ufs_lun));- if (a == NULL) {- return false;- }- for (uint32_t i = 0; i < ufs_parts && all; i++) {+ for (uint32_t i = 0; good && i < ufs_parts && n < BLOCK_MAX_UNITS; i++) {volatile struct loader_block_part *e = &ufs_page->part[i];- bool found = false;- if ((e->flags & BLOCK_PART_UNIT) == 0) {+ if ((e->flags & BLOCK_PART_UNIT) == 0 || e->unit >= 8) {continue;}- units++;- if (e->unit >= BLOCK_MAX_UNITS || e->block_size != 4096 || ufs_read_mailbox(i, 4096, a, 4096) != 0 ||- memcmp(a, "EFI PART", 8) != 0) {+ if (!all_units && (darwin == BLOCK_MAX_PARTS || ufs_page->part[darwin].unit != e->unit)) {+ continue;+ }+ if (e->block_size != 4096 || ufs_read_mailbox(i, 4096, theirs + n * 4096, 4096) != 0 ||+ memcmp(theirs + n * 4096, "EFI PART", 8) != 0) {printf("ufs: native: %s: no GPT header to know it by (block size %u)\n", e->name, e->block_size);- all = false;+ good = false;break;}- for (uint32_t k = 0; k < 8 && !found; k++) {- uint32_t lun = e->lun != 0xff ? e->lun : (k == 0 ? e->unit : k - 1);+ units[n++] = i;+ mask |= 1u << e->unit;+ }+ if (good && !all_units && (n != 1 || ufs_page->part[units[0]].unit != 0)) {+ printf("ufs: native: darwin is not on the first unit the loader lists: not LUN 0, not read here\n");+ good = false;+ }+ ufshc_set_luns(mask);+ if (good && ufs_native_in(true) != 0) {+ good = false;+ }+ for (uint32_t k = 0; good && k < n; k++) {+ volatile struct loader_block_part *e = &ufs_page->part[units[k]];- if (lun >= 8 || (taken & (1u << lun)) || (e->lun != 0xff && k > 0)) {- continue;- }- if (ufshc_read((uint8_t)lun, 4096, b, 4096) == 0 && memcmp(a, b, 4096) == 0) {- unit_lun[e->unit] = (uint8_t)lun;- taken |= 1u << lun;- found = true;- printf("ufs: native: %s is LUN %u (%s; its GPT header read both ways is the same)\n", e->name, lun,- e->lun != 0xff ? "its device path says so" : "found by its GPT header");- }- }- if (!found) {- printf("ufs: native: %s: no logical unit reads as it does through the loader\n", e->name);- all = false;+ if (ufshc_read((uint8_t)e->unit, 4096, mine, 4096) != 0 || memcmp(mine, theirs + k * 4096, 4096) != 0) {+ printf("ufs: native: %s: LUN %u's block 1 is NOT what the loader reads as its block 1\n", e->name, e->unit);+ good = false;+ } else {+ printf("ufs: native: %s is LUN %u: its GPT header read both ways is the same\n", e->name, e->unit);}}- kfree_data(a, 2 * 4096);- if (!all || units == 0) {+ if (!good) {+ ufs_native_out("the units could not be told");memset(ufs_lun, 0xff, sizeof(ufs_lun));- return false;- }- for (uint32_t i = 0; i < ufs_parts; i++) {- volatile struct loader_block_part *e = &ufs_page->part[i];+ } else {+ for (uint32_t i = 0; i < ufs_parts; i++) {+ volatile struct loader_block_part *e = &ufs_page->part[i];- if (e->unit < BLOCK_MAX_UNITS && unit_lun[e->unit] != 0xff && e->block_size == 4096) {- ufs_lun[i] = unit_lun[e->unit];- ufs_lun_offset[i] = e->start_lba * 4096;+ if (e->unit < 8 && (mask & (1u << e->unit)) && e->block_size == 4096) {+ ufs_lun[i] = (uint8_t)e->unit;+ ufs_lun_offset[i] = e->start_lba * 4096;+ }}}- return true;+ if (mine != NULL) {+ kfree_data(mine, 4096);+ }+ if (theirs != NULL) {+ kfree_data(theirs, BLOCK_MAX_UNITS * 4096);+ }+ return good;}-static uint64_t ufs_cmp_mailbox_ns, ufs_cmp_native_ns, ufs_cmp_bytes, ufs_cmp_regions, ufs_cmp_differ;-/*- * Bytes [offset, offset + len) of entry part, read both ways a megabyte at a time and compared;- * FNV-1a of what was read (each way), and of its first ref bytes if ref is not 0.+ * ufs_native_check=1: what this kernel's own driver reads is what the loader reads, byte for+ * byte. Spans are read a batch at a time, first all through the loader (UEFI's list in), then+ * all here (this kernel's list in, proved), and compared.*/+struct ufs_span {+ uint32_t part;+ uint64_t offset, len;+};++#define UFS_CMP_BUFFER (32u << 20)++static uint8_t *ufs_cmp_x, *ufs_cmp_y;+static uint64_t ufs_cmp_mailbox_ns, ufs_cmp_native_ns, ufs_cmp_bytes, ufs_cmp_spans, ufs_cmp_differ, ufs_cmp_failed;+static bool-ufs_compare(uint32_t part, uint64_t offset, uint64_t len, uint64_t ref, const char *what, bool quiet)+ufs_compare_batch(const struct ufs_span *s, uint32_t n, const char *what, bool quiet){- static uint8_t *x, *y;- uint64_t hx = 0xcbf29ce484222325ULL, hy = hx, href = 0, done = 0, t, ns, mailbox_ns = 0, native_ns = 0;- int ex = 0, ey = 0;+ uint64_t at, t, ns, hx = 0xcbf29ce484222325ULL, hy = hx, bytes = 0, mailbox_ns, native_ns;bool same = true;+ int e = 0;- if (x == NULL) {- x = kalloc_data(2 << 20, Z_WAITOK);- y = x + (1 << 20);- }- while (done < len && same && ex == 0 && ey == 0) {- uint64_t n = len - done < (1 << 20) ? len - done : (1 << 20);-- t = mach_absolute_time();- ex = ufs_read_mailbox(part, offset + done, x, n);- absolutetime_to_nanoseconds(mach_absolute_time() - t, &ns);- mailbox_ns += ns;- t = mach_absolute_time();- ey = ufs_read_native(part, offset + done, y, n);- absolutetime_to_nanoseconds(mach_absolute_time() - t, &ns);- native_ns += ns;- if (ex == 0 && ey == 0 && memcmp(x, y, n) != 0) {- uint64_t at = 0;-- while (x[at] == y[at]) {- at++;- }- printf("ufs: native check: %s %s: DIFFERENT at byte %llu (%02x through the loader, %02x here)\n",- ufs_page->part[part].name, what, offset + done + at, x[at], y[at]);- same = false;- }- if (ref != 0 && done < ref) {- uint64_t m = ref - done < n ? ref - done : n;-- href = ufs_fnv(done == 0 ? 0xcbf29ce484222325ULL : href, x, m);- }- hx = ufs_fnv(hx, x, n);- hy = ufs_fnv(hy, y, n);- done += n;+ t = mach_absolute_time();+ if (ufs_native_out("the loader reads, to compare") != 0) {+ e = EIO;+ }+ at = 0;+ for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {+ e = ufs_read_mailbox(s[i].part, s[i].offset, ufs_cmp_x + at, s[i].len);+ }+ absolutetime_to_nanoseconds(mach_absolute_time() - t, &mailbox_ns);+ t = mach_absolute_time();+ if (e == 0 && ufs_native_in(true) != 0) {+ e = EIO;}- if (ex != 0 || ey != 0) {- printf("ufs: native check: %s %s: read failed (through the loader %d, here %d)\n", ufs_page->part[part].name,- what, ex, ey);- same = false;+ at = 0;+ for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {+ e = ufs_read_native(s[i].part, s[i].offset, ufs_cmp_y + at, s[i].len);}- ufs_cmp_regions++;- ufs_cmp_bytes += done;+ absolutetime_to_nanoseconds(mach_absolute_time() - t, &native_ns);ufs_cmp_mailbox_ns += mailbox_ns;ufs_cmp_native_ns += native_ns;- ufs_cmp_differ += !same;- if (!quiet || !same) {- printf("ufs: native check: %s %s: %llu bytes from %llu %s: fnv %llx / %llx; loader %llu ms, here %llu ms\n",- ufs_page->part[part].name, what, done, offset, same ? "the same" : "NOT THE SAME", hx, hy,- mailbox_ns / 1000000, native_ns / 1000000);- if (ref != 0) {- printf("ufs: native check: %s: fnv %llx over the first %llu bytes (the loader's and ufs_check's hash)\n",- ufs_page->part[part].name, href, ref);+ if (e != 0) {+ printf("ufs: native check: %s: a read failed (%d): STOP\n", what, e);+ ufs_cmp_failed++;+ return false;+ }+ at = 0;+ for (uint32_t i = 0; i < n; i++) {+ if (memcmp(ufs_cmp_x + at, ufs_cmp_y + at, s[i].len) != 0) {+ uint64_t k = 0;++ while (ufs_cmp_x[at + k] == ufs_cmp_y[at + k]) {+ k++;+ }+ printf("ufs: native check: %s %s: DIFFERENT at byte %llu (%02x through the loader, %02x here): STOP\n",+ ufs_page->part[s[i].part].name, what, s[i].offset + k, ufs_cmp_x[at + k], ufs_cmp_y[at + k]);+ same = false;+ ufs_cmp_differ++;}+ hx = ufs_fnv(hx, ufs_cmp_x + at, s[i].len);+ hy = ufs_fnv(hy, ufs_cmp_y + at, s[i].len);+ at += s[i].len;+ bytes += s[i].len;+ }+ ufs_cmp_spans += n;+ ufs_cmp_bytes += bytes;+ if (!quiet || !same) {+ printf("ufs: native check: %s %s: %u span(s), %llu bytes from %llu: %s (fnv %llx / %llx); loader %llu ms, here "+ "%llu ms (switches included)\n", ufs_page->part[s[0].part].name, what, n, bytes, s[0].offset,+ same ? "the same" : "NOT THE SAME", hx, hy, mailbox_ns / 1000000, native_ns / 1000000);}return same;}-static uint32_t-ufs_entry(const char *name)+/* Bytes [offset, offset + len) of an entry, compared a buffer at a time; FNV-1a of the first ref bytes too */+static bool+ufs_compare(uint32_t part, uint64_t offset, uint64_t len, uint64_t ref, const char *what){- for (uint32_t i = 0; i < ufs_parts; i++) {- if (strcmp((const char *)ufs_page->part[i].name, name) == 0) {- return i;+ uint64_t done = 0, href = 0xcbf29ce484222325ULL;+ bool same = true;++ while (done < len && same) {+ struct ufs_span s = { part, offset + done, len - done < UFS_CMP_BUFFER ? len - done : UFS_CMP_BUFFER };++ same = ufs_compare_batch(&s, 1, what, true);+ if (same && ref > done) {+ href = ufs_fnv(href, ufs_cmp_x, ref - done < s.len ? ref - done : s.len);}+ done += s.len;}- return BLOCK_MAX_PARTS;+ printf("ufs: native check: %s %s: %llu bytes from %llu: %s%s", ufs_page->part[part].name, what, done, offset,+ same ? "the same" : "NOT THE SAME", ref ? "" : "\n");+ if (ref) {+ printf("; fnv %llx over the first %llu bytes (the loader's own survey hashes them)\n", href, ref);+ }+ return same;}-/*- * ufs_native_check=1: what this kernel's own driver reads is what the loader reads, byte for- * byte: every unit's GPT (protective MBR, header, table, and the backup table and header at the- * end), the partitions the loader hashes (the ROM's images: xbl, tz, modem, ...) whole, the start- * of super (its geometry and metadata), and of darwin 64 spread megabytes, 512 scattered blocks- * and some reads that begin and end inside a block.- */static voidufs_native_check(void){uint8_t *h = kalloc_data(4096, Z_WAITOK);uint32_t part;- uint64_t seed = 0x9e3779b97f4a7c15ULL;+ bool going = true;+ ufs_cmp_x = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);+ ufs_cmp_y = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);+ if (h == NULL || ufs_cmp_x == NULL || ufs_cmp_y == NULL) {+ printf("ufs: native check: no memory\n");+ return;+ }printf("ufs: native check: through the loader and through this kernel's own driver, compared\n");- for (uint32_t i = 0; h != NULL && i < ufs_parts; i++) {+ /* every unit's GPT: protective MBR, header, table; the backup table and header at the end */+ for (uint32_t i = 0; going && i < ufs_parts; i++) {volatile struct loader_block_part *e = &ufs_page->part[i];uint64_t entries_lba, count, size, table, alt, alt_entries;+ uint32_t want, got;- if ((e->flags & BLOCK_PART_UNIT) == 0 || ufs_read_mailbox(i, 4096, h, 4096) != 0) {+ if ((e->flags & BLOCK_PART_UNIT) == 0 || ufs_lun[i] == 0xff) {continue;}+ if (ufs_native_out("the loader reads, to compare") != 0 || ufs_read_mailbox(i, 4096, h, 4096) != 0) {+ going = false;+ break;+ }entries_lba = *(uint64_t *)(h + 72);count = *(uint32_t *)(h + 80);size = *(uint32_t *)(h + 84);alt = *(uint64_t *)(h + 32);table = (count * size + 4095) / 4096;- {- uint32_t want = *(uint32_t *)(h + 16), got;-- *(uint32_t *)(h + 16) = 0;- got = crc32(0, h, *(uint32_t *)(h + 12));- printf("ufs: native check: %s: GPT header crc %s, table at block %llu, %llu entries of %llu bytes, backup "- "header at block %llu\n", e->name, got == want ? "right" : "WRONG", entries_lba, count, size, alt);- }- ufs_compare(i, 0, (entries_lba + table) * 4096, 0, "GPT (MBR, header, table)", false);- if (alt < e->blocks && ufs_read_mailbox(i, alt * 4096, h, 4096) == 0 && memcmp(h, "EFI PART", 8) == 0) {- alt_entries = *(uint64_t *)(h + 72);- if (alt_entries <= alt) {- ufs_compare(i, alt_entries * 4096, (alt + 1 - alt_entries) * 4096, 0, "backup GPT (table, header)",- false);- }+ want = *(uint32_t *)(h + 16);+ *(uint32_t *)(h + 16) = 0;+ got = crc32(0, h, *(uint32_t *)(h + 12));+ printf("ufs: native check: %s (LUN %u): GPT header crc %s (%x), table at block %llu, %llu entries of %llu bytes, "+ "backup header at block %llu\n", e->name, ufs_lun[i], got == want ? "right" : "WRONG", got, entries_lba, count,+ size, alt);+ going = ufs_compare(i, 0, (entries_lba + table) * 4096, 0, "GPT (MBR, header, table)");+ if (!going || alt >= e->blocks || ufs_native_out("the loader reads, to compare") != 0 ||+ ufs_read_mailbox(i, alt * 4096, h, 4096) != 0 || memcmp(h, "EFI PART", 8) != 0 ||+ (alt_entries = *(uint64_t *)(h + 72)) > alt) {+ printf("ufs: native check: %s: no backup GPT header at block %llu: STOP\n", e->name, alt);+ going = false;+ break;}+ going = ufs_compare(i, alt_entries * 4096, (alt + 1 - alt_entries) * 4096, 0, "backup GPT (table, header)");}- for (part = 0; part < ufs_parts; part++) {- if (ufs_page->part[part].flags & BLOCK_PART_UNIT) {+ /* the partitions there are images of, whole */+ for (part = 0; going && part < ufs_parts; part++) {+ if ((ufs_page->part[part].flags & BLOCK_PART_UNIT) || ufs_lun[part] == 0xff) {continue;}- for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {+ for (unsigned r = 0; going && r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {if (ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {- ufs_compare(part, 0, ufs_part_bytes(part), ufs_references[r].bytes, "whole", false);+ going = ufs_compare(part, 0, ufs_part_bytes(part), ufs_references[r].bytes, "whole");}}}- if ((part = ufs_entry("super")) < BLOCK_MAX_PARTS) {- ufs_compare(part, 0, 8 << 20, 0, "first 8 MB (geometry, metadata)", false);+ /* super's geometry and metadata */+ if (going && (part = ufs_entry("super")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {+ going = ufs_compare(part, 0, 8 << 20, 0, "first 8 MB (geometry, metadata)");}- if ((part = ufs_entry("darwin")) < BLOCK_MAX_PARTS) {- uint64_t bytes = ufs_part_bytes(part), r0 = ufs_cmp_regions, d0 = ufs_cmp_differ;+ /* darwin: 32 spread megabytes, 256 scattered blocks, 32 pieces that begin and end inside a block */+ if (going && (part = ufs_entry("darwin")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {+ struct ufs_span s[256];+ uint64_t bytes = ufs_part_bytes(part), seed = 0x9e3779b97f4a7c15ULL, total = 0;- for (uint64_t k = 0; k < 64; k++) {- ufs_compare(part, (bytes - (1 << 20)) / 64 * k / 4096 * 4096, 1 << 20, 0, "spread megabyte", true);+ for (uint32_t k = 0; k < 32; k++) {+ s[k] = (struct ufs_span){ part, (bytes - (1 << 20)) / 32 * k / 4096 * 4096, 1 << 20 };}- for (uint64_t k = 0; k < 512; k++) {+ going = ufs_compare_batch(s, 32, "spread megabytes", false);+ for (uint32_t k = 0; k < 256; k++) {seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;- ufs_compare(part, (seed >> 16) % (bytes / 4096) * 4096, 4096, 0, "scattered block", true);+ s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes / 4096) * 4096, 4096 };}- for (uint64_t k = 0; k < 64; k++) {+ going = going && ufs_compare_batch(s, 256, "scattered blocks", false);+ for (uint32_t k = 0; k < 32; k++) {seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;- ufs_compare(part, (seed >> 16) % (bytes - (2 << 20)), 1 + (seed >> 40) % (1 << 20), 0, "unaligned piece", true);+ s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes - (2 << 20)), 1 + (seed >> 40) % (512 << 10) };+ total += s[k].len;}- printf("ufs: native check: darwin: %llu spread megabytes, scattered blocks and unaligned pieces: %llu not the same\n",- ufs_cmp_regions - r0, ufs_cmp_differ - d0);+ going = going && ufs_compare_batch(s, 32, "unaligned pieces", false);}- if (h != NULL) {- kfree_data(h, 4096);- }- printf("ufs: native check done: %llu regions, %llu bytes, %llu NOT THE SAME; through the loader %llu ms, "- "through this kernel's own driver %llu ms\n", ufs_cmp_regions, ufs_cmp_bytes, ufs_cmp_differ,- ufs_cmp_mailbox_ns / 1000000, ufs_cmp_native_ns / 1000000);+ kfree_data(h, 4096);+ printf("ufs: native check %s: %llu spans, %llu bytes, %llu NOT THE SAME, %llu failed reads; through the loader %llu ms, "+ "through this kernel's own driver %llu ms (list switches included)\n", going ? "done" : "STOPPED", ufs_cmp_spans,+ ufs_cmp_bytes, ufs_cmp_differ, ufs_cmp_failed, ufs_cmp_mailbox_ns / 1000000, ufs_cmp_native_ns / 1000000);+ ufs_native_out("after the check");ufshc_report("after the check");}@@ -835,10 +926,15 @@ ufs_sysctl_native(__unused struct sysctl_oid *oidp, __unused void *arg1, __unuseif (error != 0 || !changed) {return error;}- if (value != 0 && !(ufs_native_mapped && ufshc_available())) {- return ENXIO;+ if (value != 0) {+ if (!(ufs_native_mapped && ufshc_available()) || ufs_native_in(true) != 0) {+ return ENXIO;+ }+ ufs_native_on = true;+ } else {+ ufs_native_on = false;+ ufs_native_out("kern.redwood.ufs_native=0");}- ufs_native_on = value != 0;return 0;}SYSCTL_PROC(_kern_redwood, OID_AUTO, ufs_native, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,@@ -901,25 +997,53 @@ redwood_ufs_init(void)printf("ufs: %u partitions of the phone's storage as /dev/ufs_NAME and /dev/rufs_NAME, read only, ""through the loader's page at %llx and its %u KB buffer\n", made, pa, ufs_buffer_size >> 10);- /* ufs_native=1: through this kernel's own driver instead, if the controller is as expected; 2: only look */+ /*+ * ufs_native=1: through this kernel's own driver instead, its own transfer list in the+ * controller for the run, if all is as expected; 2: only look; 3: a dry run -- the list in and+ * out again, and in once more for the end of the run to take out, no command sent.+ */check = 0;memset(ufs_lun, 0xff, sizeof(ufs_lun));if (PE_parse_boot_argn("ufs_native", &check, sizeof(check)) && check != 0) {- if (ufshc_probe(check == 2) && ufs_native_map()) {+ struct ufshc_loader offer = {+ .hc_pa = ufs_page->native.hc_pa, .area_pa = ufs_page->native.area_pa,+ .area_size = ufs_page->native.area_size, .ucd_offset = ufs_page->native.ucd_offset,+ .ucd_size = ufs_page->native.ucd_size, .slots = ufs_page->native.slots,+ .uefi_lo = ufs_page->native.uefi_utrlba, .uefi_hi = ufs_page->native.uefi_utrlbau,+ .take_back = &ufs_page->loader_native.take_back, .list_in = &ufs_page->kernel_native.list_in,+ .switched_in = (uint32_t *)&ufs_page->kernel_native.switched_in,+ .switched_out = (uint32_t *)&ufs_page->kernel_native.switched_out,+ };+ uint32_t mode = check, compare = 0;++ PE_parse_boot_argn("ufs_native_check", &compare, sizeof(compare));+ if (!ufshc_probe(&offer, mode == 2)) {+ if (mode != 2) {+ printf("ufs: this kernel's own UFS driver is not used: through the loader\n");+ }+ } else if (mode == 3) {+ printf("ufs: native: dry run: this kernel's list in and out, no command\n");+ if (ufs_native_in(false) == 0) {+ IOSleep(1000);+ ufs_native_out("dry run");+ IOSleep(500);+ ufs_native_in(false);+ printf("ufs: native: dry run: the list stays in for the end of the run\n");+ }+ } else if (ufs_native_map(compare != 0)) {ufs_native_mapped = true;+ if (compare != 0) {+ /* right here, in bsd_init's thread, and then back to the loader */+ ufs_native_check();+ redwood_ufs_report("after the check");+ loader_return();+ }ufs_native_on = true;- printf("ufs: the partitions are read by this kernel's own UFS driver (kern.redwood.ufs_native=0: "- "through the loader)\n");- } else if (check != 2) {+ printf("ufs: the partitions on LUN 0 are read by this kernel's own UFS driver, its list in the controller "+ "(kern.redwood.ufs_native=0: through the loader)\n");+ } else {printf("ufs: this kernel's own UFS driver is not used: through the loader\n");}- check = 0;- if (ufs_native_mapped && PE_parse_boot_argn("ufs_native_check", &check, sizeof(check)) && check != 0) {- /* right here, in bsd_init's thread, and then back to the loader */- ufs_native_check();- redwood_ufs_report("after the check");- loader_return();- }}if (PE_parse_boot_argn("ufs_bench", &check, sizeof(check)) && check != 0) {diff --git a/xnu/files/bsd/dev/redwood_ufshc.c b/xnu/files/bsd/dev/redwood_ufshc.cindex 636f9fc..acaf5d6 100644--- a/xnu/files/bsd/dev/redwood_ufshc.c+++ b/xnu/files/bsd/dev/redwood_ufshc.c@@ -1,62 +1,57 @@/*- * The phone's UFS host controller, driven from this kernel -- read only, and shared with UEFI.+ * The phone's UFS host controller, driven from this kernel -- read only, next to UEFI's driver.** Reading the storage through the loader (redwood_ufs.c) is one request at a time: Qualcomm's- * UFS driver in UEFI (UFSDxe) reads synchronously on core 0, and its asynchronous reads finish- * only from a UEFI timer that does not tick there while the kernel runs. The controller itself- * takes up to 32 requests at once. This file puts requests of its own into it, next to UEFI's,- * without taking the controller away from UEFI: UEFI still reads for the loader and writes the- * run's log at its end.+ * UFSDxe reads synchronously on core 0, and its transfer request list has room for one request+ * only (UTRLBA points at a single descriptor, with UFSDxe's command descriptor right behind it,+ * where slots 8.. would be). The controller takes 32 at once. So for the length of a run this+ * kernel puts a list of its own into the controller: with nothing in flight, the list is stopped+ * (UTRLRSR 0), UTRLBA/UTRLBAU are pointed at the new one, the list is started again; at the end+ * (redwood_ufs_quiesce, from loader_return), what is in flight is waited for, the list stopped,+ * UEFI's address written back, the list started. The list and its command descriptors are in+ * memory the loader owns (it offers it in its page, with UEFI's address); the loader checks the+ * address on every way back from a run and puts UEFI's back itself if the kernel did not+ * (xnuloader/loader.c, hc_take_back), and neither reads nor writes for the kernel or for its log+ * while another list is in. While this kernel's list is in, it asks the loader for nothing.*- * What it does not do, ever: reset or enable the controller, start the link, send UIC commands- * (DME_GET/DME_SET), change the request lists' addresses, the interrupt enables, aggregation,- * clocks or power modes, clear a request (UTRLCLR), or touch the task management list. It reads- * the registers that say where UEFI's request list is and whether the controller runs, writes- * request descriptors into slots of that list that UEFI does not use, rings their doorbell bits- * (UTRLDBR is write-one-to-set: the other bits are not affected), and polls UTRLDBR until the- * controller clears them. Requests are made with the interrupt bit clear, so that they raise no- * completion status of their own in IS.+ * Nothing else of the controller is changed: no reset, no link startup, no UIC command, no+ * interrupt enable, aggregation, clock or power setting; the task management list is not+ * touched; IS and UTRLCNR are only read. UTRLCLR is written only to clear this kernel's own+ * slots when they do not finish at the end.*- * Read only by construction: every command passes through ufshc_permitted() on its way to the- * doorbell, a whitelist that lets through SCSI READ(10)/READ(16), TEST UNIT READY, REQUEST- * SENSE, INQUIRY, READ CAPACITY(10/16) to a normal logical unit and query *read* requests (read- * descriptor, attribute, flag), and nothing else; no data ever goes from host to device. What is- * refused is logged and never reaches the doorbell. This file builds READ(10) only.+ * Read only by construction: a command is made in a buffer of its own, and only one that passes+ * ufshc_permitted() is ever written into a slot; what the slot then holds is read back and must+ * be the same and pass again before the doorbell is rung. The whitelist checks three things of+ * every command: the SCSI operation (READ(10)/READ(16), TEST UNIT READY, REQUEST SENSE, INQUIRY,+ * READ CAPACITY(10/16), nothing else -- no query, no task management, no NOP), the data+ * direction in the request descriptor (from the device or none, never to it), and the command+ * UPIU's flags (never W); and the logical unit: never a well-known one (REPORT LUNS, BOOT, RPMB,+ * the device), and only those reads are allowed to go to now (ufshc_set_luns: LUN 0, where the+ * iOS volume is, unless a comparison with the loader's reads asks for more). A read's length in+ * the CDB must be what the UPIU expects. It is tried at the probe on made-up commands that must+ * pass or be refused. This file builds READ(10) and TEST UNIT READY only.*- * WHAT THE PROBE FOUND ON THIS PHONE, AND WHY THIS PATH IS NOT USED THERE: Qualcomm's UFSDxe sets- * up a list of one. The controller has 32 slots, UTRLBA points at 0xccf96000, slot 0 there is- * UEFI's request descriptor -- and its command descriptor is at 0xccf96100, inside the 1 KB the- * other 31 would take: "slots" 8..14 are its command UPIU, response and PRD table (pointing into- * the loader's buffer), and 1..7 and 15..17 hold other data of the driver. No slot but UEFI's own- * is free memory, so the probe refuses (twice over: the command descriptor inside the list, and- * slots 16..31 that are not descriptors) and the storage is read through the loader. What- * follows is how the path works where the list does have room.+ * Every slot of this list holds a whitelisted command at all times: TEST UNIT READY until it is+ * used for a read. Slot 0 is kept for the proof that the controller does read this list: after+ * every switch, before any read, slot 0's TEST UNIT READY is rung and must come back in this+ * list (its status written here). Were the controller still reading UEFI's list, the doorbell+ * would run UEFI's slot 0 instead -- so first UEFI's slot 0 must hold a read too (it does: the+ * loader's last use of the storage before a run is reading back the log it wrote), and the+ * proof costs nothing either way. Reads go in slots 1..31.*- * Which slots: UEFI's driver takes the first free slot for each request it makes, synchronously,- * so it uses slot 0 (and during a run it reads only when the loader is asked to, and writes only- * the log, after the run). This file keeps to the upper half of the list, 16..31, and makes its- * requests in the even ones: a descriptor is 32 bytes and a cache line 64, and a line is only- * ever written back here while its one request is not in flight, so nothing the controller- * writes into a descriptor (its status) is overwritten by a stale copy of the line's other half.+ * The doorbell is rung only with interrupts off, after looking that this kernel's list is the+ * one in the controller and running, that the slot is free, and that the loader is not taking+ * the controller back (it says so in its page, then waits before it does).*- * At the probe (ufshc_probe), anything unexpected leaves this path off and the loader's in use:- * the controller not enabled and running, a version other than 2.x/3.x, fewer than 32 slots, no- * 64-bit addressing, the list in memory this kernel manages, any of the slots 16..31 busy or- * holding something that is not a request descriptor, a fatal error status. The slots' original- * contents are kept and put back at the end of the run (redwood_ufs_quiesce, from- * loader_return), after waiting until none of them is busy, so that the list is as UEFI left it- * when it writes the run's log.- *- * Memory: each slot has a page for its command descriptor (command UPIU at 0, response UPIU at- * 512, the PRD table at 1024) and a contiguous buffer the controller reads into; the descriptors- * are cleaned to the point of coherency before the doorbell and invalidated before they are- * read back, and so is the data, so it does not matter whether the controller's accesses are- * coherent with the caches.+ * The list and the command descriptors are mapped here without caching (nothing of them is ever+ * in a cache line of this kernel's to be written back over what the controller writes); the data+ * buffers are this kernel's memory, cleaned and invalidated around every read.*/#include <kern/clock.h>#include <kern/locks.h>#include <kern/thread.h>#include <libkern/libkern.h>+#include <stdarg.h>#include <mach/mach_types.h>#include <mach/vm_param.h>#include <pexpert/pexpert.h>@@ -67,14 +62,15 @@#include <sys/sysctl.h>#include <vm/vm_kern.h>#include <vm/memory_types.h>+#include "redwood_ufshc.h"extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);extern uint64_t kvtophys(vm_offset_t va); /* pmap_paddr_t */extern boolean_t pmap_valid_address(uint64_t addr);extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length);-extern void CleanPoC_DcacheRegion(vm_offset_t va, size_t length);extern void IOSleep(unsigned int milliseconds);+extern boolean_t ml_set_interrupts_enabled(boolean_t enable);#define UFSHC_PHYS 0x1d84000ULL /* ufshc@1d84000, "ufs_mem", in the phone's Android device tree */#define UFSHC_SIZE 0x3000@@ -93,6 +89,7 @@ extern void IOSleep(unsigned int milliseconds);#define UFSHC_UTRLBA 0x50#define UFSHC_UTRLBAU 0x54#define UFSHC_UTRLDBR 0x58+#define UFSHC_UTRLCLR 0x5c#define UFSHC_UTRLRSR 0x60#define UFSHC_UTRLCNR 0x64#define UFSHC_UTMRLBA 0x70@@ -105,9 +102,9 @@ extern void IOSleep(unsigned int milliseconds);#define HCS_DP (1u << 0)#define HCS_UTRLRDY (1u << 1)#define IS_FATAL ((1u << 2) | (1u << 11) | (1u << 12) | (1u << 16) | (1u << 17)) /* UE, DFES, UTPES, HCFES, SBFES */-#define IS_UTRCS (1u << 0)-/* the request descriptor's first word: UFS storage command type, data direction */+/* the request descriptor's first word: command type, data direction */+#define UTRD_CT_MASK 0xf0000000u#define UTRD_CT_UFS (1u << 28)#define UTRD_DD_MASK (3u << 25)#define UTRD_DD_NONE (0u << 25)@@ -117,44 +114,53 @@ extern void IOSleep(unsigned int milliseconds);#define OCS_INVALID 0xf#define UPIU_COMMAND 0x01-#define UPIU_QUERY_REQUEST 0x16#define UPIU_RESPONSE 0x21#define UPIU_FLAG_READ 0x40#define UPIU_FLAG_WRITE 0x20+#define LUN_WELL_KNOWN 0x80-#define UCD_SIZE 0x4000 /* a page per slot */+#define SLOTS 32 /* this kernel's list */+#define SLOT_PROOF 0#define UCD_RESPONSE 512#define UCD_PRDT 1024#define PRD_MAX (256u << 10) /* the most one table entry describes */--#define SLOT_FIRST 16 /* this kernel's part of the list: 16..31 */-#define SLOTS_RESERVED 0xffff0000u-#define SLOT_MAX 8-#define SLOT_BUFFER_DEFAULT (512u << 10)+#define SLOT_BUFFER_DEFAULT (256u << 10)#define BLOCK 4096u /* the logical block size of every unit here */#define SPIN_US 5000 /* look without sleeping this long, then a millisecond at a time */#define TIMEOUT_MS 10000-struct ufshc_slot {- uint32_t number; /* in the list */- volatile uint32_t *utrd; /* its descriptor, in UEFI's list */- uint8_t *ucd;+struct hc_cmd {+ uint8_t upiu[32];+ uint32_t dw0;+};++struct hc_slot {+ uint32_t number; /* in this kernel's list */+ volatile uint32_t *utrd; /* its descriptor (the loader's memory, not cached here) */+ volatile uint32_t *ucd; /* its command descriptor (likewise) */uint64_t ucd_pa;- uint8_t *data;+ uint8_t *data; /* this kernel's memory, cached */uint64_t data_pa;- uint32_t original[8]; /* what the descriptor held at the probe, put back at the end */};static vm_offset_t hc; /* the registers */-static volatile uint32_t *hc_list; /* UEFI's request list, cacheable */-static uint64_t hc_list_pa;-static struct ufshc_slot hc_slots[SLOT_MAX];-static uint32_t hc_nslots, hc_free, hc_slot_buffer = SLOT_BUFFER_DEFAULT;-static uint32_t hc_is_at_probe;-static bool hc_usable; /* the probe found it as expected and the memory is there */-static volatile bool hc_closed, hc_dead;-static uint64_t hc_requests, hc_errors, hc_bytes, hc_wait_us, hc_refused, hc_most_in_flight;+static struct ufshc_loader hc_loader;+static uint32_t hc_list_lo, hc_list_hi; /* this kernel's list's address */+static volatile uint32_t *hc_uefi_list; /* UEFI's list, not cached here, only read */+static uint64_t hc_uefi_map_pa, hc_uefi_map_size;+static vm_offset_t hc_uefi_map_va;+static struct hc_slot hc_slots[SLOTS];+static uint32_t hc_nslots; /* slots 1..hc_nslots carry reads */+static uint32_t hc_free; /* bit i: hc_slots[i] free */+static uint32_t hc_slot_buffer = SLOT_BUFFER_DEFAULT;+static uint32_t hc_lun_mask = 1; /* the logical units reads may go to: LUN 0 */+static bool hc_usable;+static uint32_t hc_leave; /* ufs_native_leave=1 (a test): the list left in at the end, for the loader */+static volatile bool hc_in, hc_closed, hc_dead;+static uint32_t hc_is_in, hc_cnr_in; /* IS and UTRLCNR when this kernel's list went in */+static uint64_t hc_requests, hc_errors, hc_bytes, hc_wait_us, hc_refused, hc_most_in_flight, hc_proofs;+static uint64_t hc_switch_in_count, hc_switch_out_count;static uint32_t hc_in_flight;static LCK_GRP_DECLARE(hc_lck_grp, "redwood-ufshc");@@ -162,15 +168,59 @@ static LCK_MTX_DECLARE(hc_lock, &hc_lck_grp);SYSCTL_DECL(_kern_redwood);SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_requests,- "requests this kernel's own UFS driver has had answered");+ "reads this kernel's own UFS driver has had answered");SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_errors,- "requests of this kernel's own UFS driver that failed");+ "reads of this kernel's own UFS driver that failed");SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_bytes,"bytes this kernel's own UFS driver has read");SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_wait_us,- "microseconds spent waiting for the controller, summed over the requests");+ "microseconds spent waiting for the controller, summed over the reads");SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_refused, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_refused,"commands the whitelist refused (there should never be one)");+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_most_in_flight, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_most_in_flight,+ "the most reads this kernel's own UFS driver has had in flight at once");++/*+ * What this file says goes to the kernel's log, and the last lines are kept: the start of a long+ * run's log does not come back to the loader (the buffer wraps), so they are said again at the+ * end, into the loader's log directly (redwood_ufs_quiesce; printf is not for that moment).+ */+extern void loader_log_putc(char c);+#define KEPT_LINES 64+static char hc_kept[KEPT_LINES][208];+static uint32_t hc_kept_count;++static void+hc_say(const char *format, ...) __printflike(1, 2);+static void+hc_say(const char *format, ...)+{+ char line[208];+ va_list list;++ va_start(list, format);+ vsnprintf(line, sizeof(line), format, list);+ va_end(list);+ printf("%s", line);+ strlcpy(hc_kept[hc_kept_count++ % KEPT_LINES], line, sizeof(hc_kept[0]));+}++/* the same, at the end: into the loader's log only */+static void+hc_say_late(const char *format, ...) __printflike(1, 2);+static void+hc_say_late(const char *format, ...)+{+ char line[208];+ va_list list;++ va_start(list, format);+ vsnprintf(line, sizeof(line), format, list);+ va_end(list);+ for (const char *c = line; *c != 0; c++) {+ loader_log_putc(*c);+ }+}static inline uint32_thc_read(uint32_t reg)@@ -178,14 +228,12 @@ hc_read(uint32_t reg)return *(volatile uint32_t *)(hc + reg);}-/* Nothing more is sent from here on; whoever waits for a slot is told. */-static void-hc_off(void)+static inline void+hc_write(uint32_t reg, uint32_t value){- hc_dead = true;- lck_mtx_lock(&hc_lock);- wakeup(&hc_free);- lck_mtx_unlock(&hc_lock);+ __asm__ volatile ("dsb sy" ::: "memory");+ *(volatile uint32_t *)(hc + reg) = value;+ __asm__ volatile ("dsb sy" ::: "memory");}static uint64_t@@ -197,109 +245,167 @@ hc_us_since(uint64_t start)return ns / 1000;}-/* ---- the whitelist: the one check every command passes on its way to the doorbell ---- */+/* ---- the whitelist: the one check every command passes before it goes into a slot ---- */static const char *-hc_refusal(const uint8_t *upiu, uint32_t dw0)+hc_refusal(const uint8_t *upiu, uint32_t dw0, uint32_t lun_mask, bool any_command_type){const uint8_t *cdb = upiu + 16;uint32_t dd = dw0 & UTRD_DD_MASK;+ uint64_t expected = (uint64_t)upiu[12] << 24 | (uint64_t)upiu[13] << 16 | (uint64_t)upiu[14] << 8 | upiu[15];+ uint8_t lun = upiu[2];+ /* the data direction: from the device, or none -- never to it */if (dd == UTRD_DD_OUT || dd == UTRD_DD_MASK) {return "data from host to device";}- if ((dw0 & 0xf0000000u) != UTRD_CT_UFS) {+ if (!any_command_type && (dw0 & UTRD_CT_MASK) != UTRD_CT_UFS) {return "not a UFS storage command type";}- switch (upiu[0] & 0x3f) {- case UPIU_COMMAND:- if (upiu[1] & UPIU_FLAG_WRITE) {- return "a command that writes";- }- if (upiu[2] >= 8) {- return "a well-known logical unit";- }- if (upiu[4] & 0x0f) {- return "not the SCSI command set";- }- switch (cdb[0]) {- case 0x28: /* READ(10) */- case 0x88: /* READ(16) */- return dd == UTRD_DD_IN && (upiu[1] & UPIU_FLAG_READ) ? NULL : "a read without data in";- case 0x00: /* TEST UNIT READY */- return dd == UTRD_DD_NONE ? NULL : "TEST UNIT READY with data";- case 0x03: /* REQUEST SENSE */- case 0x12: /* INQUIRY */- case 0x25: /* READ CAPACITY(10) */- return NULL;- case 0x9e: /* SERVICE ACTION IN(16): READ CAPACITY(16) only */- return (cdb[1] & 0x1f) == 0x10 ? NULL : "a SERVICE ACTION IN other than READ CAPACITY(16)";- default:- return "a SCSI operation not on the list";- }- case UPIU_QUERY_REQUEST:- if (upiu[5] != 0x01) {- return "a query that is not a standard read request";+ /* a SCSI command, and its flags: never W */+ if ((upiu[0] & 0x3f) != UPIU_COMMAND) {+ return "not a SCSI command (a query, task management, NOP, ...)";+ }+ if (upiu[1] & UPIU_FLAG_WRITE) {+ return "the W flag";+ }+ if (upiu[4] & 0x0f) {+ return "not the SCSI command set";+ }+ /* a normal logical unit, and one reads may go to now */+ if (lun & LUN_WELL_KNOWN) {+ return "a well-known logical unit";+ }+ if (lun >= 8 || (lun_mask & (1u << lun)) == 0) {+ return "a logical unit reads do not go to now";+ }+ /* the operation */+ switch (cdb[0]) {+ case 0x28: /* READ(10) */+ case 0x88: { /* READ(16) */+ uint64_t blocks = cdb[0] == 0x28 ? ((uint64_t)cdb[7] << 8 | cdb[8]) :+ ((uint64_t)cdb[10] << 24 | (uint64_t)cdb[11] << 16 | (uint64_t)cdb[12] << 8 | cdb[13]);++ if (dd != UTRD_DD_IN || (upiu[1] & UPIU_FLAG_READ) == 0) {+ return "a read without data from the device";}- if (upiu[12] != 0x01 && upiu[12] != 0x03 && upiu[12] != 0x05) {- return "a query opcode other than read descriptor/attribute/flag";+ if (blocks == 0 || expected != blocks * BLOCK) {+ return "a read whose length is not what the UPIU expects";}- return dd == UTRD_DD_NONE ? NULL : "a query with data";+ return NULL;+ }+ case 0x00: /* TEST UNIT READY */+ return dd == UTRD_DD_NONE && expected == 0 && (upiu[1] & UPIU_FLAG_READ) == 0 ? NULL :+ "TEST UNIT READY with data";+ case 0x03: /* REQUEST SENSE */+ case 0x12: /* INQUIRY */+ case 0x25: /* READ CAPACITY(10) */+ return NULL;+ case 0x9e: /* SERVICE ACTION IN(16): READ CAPACITY(16) only */+ return (cdb[1] & 0x1f) == 0x10 ? NULL : "a SERVICE ACTION IN other than READ CAPACITY(16)";default:- return "a transaction that is neither a command nor a query";+ return "a SCSI operation not on the list";}}static boolufshc_permitted(const uint8_t *upiu, uint32_t dw0){- const char *why = hc_refusal(upiu, dw0);+ const char *why = hc_refusal(upiu, dw0, hc_lun_mask, false);if (why == NULL) {return true;}hc_refused++;- printf("ufs: native: REFUSED %s: UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x, "- "descriptor %08x; nothing was sent\n", why, upiu[0], upiu[1], upiu[2], upiu[3], upiu[16], upiu[17], upiu[18],+ hc_say("ufs: native: REFUSED (%s): UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x, "+ "descriptor %08x; it goes into no slot\n", why, upiu[0], upiu[1], upiu[2], upiu[3], upiu[16], upiu[17], upiu[18],upiu[19], upiu[20], upiu[21], upiu[22], upiu[23], upiu[24], upiu[25], dw0);return false;}+static void+hc_make_read(struct hc_cmd *c, uint8_t lun, uint8_t tag, uint32_t lba, uint32_t blocks)+{+ uint32_t bytes = blocks * BLOCK;++ bzero(c, sizeof(*c));+ c->upiu[0] = UPIU_COMMAND;+ c->upiu[1] = UPIU_FLAG_READ; /* simple task attribute */+ c->upiu[2] = lun;+ c->upiu[3] = tag;+ c->upiu[12] = (uint8_t)(bytes >> 24);+ c->upiu[13] = (uint8_t)(bytes >> 16);+ c->upiu[14] = (uint8_t)(bytes >> 8);+ c->upiu[15] = (uint8_t)bytes;+ c->upiu[16] = 0x28; /* READ(10) */+ c->upiu[18] = (uint8_t)(lba >> 24);+ c->upiu[19] = (uint8_t)(lba >> 16);+ c->upiu[20] = (uint8_t)(lba >> 8);+ c->upiu[21] = (uint8_t)lba;+ c->upiu[23] = (uint8_t)(blocks >> 8);+ c->upiu[24] = (uint8_t)blocks;+ c->dw0 = UTRD_CT_UFS | UTRD_DD_IN; /* no interrupt, no inline encryption */+}++static void+hc_make_tur(struct hc_cmd *c, uint8_t lun, uint8_t tag)+{+ bzero(c, sizeof(*c));+ c->upiu[0] = UPIU_COMMAND;+ c->upiu[2] = lun;+ c->upiu[3] = tag;+ c->upiu[16] = 0x00; /* TEST UNIT READY */+ c->dw0 = UTRD_CT_UFS | UTRD_DD_NONE;+}+/** The whitelist tried on commands made up here and never sent: those that must be refused,- * and the read that must pass. Pure bytes; nothing touches the controller.+ * and those that must pass. Pure bytes; nothing touches the controller.*/static boolhc_whitelist_selftest(void){static const struct {- uint8_t type, flags, lun, cdb0, cdb1, query_function, query_opcode;+ uint8_t type, flags, lun, op, cdb1;+ uint16_t blocks; /* the CDB's length (READ(10)/(16)) */+ uint32_t expected; /* the UPIU's */uint32_t dd;bool pass;const char *what;} cases[] = {- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 0, 0, UTRD_DD_IN, true, "READ(10)" },- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x88, 0, 0, 0, UTRD_DD_IN, true, "READ(16)" },- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x25, 0, 0, 0, UTRD_DD_IN, true, "READ CAPACITY(10)" },- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x01, 0x01, UTRD_DD_NONE, true, "QUERY READ DESCRIPTOR" },- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x2a, 0, 0, 0, UTRD_DD_OUT, false, "WRITE(10)" },- { UPIU_COMMAND, 0, 0, 0x2a, 0, 0, 0, UTRD_DD_NONE, false, "WRITE(10) without the W flag" },- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x2a, 0, 0, 0, UTRD_DD_IN, false, "WRITE(10) dressed as a read" },- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x8a, 0, 0, 0, UTRD_DD_OUT, false, "WRITE(16)" },- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x42, 0, 0, 0, UTRD_DD_OUT, false, "UNMAP" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, true, "READ(10) of LUN 0" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x88, 0, 1, 4096, UTRD_DD_IN, true, "READ(16) of LUN 0" },+ { UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_NONE, true, "TEST UNIT READY of LUN 0" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x25, 0, 0, 8, UTRD_DD_IN, true, "READ CAPACITY(10)" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x2a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(10)" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x8a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(16)" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x2a, 0, 1, 4096, UTRD_DD_IN, false, "WRITE(10) dressed as a read" },+ { UPIU_COMMAND, 0, 0, 0x2a, 0, 0, 0, UTRD_DD_NONE, false, "WRITE(10) with no data" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x42, 0, 0, 24, UTRD_DD_OUT, false, "UNMAP" },+ { UPIU_COMMAND, 0, 0, 0x42, 0, 0, 0, UTRD_DD_NONE, false, "UNMAP with no data" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_OUT, false, "READ(10) with data host to device" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_MASK, false, "READ(10) with direction 3" },+ { UPIU_COMMAND, UPIU_FLAG_READ | UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with the W flag" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with only the W flag" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 8192, UTRD_DD_IN, false, "READ(10) of a length other than expected" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xc4, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the RPMB unit" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xb0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the BOOT unit" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xd0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the device unit" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0x81, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of REPORT LUNS" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 3, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of LUN 3 (not allowed now)" },{ UPIU_COMMAND, 0, 0, 0x04, 0, 0, 0, UTRD_DD_NONE, false, "FORMAT UNIT" },{ UPIU_COMMAND, 0, 0, 0x1b, 0, 0, 0, UTRD_DD_NONE, false, "START STOP UNIT" },{ UPIU_COMMAND, 0, 0, 0x35, 0, 0, 0, UTRD_DD_NONE, false, "SYNCHRONIZE CACHE(10)" },- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x3b, 0, 0, 0, UTRD_DD_OUT, false, "WRITE BUFFER" },- { UPIU_COMMAND, UPIU_FLAG_READ, 0x44, 0x28, 0, 0, 0, UTRD_DD_IN, false, "READ(10) to the RPMB unit" },- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x9e, 0x11, 0, 0, UTRD_DD_IN, false, "SERVICE ACTION IN 0x11" },- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x81, 0x02, UTRD_DD_NONE, false, "QUERY WRITE DESCRIPTOR" },- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x81, 0x06, UTRD_DD_NONE, false, "QUERY SET FLAG" },- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x01, 0x04, UTRD_DD_NONE, false, "QUERY WRITE ATTRIBUTE as a read" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x3b, 0, 0, 4096, UTRD_DD_OUT, false, "WRITE BUFFER" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0xb5, 0, 0, 512, UTRD_DD_OUT, false, "SECURITY PROTOCOL OUT" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0xa2, 0, 0, 512, UTRD_DD_IN, false, "SECURITY PROTOCOL IN" },+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x15, 0, 0, 24, UTRD_DD_OUT, false, "MODE SELECT" },+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x9e, 0x11, 0, 32, UTRD_DD_IN, false, "SERVICE ACTION IN 0x11" },+ { UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_IN, false, "TEST UNIT READY with data in" },+ { 0x16, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "a QUERY REQUEST" },{ 0x04, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "TASK MANAGEMENT" },{ 0x00, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "NOP OUT" },};- uint64_t refused = hc_refused;bool right = true;for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {@@ -308,53 +414,77 @@ hc_whitelist_selftest(void)upiu[0] = cases[i].type;upiu[1] = cases[i].flags;- upiu[2] = cases[i].lun == 0x44 ? 0xc4 : cases[i].lun;- upiu[5] = cases[i].query_function;- upiu[12] = cases[i].query_opcode;- upiu[16] = cases[i].cdb0;+ upiu[2] = cases[i].lun;+ upiu[12] = (uint8_t)(cases[i].expected >> 24);+ upiu[13] = (uint8_t)(cases[i].expected >> 16);+ upiu[14] = (uint8_t)(cases[i].expected >> 8);+ upiu[15] = (uint8_t)cases[i].expected;+ upiu[16] = cases[i].op;upiu[17] = cases[i].cdb1;- passed = hc_refusal(upiu, UTRD_CT_UFS | cases[i].dd) == NULL;+ if (cases[i].op == 0x88) {+ upiu[16 + 12] = (uint8_t)(cases[i].blocks >> 8);+ upiu[16 + 13] = (uint8_t)cases[i].blocks;+ } else {+ upiu[16 + 7] = (uint8_t)(cases[i].blocks >> 8);+ upiu[16 + 8] = (uint8_t)cases[i].blocks;+ }+ passed = hc_refusal(upiu, UTRD_CT_UFS | cases[i].dd, 1u << 0, false) == NULL;if (passed != cases[i].pass) {- printf("ufs: native: the whitelist %s %s\n", passed ? "LETS THROUGH" : "refuses", cases[i].what);+ hc_say("ufs: native: the whitelist %s %s\n", passed ? "LETS THROUGH" : "refuses", cases[i].what);right = false;}}- hc_refused = refused;- printf("ufs: native: whitelist tried on %u commands made up and never sent (READ(10/16), READ CAPACITY, "- "QUERY READ DESCRIPTOR pass; WRITE(10/16), UNMAP, FORMAT UNIT, START STOP UNIT, SYNCHRONIZE CACHE, "- "WRITE BUFFER, RPMB, query writes and set flag, task management, NOP OUT refused): %s\n",- (unsigned)(sizeof(cases) / sizeof(cases[0])), right ? "as it should" : "WRONG");+ /* and what this file makes */+ {+ struct hc_cmd c;++ hc_make_read(&c, 0, 1, 12345, 64);+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;+ hc_make_read(&c, 2, 1, 12345, 64);+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) != NULL;+ hc_make_tur(&c, 0, 0);+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;+ }+ hc_say("ufs: native: whitelist tried on %u commands made up and never sent -- READ(10/16), TEST UNIT READY, READ "+ "CAPACITY of LUN 0 pass; WRITE(10/16), UNMAP, a READ with data host to device, a READ with the W flag, the "+ "RPMB/BOOT/device/REPORT LUNS units, LUN 3, FORMAT, START STOP, SYNCHRONIZE CACHE, WRITE BUFFER, SECURITY "+ "PROTOCOL, MODE SELECT, query, task management, NOP refused: %s\n",+ (unsigned)(sizeof(cases) / sizeof(cases[0])) + 3, right ? "as it should" : "WRONG");return right;}-/* ---- one request in one slot ---- */+/* ---- one command in one slot ---- */+static void+hc_put(volatile uint32_t *to, const void *from, uint32_t bytes)+{+ for (uint32_t i = 0; i < bytes / 4; i++) {+ uint32_t w;++ memcpy(&w, (const uint8_t *)from + 4 * i, 4);+ to[i] = w;+ }+}++/*+ * Into the slot: the command UPIU, an empty response, the PRD table for bytes of the slot's+ * buffer, the descriptor -- only a command the whitelist passes, and only if what the slot then+ * holds is the same and passes again.+ */static int-hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)+hc_place(struct hc_slot *s, const struct hc_cmd *c, uint32_t bytes){- uint32_t bytes = blocks * BLOCK, n = 0, dw0 = UTRD_CT_UFS | UTRD_DD_IN;- uint8_t *cmd = s->ucd;-- /* the command UPIU and an empty response */- bzero(s->ucd, UCD_PRDT);- cmd[0] = UPIU_COMMAND;- cmd[1] = UPIU_FLAG_READ; /* simple task attribute */- cmd[2] = lun;- cmd[3] = (uint8_t)s->number; /* task tag */- cmd[12] = (uint8_t)(bytes >> 24);- cmd[13] = (uint8_t)(bytes >> 16);- cmd[14] = (uint8_t)(bytes >> 8);- cmd[15] = (uint8_t)bytes;- cmd[16] = 0x28; /* READ(10) */- cmd[18] = (uint8_t)(lba >> 24);- cmd[19] = (uint8_t)(lba >> 16);- cmd[20] = (uint8_t)(lba >> 8);- cmd[21] = (uint8_t)lba;- cmd[23] = (uint8_t)(blocks >> 8);- cmd[24] = (uint8_t)blocks;- /* the data's place, in pieces of at most 256 KB */+ uint32_t n = 0, back[8];++ if (!ufshc_permitted(c->upiu, c->dw0)) {+ return EPERM;+ }+ hc_put(s->ucd, c->upiu, sizeof(c->upiu));+ for (uint32_t i = sizeof(c->upiu) / 4; i < UCD_PRDT / 4; i++) {+ s->ucd[i] = 0;+ }for (uint32_t at = 0; at < bytes; at += PRD_MAX, n++) {- volatile uint32_t *prd = (volatile uint32_t *)(s->ucd + UCD_PRDT + 16 * n);+ volatile uint32_t *prd = s->ucd + (UCD_PRDT + 16 * n) / 4;uint64_t pa = s->data_pa + at;uint32_t len = bytes - at < PRD_MAX ? bytes - at : PRD_MAX;@@ -363,11 +493,7 @@ hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)prd[2] = 0;prd[3] = len - 1;}- FlushPoC_DcacheRegion((vm_offset_t)s->ucd, UCD_PRDT + 16 * n);- /* nothing of the buffer is left in a cache to be written back over what the controller writes */- FlushPoC_DcacheRegion((vm_offset_t)s->data, bytes);-- s->utrd[0] = dw0; /* no interrupt, no inline encryption */+ s->utrd[0] = c->dw0;s->utrd[1] = 0;s->utrd[2] = OCS_INVALID;s->utrd[3] = 0;@@ -375,31 +501,56 @@ hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)s->utrd[5] = (uint32_t)(s->ucd_pa >> 32);s->utrd[6] = ((UCD_RESPONSE / 4) << 16) | (UCD_RESPONSE / 4); /* response: offset, length in dwords */s->utrd[7] = ((UCD_PRDT / 4) << 16) | n; /* PRD table: offset in dwords, entries */- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);-- /* what the controller will read is what the whitelist sees */- if (!ufshc_permitted(s->ucd, s->utrd[0])) {- return EPERM;- }- if (hc_closed || hc_dead) {- return ENXIO;+ __asm__ volatile ("dsb sy" ::: "memory");+ for (int i = 0; i < 8; i++) {+ back[i] = s->ucd[i];}- if (hc_read(UFSHC_UTRLDBR) & (1u << s->number)) {- printf("ufs: native: slot %u is busy before it is used: this path is off from now on\n", s->number);- hc_off();- return EIO;+ if (memcmp(back, c->upiu, sizeof(back)) != 0 || s->utrd[0] != c->dw0 ||+ !ufshc_permitted((const uint8_t *)back, s->utrd[0])) {+ hc_say("ufs: native: slot %u does not hold what was put in it: not rung\n", s->number);+ hc_refused++;+ return EPERM;}- __asm__ volatile ("dsb sy" ::: "memory");- *(volatile uint32_t *)(hc + UFSHC_UTRLDBR) = 1u << s->number;return 0;}-/* Until the controller is done with the slot's request; then whether it went well. */+/* The doorbell: only while this kernel's list is the one in, running, and the loader not taking it back */static int-hc_complete(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)+hc_ring(struct hc_slot *s)+{+ uint32_t bit = 1u << s->number;+ boolean_t istate;+ bool ok;++ lck_mtx_lock(&hc_lock);+ istate = ml_set_interrupts_enabled(FALSE);+ ok = hc_in && !hc_closed && !hc_dead && *hc_loader.take_back == 0 &&+ hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi &&+ (hc_read(UFSHC_UTRLRSR) & 1) != 0 && (hc_read(UFSHC_UTRLDBR) & bit) == 0;+ if (ok) {+ hc_write(UFSHC_UTRLDBR, bit);+ }+ ml_set_interrupts_enabled(istate);+ lck_mtx_unlock(&hc_lock);+ return ok ? 0 : ENXIO;+}++/* Nothing more is sent from here on; whoever waits for a slot is told. */+static void+hc_off(void)+{+ hc_dead = true;+ lck_mtx_lock(&hc_lock);+ wakeup(&hc_free);+ lck_mtx_unlock(&hc_lock);+}++/* Until the controller is done with the slot's command; then whether it went well. */+static int+hc_complete(struct hc_slot *s, const char *what, bool tur){uint64_t started = mach_absolute_time(), us;- const uint8_t *r = s->ucd + UCD_RESPONSE;+ volatile const uint8_t *r = (volatile const uint8_t *)s->ucd + UCD_RESPONSE;uint32_t ocs;while (hc_read(UFSHC_UTRLDBR) & (1u << s->number)) {@@ -409,44 +560,43 @@ hc_complete(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)continue;}if (us > TIMEOUT_MS * 1000ULL) {- printf("ufs: native: slot %u (LUN %u, blocks %u..%u) not done in %u s (doorbell %08x, IS %08x, HCS %08x): "- "this path is off from now on\n", s->number, lun, lba, lba + blocks - 1, TIMEOUT_MS / 1000,- hc_read(UFSHC_UTRLDBR), hc_read(UFSHC_IS), hc_read(UFSHC_HCS));+ hc_say("ufs: native: slot %u (%s) not done in %u s (doorbell %08x, IS %08x, HCS %08x): "+ "this path is off from now on\n", s->number, what, TIMEOUT_MS / 1000, hc_read(UFSHC_UTRLDBR),+ hc_read(UFSHC_IS), hc_read(UFSHC_HCS));hc_off();return ETIMEDOUT;}IOSleep(1);}__asm__ volatile ("dsb sy" ::: "memory");- hc_wait_us += hc_us_since(started);- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);- FlushPoC_DcacheRegion((vm_offset_t)r, 64);+ if (!tur) {+ hc_wait_us += hc_us_since(started);+ }ocs = s->utrd[2] & 0xff;- if (ocs != OCS_SUCCESS || (r[0] & 0x3f) != UPIU_RESPONSE || r[3] != s->number || r[6] != 0 || r[7] != 0) {- /* sense data follows the 32 bytes of the header: length (2), then the fixed format */- FlushPoC_DcacheRegion((vm_offset_t)r + 64, 64);- printf("ufs: native: slot %u, LUN %u, blocks %u..%u: OCS %x, response %02x tag %02x, response %02x status %02x, "- "sense key %x asc %02x ascq %02x\n", s->number, lun, lba, lba + blocks - 1, ocs, r[0], r[3], r[6], r[7],+ if (ocs != OCS_SUCCESS || (r[0] & 0x3f) != UPIU_RESPONSE || r[3] != s->number || r[6] != 0 ||+ (r[7] != 0 && !(tur && r[7] == 0x02))) {+ /* sense data follows the 32 bytes of the header: its length (2), then the fixed format */+ hc_say("ufs: native: slot %u (%s): OCS %x, response %02x tag %02x, response %02x status %02x, "+ "sense key %x asc %02x ascq %02x\n", s->number, what, ocs, r[0], r[3], r[6], r[7],r[34 + 2] & 0x0f, r[34 + 12], r[34 + 13]);- hc_errors++;+ if (!tur) {+ hc_errors++;+ }return EIO;}- FlushPoC_DcacheRegion((vm_offset_t)s->data, blocks * BLOCK);- hc_requests++;- hc_bytes += blocks * BLOCK;return 0;}-static struct ufshc_slot *+static struct hc_slot *hc_take(bool wait){- struct ufshc_slot *s = NULL;+ struct hc_slot *s = NULL;lck_mtx_lock(&hc_lock);- while (hc_free == 0 && wait && !hc_dead) {+ while (hc_free == 0 && wait && !hc_dead && hc_in) {msleep(&hc_free, &hc_lock, PRIBIO, "ufshc", NULL);}- if (hc_free != 0) {+ if (hc_free != 0 && hc_in && !hc_dead) {uint32_t i = (uint32_t)__builtin_ctz(hc_free);hc_free &= ~(1u << i);@@ -460,7 +610,7 @@ hc_take(bool wait)}static void-hc_give(struct ufshc_slot *s)+hc_give(struct hc_slot *s){lck_mtx_lock(&hc_lock);hc_free |= 1u << (s - hc_slots);@@ -478,27 +628,28 @@ intufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len){struct {- struct ufshc_slot *slot;+ struct hc_slot *slot;uint32_t lba, blocks, skip, take;uint8_t *to;- } q[SLOT_MAX];+ } q[SLOTS];uint32_t head = 0, tail = 0, per = hc_slot_buffer / BLOCK;int error = 0;- if (!hc_usable || hc_dead || hc_closed) {+ if (!hc_usable || !hc_in || hc_dead || hc_closed) {return ENXIO;}while ((len > 0 && error == 0) || head != tail) {- /* as many more requests as there are free slots (and one, waiting for it, if none is in flight) */- while (len > 0 && error == 0 && tail - head < SLOT_MAX) {+ /* as many more reads as there are free slots (and one, waiting for it, if none is in flight) */+ while (len > 0 && error == 0 && tail - head < SLOTS) {uint64_t first = offset / BLOCK, skip = offset % BLOCK;uint64_t blocks = (skip + len + BLOCK - 1) / BLOCK, take;- struct ufshc_slot *s = hc_take(head == tail);+ struct hc_slot *s = hc_take(head == tail);+ struct hc_cmd c;int e;if (s == NULL) {if (head == tail) {- error = EIO; /* the path went off while waiting */+ error = ENXIO; /* the path went off while waiting */}break;}@@ -514,18 +665,24 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)if (take > len) {take = len;}- e = hc_submit(s, lun, (uint32_t)first, (uint32_t)blocks);+ hc_make_read(&c, lun, (uint8_t)s->number, (uint32_t)first, (uint32_t)blocks);+ /* nothing of the buffer is left in a cache to be written back over what the controller writes */+ FlushPoC_DcacheRegion((vm_offset_t)s->data, (size_t)blocks * BLOCK);+ e = hc_place(s, &c, (uint32_t)blocks * BLOCK);+ if (e == 0) {+ e = hc_ring(s);+ }if (e != 0) {hc_give(s);error = e;break;}- q[tail % SLOT_MAX].slot = s;- q[tail % SLOT_MAX].lba = (uint32_t)first;- q[tail % SLOT_MAX].blocks = (uint32_t)blocks;- q[tail % SLOT_MAX].skip = (uint32_t)skip;- q[tail % SLOT_MAX].take = (uint32_t)take;- q[tail % SLOT_MAX].to = dst;+ q[tail % SLOTS].slot = s;+ q[tail % SLOTS].lba = (uint32_t)first;+ q[tail % SLOTS].blocks = (uint32_t)blocks;+ q[tail % SLOTS].skip = (uint32_t)skip;+ q[tail % SLOTS].take = (uint32_t)take;+ q[tail % SLOTS].to = dst;tail++;offset += take;dst += take;@@ -533,11 +690,19 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)}/* the oldest one done, copied out, its slot free again */if (head != tail) {- typeof(q[0]) *e = &q[head % SLOT_MAX];- int r = hc_complete(e->slot, lun, e->lba, e->blocks);+ typeof(q[0]) *e = &q[head % SLOTS];+ char what[48];+ int r;- if (r == 0 && error == 0) {- memcpy(e->to, e->slot->data + e->skip, e->take);+ snprintf(what, sizeof(what), "LUN %u, blocks %u..%u", lun, e->lba, e->lba + e->blocks - 1);+ r = hc_complete(e->slot, what, false);+ if (r == 0) {+ FlushPoC_DcacheRegion((vm_offset_t)e->slot->data, (size_t)e->blocks * BLOCK);+ hc_requests++;+ hc_bytes += (uint64_t)e->blocks * BLOCK;+ if (error == 0) {+ memcpy(e->to, e->slot->data + e->skip, e->take);+ }} else if (error == 0) {error = r;}@@ -550,6 +715,253 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)return error;}+/* ---- the proof, and the switches ---- */++/* UEFI's slot 0 holds a read (what the controller would run were it to take no notice of this list) */+static const char *+hc_uefi_slot0(uint8_t *upiu, uint32_t *dw0)+{+ uint64_t ucd = (uint64_t)hc_uefi_list[5] << 32 | hc_uefi_list[4];+ volatile const uint8_t *u;++ *dw0 = hc_uefi_list[0];+ if (ucd < hc_uefi_map_pa || ucd + 32 > hc_uefi_map_pa + hc_uefi_map_size) {+ return "UEFI's command descriptor is not where it was";+ }+ u = (volatile const uint8_t *)(hc_uefi_map_va + (ucd - hc_uefi_map_pa));+ for (int i = 0; i < 32; i++) {+ upiu[i] = u[i];+ }+ return hc_refusal(upiu, *dw0, 0xff, true) == NULL ? NULL : "UEFI's slot 0 does not hold a read";+}++/* This kernel's slot 0: TEST UNIT READY rung, and its status must come back in this list */+static int+hc_prove(void)+{+ struct hc_slot *s = &hc_slots[SLOT_PROOF];+ struct hc_cmd c;+ uint32_t ocs;+ int e;++ hc_make_tur(&c, (uint8_t)__builtin_ctz(hc_lun_mask), SLOT_PROOF);+ e = hc_place(s, &c, 0);+ if (e == 0) {+ e = hc_ring(s);+ }+ if (e == 0) {+ e = hc_complete(s, "the proof's TEST UNIT READY", true);+ }+ ocs = s->utrd[2] & 0xff;+ hc_proofs++;+ if (e != 0 || ocs != OCS_SUCCESS) {+ hc_say("ufs: native: proof FAILED (%d): this list's slot 0 has OCS %x after its doorbell -- the controller "+ "did not run it from this list\n", e, ocs);+ return e != 0 ? e : EIO;+ }+ hc_say("ufs: native: proof: TEST UNIT READY in this list's slot 0 came back here (OCS 0, response %02x, status %02x)\n",+ ((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 6], ((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 7]);+ return 0;+}++struct hc_put_back_result {+ uint32_t db_before, db_after_clear, lo, hi, rsr, db, is, cnr;+ bool cleared, ok;+};++/*+ * UEFI's list back into the controller: what is in flight waited for (1 s), cleared if it does+ * not finish (every slot of this list is this kernel's), the list stopped, UEFI's address+ * written, the list started, read back. Interrupts off, no locks, no printing (the quiesce).+ */+static void+hc_put_back(struct hc_put_back_result *r)+{+ uint64_t t;++ bzero(r, sizeof(*r));+ for (t = mach_absolute_time(); (r->db_before = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {+ __builtin_arm_yield();+ }+ if (r->db_before != 0 && hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi) {+ r->cleared = true;+ hc_write(UFSHC_UTRLCLR, 0);+ for (t = mach_absolute_time(); (r->db_after_clear = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {+ __builtin_arm_yield();+ }+ }+ hc_write(UFSHC_UTRLRSR, 0);+ for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {+ __builtin_arm_yield();+ }+ hc_write(UFSHC_UTRLBA, hc_loader.uefi_lo);+ hc_write(UFSHC_UTRLBAU, hc_loader.uefi_hi);+ hc_write(UFSHC_UTRLRSR, 1);+ r->lo = hc_read(UFSHC_UTRLBA);+ r->hi = hc_read(UFSHC_UTRLBAU);+ r->rsr = hc_read(UFSHC_UTRLRSR);+ r->db = hc_read(UFSHC_UTRLDBR);+ r->is = hc_read(UFSHC_IS);+ r->cnr = hc_read(UFSHC_UTRLCNR);+ r->ok = r->lo == hc_loader.uefi_lo && r->hi == hc_loader.uefi_hi && (r->rsr & 1);+ if (r->ok) {+ *hc_loader.list_in = 0;+ (*hc_loader.switched_out)++;+ hc_switch_out_count++;+ }+}++static void+hc_put_back_print(const char *why, const struct hc_put_back_result *r, bool late)+{+ void (*say)(const char *, ...) = late ? hc_say_late : hc_say;++ if (r->cleared) {+ say("ufs: native: %s: doorbell %08x did not clear in 1 s: this kernel's slots cleared (UTRLCLR), doorbell "+ "then %08x\n", why, r->db_before, r->db_after_clear);+ }+ say("ufs: native: %s: transfer list %s UEFI's %08x:%08x (run %u, doorbell %08x; IS %08x, at the switch %08x; "+ "completion %08x, at the switch %08x)\n", why, r->ok ? "back to" : "NOT BACK TO", r->hi, r->lo, r->rsr, r->db,+ r->is, hc_is_in, r->cnr, hc_cnr_in);+}++/*+ * This kernel's list into the controller (the caller holds the loader's mailbox: nothing is asked+ * of UEFI meanwhile). prove: then slot 0's TEST UNIT READY, before anything is read.+ */+int+ufshc_switch_in(bool prove)+{+ uint32_t lo, hi, db, rsr, hcs, uefi_dw0 = 0;+ uint8_t uefi_upiu[32] = { 0 };+ const char *no = NULL;+ struct hc_put_back_result undo;+ boolean_t istate;+ bool undone = false;++ if (!hc_usable || hc_dead || hc_closed) {+ return ENXIO;+ }+ lck_mtx_lock(&hc_lock);+ if (hc_in) {+ lck_mtx_unlock(&hc_lock);+ return 0;+ }+ istate = ml_set_interrupts_enabled(FALSE);+ if (*hc_loader.take_back != 0) {+ no = "the loader has taken the controller back";+ } else {+ no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);+ }+ lo = hc_read(UFSHC_UTRLBA);+ hi = hc_read(UFSHC_UTRLBAU);+ db = hc_read(UFSHC_UTRLDBR);+ rsr = hc_read(UFSHC_UTRLRSR);+ hcs = hc_read(UFSHC_HCS);+ if (no == NULL) {+ if (lo != hc_loader.uefi_lo || hi != hc_loader.uefi_hi) {+ no = "the controller's list is not UEFI's";+ } else if (db != 0) {+ no = "a request in flight";+ } else if ((rsr & 1) == 0 || (hcs & HCS_UTRLRDY) == 0) {+ no = "the list is not running, or not ready";+ }+ }+ if (no == NULL) {+ uint64_t t;++ hc_is_in = hc_read(UFSHC_IS);+ hc_cnr_in = hc_read(UFSHC_UTRLCNR);+ hc_write(UFSHC_UTRLRSR, 0);+ for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {+ __builtin_arm_yield();+ }+ if (hc_read(UFSHC_UTRLRSR) & 1) {+ no = "the list did not stop";+ hc_write(UFSHC_UTRLRSR, 1); /* (UEFI's list, running, as it was) */+ } else {+ hc_write(UFSHC_UTRLBA, hc_list_lo);+ hc_write(UFSHC_UTRLBAU, hc_list_hi);+ hc_write(UFSHC_UTRLRSR, 1);+ lo = hc_read(UFSHC_UTRLBA);+ hi = hc_read(UFSHC_UTRLBAU);+ rsr = hc_read(UFSHC_UTRLRSR);+ if (lo != hc_list_lo || hi != hc_list_hi || (rsr & 1) == 0) {+ no = "the controller did not take this list";+ hc_put_back(&undo);+ undone = true;+ } else {+ hc_in = true;+ *hc_loader.list_in = 1;+ (*hc_loader.switched_in)++;+ hc_switch_in_count++;+ }+ }+ }+ ml_set_interrupts_enabled(istate);+ lck_mtx_unlock(&hc_lock);+ if (undone) {+ hc_put_back_print("undoing", &undo, false);+ }+ if (no != NULL) {+ hc_say("ufs: native: this kernel's list not put in: %s (list %08x:%08x, doorbell %08x, run %08x, HCS %08x; UEFI's "+ "slot 0 %08x, CDB %02x)\n", no, hi, lo, db, rsr, hcs, uefi_dw0, uefi_upiu[16]);+ return EIO;+ }+ hc_say("ufs: native: transfer list switched from UEFI's %08x:%08x to this kernel's %08x:%08x (doorbell 0, run %u; "+ "IS %08x, completion %08x; UEFI's slot 0 holds a read: %08x, CDB %02x)\n", hc_loader.uefi_hi,+ hc_loader.uefi_lo, hi, lo, rsr, hc_is_in, hc_cnr_in, uefi_dw0, uefi_upiu[16]);+ if (prove) {+ int e = hc_prove();++ if (e != 0) {+ ufshc_switch_out("after the failed proof");+ hc_off();+ return e;+ }+ }+ return 0;+}++/* UEFI's list back (the caller holds the loader's mailbox) */+int+ufshc_switch_out(const char *why)+{+ struct hc_put_back_result r;+ uint64_t started;+ boolean_t istate;++ lck_mtx_lock(&hc_lock);+ if (!hc_in) {+ lck_mtx_unlock(&hc_lock);+ return 0;+ }+ hc_in = false; /* nothing more is rung */+ wakeup(&hc_free);+ lck_mtx_unlock(&hc_lock);+ /* what is in flight finishes (whoever waits for it sees it done) */+ for (started = mach_absolute_time(); hc_read(UFSHC_UTRLDBR) != 0 && hc_us_since(started) < 2000000;) {+ IOSleep(1);+ }+ lck_mtx_lock(&hc_lock);+ istate = ml_set_interrupts_enabled(FALSE);+ hc_put_back(&r);+ ml_set_interrupts_enabled(istate);+ lck_mtx_unlock(&hc_lock);+ hc_put_back_print(why, &r, false);+ if (!r.ok) {+ hc_off();+ return EIO;+ }+ return 0;+}++bool+ufshc_list_in(void)+{+ return hc_in;+}+boolufshc_available(void){@@ -557,49 +969,70 @@ ufshc_available(void)}void-ufshc_report(const char *when)+ufshc_set_luns(uint32_t mask){- if (!hc_usable) {- return;- }- printf("ufs: native: %s: %llu requests, %llu errors, %llu bytes, %llu ms waiting (summed over requests), "- "at most %llu in flight, %llu refused%s\n", when, hc_requests, hc_errors, hc_bytes, hc_wait_us / 1000,- hc_most_in_flight, hc_refused, hc_dead ? "; THE PATH IS OFF" : "");+ hc_lun_mask = mask & 0xff;}-/* ---- the probe ---- */--static bool-hc_looks_like_descriptor(const volatile uint32_t *d)+static void+hc_report(const char *when, bool late){- bool zero = true;+ (late ? hc_say_late : hc_say)("ufs: native: %s: %llu reads, %llu errors, %llu bytes, %llu ms waiting (summed over reads), at most %llu "+ "in flight, %llu refused, list in %llu times and out %llu times, %llu proofs%s\n", when, hc_requests, hc_errors,+ hc_bytes, hc_wait_us / 1000, hc_most_in_flight, hc_refused, hc_switch_in_count, hc_switch_out_count, hc_proofs,+ hc_dead ? "; THE PATH IS OFF" : "");+}- for (int i = 0; i < 8; i++) {- zero &= d[i] == 0;+void+ufshc_report(const char *when)+{+ if (hc_usable) {+ hc_report(when, false);}- /* never used, or a UFS storage request with its command descriptor 128-byte aligned */- return zero || ((d[0] & 0xf0000000u) == UTRD_CT_UFS && (d[4] & 0x7f) == 0 && (d[0] & UTRD_DD_MASK) != UTRD_DD_MASK);}+/* ---- the probe ---- */+/*- * What the controller looks like as UEFI left it, and whether this kernel may put requests of- * its own into it (only == true: say so and do not touch anything but registers' reads).+ * What the controller looks like as UEFI left it, and whether this kernel may put a list of its+ * own in (only == true: say so and do not use it).*/bool-ufshc_probe(bool only)+ufshc_probe(const struct ufshc_loader *loader, bool only){uint32_t cap, ver, hcs, hce, is, ie, iacr, ahit, base, baseu, db, rsr, cnr, tm_base, tm_baseu, tm_db, tm_rsr;- uint64_t ucd0 = 0, list_end;+ uint32_t uefi_dw0 = 0, slots = 31;+ uint8_t uefi_upiu[32] = { 0 };+ uint64_t uefi_pa;const char *no = NULL;- vm_offset_t list_va;+ vm_offset_t area;if (!hc_whitelist_selftest()) {return false;}+ hc_loader = *loader;PE_parse_boot_argn("ufs_native_buffer", &hc_slot_buffer, sizeof(hc_slot_buffer));if (hc_slot_buffer < BLOCK || hc_slot_buffer > (4u << 20) || hc_slot_buffer % BLOCK) {hc_slot_buffer = SLOT_BUFFER_DEFAULT;}+ PE_parse_boot_argn("ufs_native_slots", &slots, sizeof(slots));+ if (slots < 1 || slots > SLOTS - 1) {+ slots = SLOTS - 1;+ }+ PE_parse_boot_argn("ufs_native_leave", &hc_leave, sizeof(hc_leave));+ if (loader->hc_pa != UFSHC_PHYS || loader->area_pa == 0 || loader->slots != SLOTS || (loader->area_pa & 0x3ff) ||+ loader->ucd_size < UCD_PRDT + 16 * (hc_slot_buffer / PRD_MAX + 1) || (loader->ucd_offset & 0x7f) ||+ (loader->ucd_size & 0x7f) || loader->ucd_offset < SLOTS * 32 ||+ loader->area_size < loader->ucd_offset + SLOTS * loader->ucd_size) {+ hc_say("ufs: native: not used: the loader offers no area for a list (or not one as expected: controller %llx, "+ "area %llx of %u, command descriptors at %u of %u)\n", loader->hc_pa, loader->area_pa, loader->area_size,+ loader->ucd_offset, loader->ucd_size);+ return false;+ }+ if (pmap_valid_address(loader->area_pa) || pmap_valid_address(loader->area_pa + loader->area_size - 1)) {+ hc_say("ufs: native: not used: the loader's area %llx is memory this kernel manages\n", loader->area_pa);+ return false;+ }hc = ml_io_map(UFSHC_PHYS, UFSHC_SIZE);cap = hc_read(UFSHC_CAP);ver = hc_read(UFSHC_VER);@@ -618,16 +1051,14 @@ ufshc_probe(bool only)tm_baseu = hc_read(UFSHC_UTMRLBAU);tm_db = hc_read(UFSHC_UTMRLDBR);tm_rsr = hc_read(UFSHC_UTMRLRSR);- hc_is_at_probe = is;- printf("ufs: native: controller at %llx: CAP %08x (%u slots, 64-bit %s), VER %08x, HCDDID %08x, HCPMID %08x\n",+ hc_say("ufs: native: controller at %llx: CAP %08x (%u slots, 64-bit %s), VER %08x, HCDDID %08x, HCPMID %08x\n",UFSHC_PHYS, cap, CAP_NUTRS(cap), (cap & CAP_64AS) ? "yes" : "no", ver, hc_read(UFSHC_HCDDID),hc_read(UFSHC_HCPMID));- printf("ufs: native: HCE %08x, HCS %08x, IS %08x, IE %08x, UTRIACR %08x, AHIT %08x\n", hce, hcs, is, ie, iacr, ahit);- printf("ufs: native: transfer list %08x:%08x, doorbell %08x, run %08x, completion %08x; "- "task management list %08x:%08x, doorbell %08x, run %08x\n", baseu, base, db, rsr, cnr, tm_baseu, tm_base,- tm_db, tm_rsr);- hc_list_pa = (uint64_t)baseu << 32 | base;- list_end = hc_list_pa + 32ULL * CAP_NUTRS(cap);+ hc_say("ufs: native: HCE %08x, HCS %08x, IS %08x, IE %08x, UTRIACR %08x, AHIT %08x\n", hce, hcs, is, ie, iacr, ahit);+ hc_say("ufs: native: transfer list %08x:%08x (the loader's note of UEFI's: %08x:%08x), doorbell %08x, run %08x, "+ "completion %08x; task management list %08x:%08x, doorbell %08x, run %08x\n", baseu, base, loader->uefi_hi,+ loader->uefi_lo, db, rsr, cnr, tm_baseu, tm_base, tm_db, tm_rsr);+ uefi_pa = (uint64_t)baseu << 32 | base;if ((hce & 1) == 0) {no = "the controller is not enabled";@@ -637,133 +1068,120 @@ ufshc_probe(bool only)no = "the transfer list is not running";} else if (ver < 0x200 || ver >= 0x400) {no = "a controller version other than 2.x or 3.x";- } else if (CAP_NUTRS(cap) < 32) {+ } else if (CAP_NUTRS(cap) < SLOTS) {no = "fewer than 32 slots";} else if ((cap & CAP_64AS) == 0) {no = "no 64-bit addressing";} else if (is & IS_FATAL) {no = "an error status";- } else if (hc_list_pa == 0 || (hc_list_pa & 0x3ff) != 0) {- no = "no transfer list, or one not 1 KB aligned";- } else if (pmap_valid_address(hc_list_pa) || pmap_valid_address(list_end - 1)) {- no = "the transfer list is in memory this kernel manages";- } else if (db & SLOTS_RESERVED) {- no = "a slot of 16..31 is busy";+ } else if (base != loader->uefi_lo || baseu != loader->uefi_hi) {+ no = "the transfer list is not the one the loader noted as UEFI's";+ } else if (db != 0) {+ no = "a request in flight";+ } else if (pmap_valid_address(uefi_pa)) {+ no = "UEFI's transfer list is in memory this kernel manages";}- if (no != NULL) {- printf("ufs: native: not used: %s; the storage is read through the loader\n", no);- return false;- }-- /* UEFI's list: what each slot holds now (the ones that hold anything) */- list_va = ml_io_map_unmappable((vm_offset_t)(hc_list_pa & ~(vm_offset_t)PAGE_MASK),- (vm_size_t)round_page(list_end - (hc_list_pa & ~(uint64_t)PAGE_MASK)), VM_WIMG_DEFAULT);- hc_list = (volatile uint32_t *)(list_va + (hc_list_pa & PAGE_MASK));- FlushPoC_DcacheRegion((vm_offset_t)hc_list, 32 * 32);- for (uint32_t i = 0; i < 32; i++) {- const volatile uint32_t *d = hc_list + 8 * i;-- if ((d[0] | d[1] | d[2] | d[3] | d[4] | d[5] | d[6] | d[7]) != 0) {- printf("ufs: native: slot %2u: %08x %08x %08x %08x %08x %08x %08x %08x\n", i, d[0], d[1], d[2], d[3], d[4],- d[5], d[6], d[7]);- }- if (i >= SLOT_FIRST && !hc_looks_like_descriptor(d)) {- no = "a slot of 16..31 holds something that is not a request descriptor";- }- if (i < SLOT_FIRST && ucd0 == 0 && (d[4] | d[5]) != 0) {- ucd0 = (uint64_t)d[5] << 32 | d[4];- }- }- if (ucd0 != 0) {- /* UEFI's own command descriptor: the last command it sent, and where it is */- vm_offset_t va = ml_io_map_unmappable((vm_offset_t)(ucd0 & ~(uint64_t)PAGE_MASK), PAGE_SIZE * 2, VM_WIMG_DEFAULT);- const volatile uint8_t *u = (const volatile uint8_t *)(va + (ucd0 & PAGE_MASK));-- FlushPoC_DcacheRegion((vm_offset_t)u, 1024);- printf("ufs: native: UEFI's command descriptor at %llx: command UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x "- "%02x %02x %02x %02x %02x; response %02x .. %02x %02x\n", ucd0, u[0], u[1], u[2], u[3], u[16], u[17], u[18],- u[19], u[20], u[21], u[22], u[23], u[24], u[25], u[512], u[518], u[519]);- if (ucd0 < list_end && ucd0 + 1024 > hc_list_pa) {- no = "UEFI's command descriptor is inside the list";- }+ if (no == NULL) {+ /* UEFI's list and command descriptor (the page they are in), not cached here, only read */+ hc_uefi_map_pa = uefi_pa & ~(uint64_t)PAGE_MASK;+ hc_uefi_map_size = PAGE_SIZE;+ hc_uefi_map_va = ml_io_map_unmappable((vm_offset_t)hc_uefi_map_pa, PAGE_SIZE, VM_WIMG_WCOMB);+ hc_uefi_list = (volatile uint32_t *)(hc_uefi_map_va + (uefi_pa - hc_uefi_map_pa));+ hc_say("ufs: native: UEFI's slot 0: %08x %08x %08x %08x %08x %08x %08x %08x\n", hc_uefi_list[0], hc_uefi_list[1],+ hc_uefi_list[2], hc_uefi_list[3], hc_uefi_list[4], hc_uefi_list[5], hc_uefi_list[6], hc_uefi_list[7]);+ no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);+ hc_say("ufs: native: UEFI's last command: UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x "+ "%02x %02x\n", uefi_upiu[0], uefi_upiu[1], uefi_upiu[2], uefi_upiu[3], uefi_upiu[16], uefi_upiu[17],+ uefi_upiu[18], uefi_upiu[19], uefi_upiu[20], uefi_upiu[21], uefi_upiu[22], uefi_upiu[23], uefi_upiu[24],+ uefi_upiu[25]);}if (no != NULL) {- printf("ufs: native: not used: %s; the storage is read through the loader\n", no);+ hc_say("ufs: native: not used: %s; the storage is read through the loader\n", no);return false;}if (only) {- printf("ufs: native: as expected; ufs_native=2 asks for no more than this look\n");+ hc_say("ufs: native: as expected; ufs_native=2 asks for no more than this look\n");return false;}- /* this kernel's slots: the even ones of 16..31, a page of descriptor and a buffer each */- for (uint32_t i = 0; i < SLOT_MAX; i++) {- struct ufshc_slot *s = &hc_slots[i];- vm_offset_t ucd = 0, data = 0;+ /* this kernel's list and command descriptors, in the loader's area; every slot TEST UNIT READY to begin with */+ area = ml_io_map_unmappable((vm_offset_t)(loader->area_pa & ~(uint64_t)PAGE_MASK),+ (vm_size_t)round_page((loader->area_pa & PAGE_MASK) + loader->area_size), VM_WIMG_WCOMB) ++ (vm_offset_t)(loader->area_pa & PAGE_MASK);+ hc_list_lo = (uint32_t)loader->area_pa;+ hc_list_hi = (uint32_t)(loader->area_pa >> 32);+ for (uint32_t i = 0; i < SLOTS; i++) {+ struct hc_slot *s = &hc_slots[i];+ struct hc_cmd c;- if (kmem_alloc_contig(kernel_map, &ucd, UCD_SIZE, PAGE_MASK, 0, 0, KMA_KOBJECT | KMA_ZERO | KMA_DATA,- VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS ||- kmem_alloc_contig(kernel_map, &data, round_page(hc_slot_buffer), PAGE_MASK, 0, 0,- KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {- printf("ufs: native: no memory for slot buffers; the storage is read through the loader\n");- return false;+ s->number = i;+ s->utrd = (volatile uint32_t *)(area + 32 * i);+ s->ucd = (volatile uint32_t *)(area + loader->ucd_offset + i * loader->ucd_size);+ s->ucd_pa = loader->area_pa + loader->ucd_offset + i * loader->ucd_size;+ if (i >= 1 && i <= slots) {+ vm_offset_t data = 0;++ if (kmem_alloc_contig(kernel_map, &data, round_page(hc_slot_buffer), PAGE_MASK, 0, 0,+ KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {+ hc_say("ufs: native: not used: no memory for the slots' buffers\n");+ return false;+ }+ s->data = (uint8_t *)data;+ s->data_pa = kvtophys(data);+ FlushPoC_DcacheRegion(data, hc_slot_buffer);}- s->number = SLOT_FIRST + 2 * i;- s->utrd = hc_list + 8 * s->number;- s->ucd = (uint8_t *)ucd;- s->ucd_pa = kvtophys(ucd);- s->data = (uint8_t *)data;- s->data_pa = kvtophys(data);- for (int w = 0; w < 8; w++) {- s->original[w] = s->utrd[w];+ hc_make_tur(&c, 0, (uint8_t)i);+ if (hc_place(s, &c, 0) != 0) {+ hc_say("ufs: native: not used: slot %u would not take its TEST UNIT READY\n", i);+ return false;}- FlushPoC_DcacheRegion(ucd, UCD_SIZE);- FlushPoC_DcacheRegion(data, hc_slot_buffer);}- hc_nslots = SLOT_MAX;- hc_free = (1u << SLOT_MAX) - 1;+ hc_nslots = slots;+ hc_free = (uint32_t)(((1ULL << (slots + 1)) - 1) & ~1ULL); /* slots 1..n */hc_usable = true;- printf("ufs: native: in use: slots 16, 18, .. 30 of UEFI's list at %llx, %u KB a request; UEFI keeps the rest\n",- hc_list_pa, hc_slot_buffer >> 10);+ hc_say("ufs: native: usable: a list of %u slots at %llx (the loader's), reads in %u of them, %u KB each, LUN 0 only%s\n",+ SLOTS, loader->area_pa, slots, hc_slot_buffer >> 10, hc_leave ? "; ufs_native_leave=1: the list is left in "+ "at the end, for the loader to take back" : "");return true;}/** Before the kernel gives the phone back to the loader (loader_return, the other processors- * already stopped): nothing more is sent, what is in flight is waited for, and the slots'- * descriptors are put back as they were, so that UEFI finds its list as it left it.+ * already stopped): nothing more is sent, what is in flight is waited for, UEFI's list put back.+ * No locks (a stopped processor may hold one).*/voidredwood_ufs_quiesce(void){- uint64_t started = mach_absolute_time();- uint32_t db = 0, is;+ struct hc_put_back_result r;++ if (hc_kept_count != 0) {+ hc_say_late("ufs: native: what happened in this run, again:\n");+ for (uint32_t i = hc_kept_count > KEPT_LINES ? hc_kept_count - KEPT_LINES : 0; i < hc_kept_count; i++) {+ const char *line = hc_kept[i % KEPT_LINES];+ hc_say_late("> %s%s", line, line[0] != 0 && line[strlen(line) - 1] == '\n' ? "" : "...\n");+ }+ }if (!hc_usable) {return;}hc_closed = true;__asm__ volatile ("dsb sy" ::: "memory");- while ((db = hc_read(UFSHC_UTRLDBR)) & SLOTS_RESERVED) {- if (hc_us_since(started) > 2000000) {- break;- }- }- is = hc_read(UFSHC_IS);- if (db & SLOTS_RESERVED) {- printf("ufs: native: at the end, slots still busy after 2 s (doorbell %08x, IS %08x): the descriptors "- "are left as they are\n", db, is);+ if (!hc_in && hc_read(UFSHC_UTRLBA) == hc_loader.uefi_lo && hc_read(UFSHC_UTRLBAU) == hc_loader.uefi_hi &&+ (hc_read(UFSHC_UTRLRSR) & 1)) {+ hc_say_late("ufs: native: at the end: UEFI's list is in (doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));+ hc_report("at the end", true);return;}- for (uint32_t i = 0; i < hc_nslots; i++) {- struct ufshc_slot *s = &hc_slots[i];-- for (int w = 0; w < 8; w++) {- s->utrd[w] = s->original[w];- }- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);+ hc_in = false;+ if (hc_leave) {+ hc_say_late("ufs: native: at the end: this kernel's list LEFT IN for the loader to take back "+ "(ufs_native_leave=1; doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));+ hc_report("at the end", true);+ return;}- printf("ufs: native: at the end: none of slots 16..31 busy (doorbell %08x), their descriptors as they were; "- "IS %08x (at the probe %08x); %llu requests, %llu errors, %llu refused\n", db, is, hc_is_at_probe,- hc_requests, hc_errors, hc_refused);+ hc_put_back(&r);+ hc_put_back_print("at the end", &r, true);+ hc_report("at the end", true);}diff --git a/xnu/files/bsd/dev/redwood_ufshc.h b/xnu/files/bsd/dev/redwood_ufshc.hnew file mode 100644index 0000000..6703ac9--- /dev/null+++ b/xnu/files/bsd/dev/redwood_ufshc.h@@ -0,0 +1,31 @@+/*+ * The kernel's own UFS driver (redwood_ufshc.c), as redwood_ufs.c uses it.+ */+#ifndef _REDWOOD_UFSHC_H_+#define _REDWOOD_UFSHC_H_++#include <stdbool.h>+#include <stdint.h>++/* what the loader's page says about it (struct loader_block's native sections) */+struct ufshc_loader {+ uint64_t hc_pa; /* the controller's registers; 0: not offered */+ uint64_t area_pa; /* the loader's memory for this kernel's list and command descriptors */+ uint32_t area_size, ucd_offset, ucd_size, slots;+ uint32_t uefi_lo, uefi_hi; /* UEFI's list */+ volatile uint32_t *take_back; /* the loader's: it takes the controller back, send nothing more */+ volatile uint32_t *list_in; /* this kernel's: its list is in */+ uint32_t *switched_in, *switched_out;+};++bool ufshc_probe(const struct ufshc_loader *loader, bool only);+bool ufshc_available(void);+void ufshc_set_luns(uint32_t mask);+int ufshc_switch_in(bool prove);+int ufshc_switch_out(const char *why);+bool ufshc_list_in(void);+int ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len);+void ufshc_report(const char *when);+void redwood_ufs_quiesce(void);++#endifdiff --git a/xnuloader/loader.c b/xnuloader/loader.cindex ff02f56..22602a3 100644--- a/xnuloader/loader.c+++ b/xnuloader/loader.c@@ -322,6 +322,11 @@ compose_logs(const char *klog, uint64_t klog_len)staged_len = n;}+static int hc_list_is_uefis(void);+static int hc_log_skips; /* logs not written this run (said twice at most) */+static void hc_print_state(const char *when);+static void hc_take_back(const char *when);+static voidwrite_logs(const char *klog, uint64_t klog_len){@@ -341,9 +346,20 @@ write_logs(const char *klog, uint64_t klog_len)}uint64_t bs = bio->media->block_size;uint64_t len = (staged_len + 1 + bs - 1) / bs * bs;- efi_status ws = bio->write_blocks(bio, bio->media->media_id, 0, len, out_buf), rs = 0;+ efi_status ws, rs = 0;int same = 0;+ /* only through UEFI's own transfer list (see hc_take_back) */+ if (!hc_list_is_uefis()) {+ if (hc_log_skips++ < 2) {+ hc_print_state("log not written");+ print("log: NOT WRITTEN to logdump: UEFI's transfer list is not the one in the controller\n");+ }+ compose_logs(klog, klog_len);+ return;+ }+ ws = bio->write_blocks(bio, bio->media->media_id, 0, len, out_buf);+if (!EFI_ERROR(ws)) {bio->flush_blocks(bio);/*@@ -540,6 +556,7 @@ static const struct platform platform_redwood = {static voidreboot(void){+ hc_take_back("before the reset");flush_log();ST->rt->reset_system(0 /* EfiResetCold */, 0, 0, NULL);for (;;) {@@ -1441,7 +1458,7 @@ ufs_survey(void)* back over the data.*/#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */-#define BLOCK_VERSION 2+#define BLOCK_VERSION 3#define BLOCK_MAX_PARTS 160#define BLOCK_MAX_UNITS 16#define BLOCK_PART_UNIT 1u /* the entry is a whole logical unit, not a partition */@@ -1483,6 +1500,26 @@ struct loader_block {uint64_t served, failed, bytes, ticks;} answer __attribute__((aligned(128)));struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));+ /*+ * Written by the loader before the kernel starts, with ufs_native= on the command line: where+ * the kernel may put a transfer request list of its own into the UFS host controller for the+ * run (hc_area, below), and UEFI's, which the loader puts back. hc_pa 0: it may not.+ */+ struct {+ uint64_t hc_pa; /* the controller's registers */+ uint64_t area_pa; /* the list (1 KB, at the start) and command descriptors */+ uint32_t area_size, ucd_offset, ucd_size, slots;+ uint32_t uefi_utrlba, uefi_utrlbau;+ } native __attribute__((aligned(128)));+ /* written by the loader during a run: it takes the controller back (a hung kernel); send nothing more */+ struct {+ volatile uint32_t take_back;+ } loader_native __attribute__((aligned(128)));+ /* written by the kernel: its list is in the controller (for the loader's log) */+ struct {+ volatile uint32_t list_in;+ uint32_t switched_in, switched_out, reserved;+ } kernel_native __attribute__((aligned(128)));};/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */@@ -1539,6 +1576,174 @@ device_path_print(uint64_t unit, const efi_device_path *dp)return lun;}+/*+ * The UFS host controller, for the kernel's own driver (ufs_native= on the command line; without it+ * this loader never touches the controller's registers). The kernel may, for the length of a run,+ * stop the controller's transfer request list, point it at a list of its own in hc_area (memory of+ * this loader, so that putting UEFI's back never depends on the kernel's memory), and start it+ * again; UEFI's list is one slot and its command descriptor, with no room for another request.+ * This loader keeps UEFI's list address (UTRLBA/UTRLBAU) from before the first run and checks it,+ * and puts it back if need be, on every way back from a run (hc_take_back) before anything here+ * touches the storage; it reads for the kernel (block_poll) and writes the log (write_logs) only+ * while UEFI's list is the one in the controller.+ */+#define HC_PA 0x1d84000ULL /* ufshc@1d84000, "ufs_mem", in this phone's Android device tree */+#define HC_CAP 0x00+#define HC_VER 0x08+#define HC_IS 0x20+#define HC_IE 0x24+#define HC_HCS 0x30+#define HC_HCE 0x34+#define HC_UTRLBA 0x50+#define HC_UTRLBAU 0x54+#define HC_UTRLDBR 0x58+#define HC_UTRLCLR 0x5c+#define HC_UTRLRSR 0x60+#define HC_UTRLCNR 0x64+#define HC_UTMRLDBR 0x78+#define HC_SLOTS 32+#define HC_UCD_OFFSET 0x1000+#define HC_UCD_SIZE 0x800+#define HC_AREA_SIZE (HC_UCD_OFFSET + HC_SLOTS * HC_UCD_SIZE)++static uint8_t hc_area[HC_AREA_SIZE] __attribute__((aligned(0x2000)));+static int hc_requested; /* ufs_native= on the command line */+static int hc_known; /* UEFI's list address is kept here: check it, put it back */+static uint32_t hc_uefi_lo, hc_uefi_hi;+static int hc_refused_reads;++static inline uint32_t+hc_rd(uint32_t off)+{+ return *(volatile uint32_t *)(uintptr_t)(HC_PA + off);+}++static inline void+hc_wr(uint32_t off, uint32_t v)+{+ __asm__ volatile ("dsb sy" ::: "memory");+ *(volatile uint32_t *)(uintptr_t)(HC_PA + off) = v;+ __asm__ volatile ("dsb sy" ::: "memory");+}++static uint64_t+hc_ms_since(uint64_t then)+{+ return (READ_SYSREG(CNTVCT_EL0) - then) * 1000 / READ_SYSREG(CNTFRQ_EL0);+}++/* UEFI's list in, and running: what UEFI's driver needs to read or write */+static int+hc_list_is_uefis(void)+{+ return !hc_known || (hc_rd(HC_UTRLBA) == hc_uefi_lo && hc_rd(HC_UTRLBAU) == hc_uefi_hi &&+ (hc_rd(HC_UTRLRSR) & 1) != 0);+}++static void+hc_print_state(const char *when)+{+ print("ufs: %s: list %x:%x (UEFI's %x:%x), run %x, doorbell %x, HCS %x, IS %x, IE %x, completion %x, "+ "task doorbell %x\n", when, (uint64_t)hc_rd(HC_UTRLBAU), (uint64_t)hc_rd(HC_UTRLBA), (uint64_t)hc_uefi_hi,+ (uint64_t)hc_uefi_lo, (uint64_t)hc_rd(HC_UTRLRSR), (uint64_t)hc_rd(HC_UTRLDBR), (uint64_t)hc_rd(HC_HCS),+ (uint64_t)hc_rd(HC_IS), (uint64_t)hc_rd(HC_IE), (uint64_t)hc_rd(HC_UTRLCNR), (uint64_t)hc_rd(HC_UTMRLDBR));+}++/*+ * Whatever the kernel left: UEFI's list back in the controller. The kernel is told to send nothing+ * more (it looks before every doorbell, with interrupts off), given 50 ms, then what is in flight+ * is waited for (1 s), cleared if it does not finish (UTRLCLR: every slot of that list is the+ * kernel's), the list stopped, UEFI's address written back, the list started, all read back.+ */+static void+hc_take_back(const char *when)+{+ uint32_t lo, hi, rsr, db;+ uint64_t t;++ if (!hc_known) {+ return;+ }+ lo = hc_rd(HC_UTRLBA);+ hi = hc_rd(HC_UTRLBAU);+ rsr = hc_rd(HC_UTRLRSR);+ db = hc_rd(HC_UTRLDBR);+ if (lo == hc_uefi_lo && hi == hc_uefi_hi && (rsr & 1)) {+ print("ufs: %s: UEFI's transfer list is in the controller (%x), running, doorbell %x\n", when, (uint64_t)lo,+ (uint64_t)db);+ return;+ }+ print("ufs: %s: the controller has list %x:%x (run %x, doorbell %x), not UEFI's %x:%x: taking it back\n", when,+ (uint64_t)hi, (uint64_t)lo, (uint64_t)rsr, (uint64_t)db, (uint64_t)hc_uefi_hi, (uint64_t)hc_uefi_lo);+ block_shared.loader_native.take_back = 1;+ clean_dcache((uint64_t)(uintptr_t)&block_shared.loader_native,+ (uint64_t)(uintptr_t)&block_shared.loader_native + sizeof(block_shared.loader_native));+ for (t = READ_SYSREG(CNTVCT_EL0); hc_ms_since(t) < 50;) {+ }+ for (t = READ_SYSREG(CNTVCT_EL0); (db = hc_rd(HC_UTRLDBR)) != 0 && hc_ms_since(t) < 1000;) {+ }+ if (db != 0) {+ print("ufs: %s: doorbell %x still set after 1 s\n", when, (uint64_t)db);+ if (hc_rd(HC_UTRLBA) != hc_uefi_lo || hc_rd(HC_UTRLBAU) != hc_uefi_hi) {+ print("ufs: %s: clearing the kernel's slots (UTRLCLR)\n", when);+ hc_wr(HC_UTRLCLR, 0);+ for (t = READ_SYSREG(CNTVCT_EL0); (db = hc_rd(HC_UTRLDBR)) != 0 && hc_ms_since(t) < 1000;) {+ }+ print("ufs: %s: doorbell %x after the clear\n", when, (uint64_t)db);+ }+ }+ hc_wr(HC_UTRLRSR, 0);+ for (t = READ_SYSREG(CNTVCT_EL0); (hc_rd(HC_UTRLRSR) & 1) && hc_ms_since(t) < 100;) {+ }+ hc_wr(HC_UTRLBA, hc_uefi_lo);+ hc_wr(HC_UTRLBAU, hc_uefi_hi);+ hc_wr(HC_UTRLRSR, 1);+ for (t = READ_SYSREG(CNTVCT_EL0); hc_ms_since(t) < 50;) {+ }+ hc_print_state(when);+ print("ufs: %s: UEFI's list %s\n", when, hc_list_is_uefis() ? "put back" : "NOT PUT BACK: the storage is not used");+}++/*+ * Before a run (ufs_native=): what the controller looks like, and UEFI's list address kept -- the+ * first time; later it must be the same (or this loader's own area, which is then taken back).+ */+static int+hc_init(void)+{+ uint32_t lo, hi, area = (uint32_t)(uintptr_t)hc_area;++ if (!hc_requested) {+ return 0;+ }+ lo = hc_rd(HC_UTRLBA);+ hi = hc_rd(HC_UTRLBAU);+ print("ufs: controller at %x: CAP %x, VER %x, HCE %x, HCS %x, IS %x, IE %x, list %x:%x, run %x, doorbell %x\n",+ HC_PA, (uint64_t)hc_rd(HC_CAP), (uint64_t)hc_rd(HC_VER), (uint64_t)hc_rd(HC_HCE), (uint64_t)hc_rd(HC_HCS),+ (uint64_t)hc_rd(HC_IS), (uint64_t)hc_rd(HC_IE), (uint64_t)hi, (uint64_t)lo, (uint64_t)hc_rd(HC_UTRLRSR),+ (uint64_t)hc_rd(HC_UTRLDBR));+ if (!hc_known) {+ if (hi == 0 && lo == area) {+ print("ufs: the controller has this loader's own area for its list before any run: left alone\n");+ return 0;+ }+ hc_uefi_lo = lo;+ hc_uefi_hi = hi;+ hc_known = 1;+ print("ufs: UEFI's transfer list is at %x:%x; it is put back after every run\n", (uint64_t)hi, (uint64_t)lo);+ } else if (lo != hc_uefi_lo || hi != hc_uefi_hi) {+ if (hi == 0 && lo == area) {+ hc_take_back("before the run");+ } else {+ print("ufs: UEFI's list moved from %x:%x to %x:%x: that is UEFI's now\n", (uint64_t)hc_uefi_hi,+ (uint64_t)hc_uefi_lo, (uint64_t)hi, (uint64_t)lo);+ hc_uefi_lo = lo;+ hc_uefi_hi = hi;+ }+ }+ return hc_list_is_uefis();+}+/** The firmware's configuration tables, by GUID (an ACPI or device tree table among them would say* where the storage controller is).@@ -1648,6 +1853,25 @@ block_start(void)bp->flags = BLOCK_PART_UNIT;block_bios[block_shared.parts++] = units[u].bio;}+ /* the kernel's own transfer list: this loader's area, if the controller is as UEFI keeps it */+ if (hc_known && hc_list_is_uefis() && hc_rd(HC_UTRLDBR) == 0 && (hc_rd(HC_HCE) & 1)) {+ memset(hc_area, 0, sizeof(hc_area));+ flush_dcache((uint64_t)(uintptr_t)hc_area, (uint64_t)(uintptr_t)hc_area + sizeof(hc_area));+ block_shared.native.hc_pa = HC_PA;+ block_shared.native.area_pa = (uint64_t)(uintptr_t)hc_area;+ block_shared.native.area_size = sizeof(hc_area);+ block_shared.native.ucd_offset = HC_UCD_OFFSET;+ block_shared.native.ucd_size = HC_UCD_SIZE;+ block_shared.native.slots = HC_SLOTS;+ block_shared.native.uefi_utrlba = hc_uefi_lo;+ block_shared.native.uefi_utrlbau = hc_uefi_hi;+ print("ufs: the kernel may put its own transfer list in, at %x (this loader's), for the run\n",+ (uint64_t)(uintptr_t)hc_area);+ } else if (hc_requested) {+ print("ufs: the kernel's own transfer list is not offered: the controller is not as UEFI keeps it\n");+ }+ hc_refused_reads = 0;+ hc_log_skips = 0;block_shared.magic = BLOCK_MAGIC;block_shared.version = BLOCK_VERSION;block_shared.size = sizeof(block_shared);@@ -1764,7 +1988,16 @@ block_poll(void)efi_block_io *bio = block_bios[part];bytes = count * block_shared.part[part].block_size;- s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);+ if (!hc_list_is_uefis()) {+ /* the kernel's own list is in: UEFI's driver must not ring now (the kernel should not ask) */+ if (hc_refused_reads++ < 4) {+ print("ufs: a read asked for while the kernel's own transfer list is in the controller: refused\n");+ }+ s = 0x800000000000000FULL; /* EFI_ACCESS_DENIED */+ bytes = 0;+ } else {+ s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);+ }/* what the controller wrote is in memory; drop whatever this core held of the buffer */flush_dcache((uint64_t)(uintptr_t)block_buffer, (uint64_t)(uintptr_t)block_buffer + bytes);}@@ -1951,7 +2184,8 @@ run_on_other_cpu(uint64_t mpidr, uint64_t entry, uint64_t boot_args_pa, const stlast_pet = READ_SYSREG(CNTVCT_EL0);}/* Qualcomm's UEFI charger stack owns the PMIC/GLINK side; keep XNU's cache live. */- if (charger_enabled && seconds_since(last_battery) >= 5) {+ if (charger_enabled && seconds_since(last_battery) >= 5 && hc_list_is_uefis()) {+ /* (not while the kernel's own transfer list is in: Qualcomm's DXEs are not to touch the storage) */charger_poll();last_battery = READ_SYSREG(CNTVCT_EL0);}@@ -2481,10 +2715,15 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)ufs_requested = c[4] - '0';}block_probe_requested |= starts_with(c, "ufs_probe=1") && (c == pi.cmdline || c[-1] == ' ');+ hc_requested |= starts_with(c, "ufs_native=") && (c == pi.cmdline || c[-1] == ' ') && c[11] >= '1' &&+ c[11] <= '9';}if (battery_requested) {charger_start();}+ if (ufs_requested >= 2) {+ hc_init(); /* (ufs_native=: UEFI's transfer list kept, before anything reads) */+ }if (ufs_requested >= 1) {ufs_survey();}@@ -2908,6 +3147,10 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)int64_t status = run_on_other_cpu(kernel_mpidr, entry_va + slide, ba_pa, kernel_live, slide, kernel_budget);wdt_set_ms(WDT_BITE_MS);+ if (status != 0) {+ /* a kernel that hung may have left its own transfer list in the controller */+ hc_take_back(status == OTHER_CPU_HUNG ? "the kernel hung" : "the kernel did not start");+ }if (status == OTHER_CPU_HUNG) {/* the kernel still has that core; what it logged can be had all the same */print("THE KERNEL HUNG on core %x: no sign of life for %d seconds; its log follows\n", kernel_mpidr,@@ -2933,6 +3176,13 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)/* --- the kernel came back ------------------------------------------- */wdt_set_ms(WDT_BITE_MS);+ hc_take_back("the kernel came back"); /* before anything here touches the storage */+ if (hc_known) {+ flush_dcache((uint64_t)(uintptr_t)&block_shared.kernel_native,+ (uint64_t)(uintptr_t)&block_shared.kernel_native + sizeof(block_shared.kernel_native));+ print("ufs: the kernel put its own list in %d times and took it out %d times\n",+ (uint64_t)block_shared.kernel_native.switched_in, (uint64_t)block_shared.kernel_native.switched_out);+ }/* ... and its other cores should all be off again: the next kernel cannot start one that is not */for (int i = 1; i < cpu_count; i++) {int64_t state = psci_call(PSCI_AFFINITY_INFO, cpu_mpidr[i], 0, 0);