Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

UFS: the kernel's own transfer list, in the loader's memory, put in for a run and taken out again

vcvkkf25e6a6f25d6
diff --git a/tools/flash-fingerprint.sh b/tools/flash-fingerprint.sh
new file mode 100755
index 0000000..2bdd23a
--- /dev/null
+++ b/tools/flash-fingerprint.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# A read-only fingerprint of the phone's flash, to tell whether anything has changed it.
+#
+# flash-fingerprint.sh android DIR from Android (adb, root): for each logical unit sda..sdf its
+# size, sha256 of its first and last 8 blocks of 4 KB and
+# `sgdisk --android-dump`; sha256 of userdata's first MB, of
+# darwin's first 5628755968 bytes (the iOS volume image), the
+# number of files under /sdcard, and whether Android is up
+# (sys.boot_completed, /data mounted) -> DIR/android.txt
+# flash-fingerprint.sh survey LOG DIR the loader's own survey (ufs=1 or 2) out of a run's log:
+# each unit's GPT header and table CRCs, every partition's
+# place, the hashes of the partitions there are images of,
+# super's metadata, userdata's superblock (timings dropped)
+# -> DIR/survey.txt
+# flash-fingerprint.sh diff DIR1 DIR2 what differs
+#
+# Nothing here writes to the phone: dd only reads (if=), sgdisk only prints (--android-dump), and the
+# script goes to the phone's shell on its standard input, not as a file.
+set -e
+case "$1" in
+android)
+ mkdir -p "$2"
+ adb shell 'su -c sh' > "$2/android.txt.new" <<'PHONE'
+for d in sda sdb sdc sdd sde sdf; do
+ size=$(blockdev --getsize64 /dev/block/$d)
+ n=$(($(blockdev --getsz /dev/block/$d) / 8)) # (the phone's shell counts in 32 bits: sectors, not bytes)
+ first=$(dd if=/dev/block/$d bs=4096 count=8 2>/dev/null | sha256sum | cut -d' ' -f1)
+ last=$(dd if=/dev/block/$d bs=4096 skip=$((n - 8)) count=8 2>/dev/null | sha256sum | cut -d' ' -f1)
+ echo "$d: $size bytes, first 8 blocks $first, last 8 blocks $last"
+ sgdisk --android-dump /dev/block/$d 2>&1 | sed "s/^/$d sgdisk: /"
+done
+echo "userdata ($(readlink /dev/block/by-name/userdata)) first MB: $(dd if=/dev/block/by-name/userdata bs=1048576 count=1 2>/dev/null | sha256sum | cut -d' ' -f1)"
+echo "darwin ($(readlink /dev/block/by-name/darwin)) first 5628755968 bytes: $(dd if=/dev/block/by-name/darwin bs=4194304 count=1342 2>/dev/null | sha256sum | cut -d' ' -f1)"
+echo "files under /sdcard: $(find /sdcard/ -type f 2>/dev/null | wc -l)"
+echo "android: boot_completed=$(getprop sys.boot_completed), /data: $(mount | grep ' /data ' | cut -d' ' -f1,5)"
+PHONE
+ mv "$2/android.txt.new" "$2/android.txt"
+ grep -E '^(sd[a-f]:|userdata|darwin|files|android)' "$2/android.txt"
+ ;;
+survey)
+ mkdir -p "$3"
+ awk '/^== KERNEL LOG ==/{exit} {print}' "$2" | tr -d '\r' | grep -a '^ufs: ' |
+ grep -E '^ufs: (unit [0-9]+: (media|GPT|path)|part |[A-Za-z0-9_]+: fnv |super |userdata sb |no partition)' |
+ sed -E 's/, [0-9]+ ms$//' > "$3/survey.txt"
+ echo "$(wc -l < "$3/survey.txt") survey lines from $2"
+ grep -E 'GPT header|fnv' "$3/survey.txt" | head -40
+ ;;
+diff)
+ for f in android.txt survey.txt; do
+ if [ -f "$2/$f" ] && [ -f "$3/$f" ]; then
+ if diff -u "$2/$f" "$3/$f"; then
+ echo "$f: the same"
+ fi
+ else
+ echo "$f: not in both"
+ fi
+ done
+ ;;
+*)
+ sed -n '2,20p' "$0"
+ exit 1
+ ;;
+esac
diff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.c
index 8912d87..6a5403e 100644
--- a/xnu/files/bsd/dev/redwood_ufs.c
+++ b/xnu/files/bsd/dev/redwood_ufs.c
@@ -43,6 +43,7 @@
#include <sys/redwood_input.h>
#include <libkern/crc.h>
#include <vm/memory_types.h>
+#include "redwood_ufshc.h"
extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
extern void IOSleep(unsigned int milliseconds);
@@ -51,7 +52,7 @@ extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length); /* osfmk/arm
/* the page the loader publishes: the same layout as struct loader_block in xnuloader/loader.c */
#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */
-#define BLOCK_VERSION 2
+#define BLOCK_VERSION 3
#define BLOCK_MAX_PARTS 160
#define BLOCK_MAX_UNITS 16
#define BLOCK_PART_UNIT 1u /* a whole logical unit, not a partition */
@@ -84,6 +85,20 @@ struct loader_block {
uint64_t served, failed, bytes, ticks;
} answer __attribute__((aligned(128)));
struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));
+ /* the loader's offer of an area for this kernel's own transfer request list, and UEFI's list */
+ struct {
+ uint64_t hc_pa;
+ uint64_t area_pa;
+ uint32_t area_size, ucd_offset, ucd_size, slots;
+ uint32_t uefi_utrlba, uefi_utrlbau;
+ } native __attribute__((aligned(128)));
+ struct {
+ volatile uint32_t take_back; /* the loader's: it takes the controller back, send nothing more */
+ } loader_native __attribute__((aligned(128)));
+ struct {
+ volatile uint32_t list_in; /* this kernel's: its list is in */
+ uint32_t switched_in, switched_out, reserved;
+ } kernel_native __attribute__((aligned(128)));
};
#define UFS_PAGE 0x4000UL
@@ -105,17 +120,13 @@ static uint64_t ufs_last_end;
/*
* This kernel's own UFS driver (redwood_ufshc.c), with ufs_native=1: each partition's logical
* unit and where it begins there, in bytes; reads go that way unless it fails, and then through
- * the loader again.
+ * the loader again. While its list is in the controller the loader is asked for nothing.
*/
-extern bool ufshc_probe(bool only);
-extern bool ufshc_available(void);
-extern int ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len);
-extern void ufshc_report(const char *when);
static uint8_t ufs_lun[BLOCK_MAX_PARTS]; /* 0xff: not known, read through the loader */
static uint64_t ufs_lun_offset[BLOCK_MAX_PARTS];
static bool ufs_native_mapped;
static volatile bool ufs_native_on;
-static uint64_t ufs_native_fallbacks;
+static uint64_t ufs_native_fallbacks, ufs_mailbox_refused;
static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");
static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);
@@ -268,6 +279,15 @@ ufs_read_mailbox(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
}
}
lck_mtx_lock(&ufs_lock);
+ if (ufshc_list_in()) {
+ /* this kernel's own transfer list is in the controller: UEFI's driver must not be asked to ring */
+ if (ufs_mailbox_refused++ < 4) {
+ printf("ufs: %s: not asked of the loader while this kernel's own transfer list is in\n",
+ ufs_page->part[part].name);
+ }
+ lck_mtx_unlock(&ufs_lock);
+ return EIO;
+ }
while (len > 0 && error == 0) {
uint64_t first = offset / nbs, skip = offset % nbs;
uint64_t count = (skip + len + nbs - 1) / nbs, take;
@@ -305,6 +325,30 @@ ufs_read_native(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
return ufshc_read(ufs_lun[part], ufs_lun_offset[part] + offset, dst, len);
}
+/* The switches of lists, with the loader's mailbox held: nothing is asked of UEFI meanwhile */
+static int
+ufs_native_in(bool prove)
+{
+ int error;
+
+ lck_mtx_lock(&ufs_lock);
+ /* (a loader that did not answer may still be inside UEFI's driver) */
+ error = ufs_dead ? EIO : ufshc_switch_in(prove);
+ lck_mtx_unlock(&ufs_lock);
+ return error;
+}
+
+static int
+ufs_native_out(const char *why)
+{
+ int error;
+
+ lck_mtx_lock(&ufs_lock);
+ error = ufshc_switch_out(why);
+ lck_mtx_unlock(&ufs_lock);
+ return error;
+}
+
/* The way chosen (ufs_native=, kern.redwood.ufs_native); if this kernel's own fails, the loader's */
static int
ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
@@ -315,10 +359,13 @@ ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
if (error == 0) {
return 0;
}
- ufs_native_on = false;
- ufs_native_fallbacks++;
- printf("ufs: %s: bytes %llu..%llu: this kernel's own read failed (%d); through the loader from now on\n",
- ufs_page->part[part].name, offset, offset + len - 1, error);
+ if (ufs_native_on) {
+ ufs_native_on = false;
+ ufs_native_fallbacks++;
+ printf("ufs: %s: bytes %llu..%llu: this kernel's own read failed (%d); through the loader from now on\n",
+ ufs_page->part[part].name, offset, offset + len - 1, error);
+ }
+ ufs_native_out("a read failed");
}
return ufs_read_mailbox(part, offset, dst, len);
}
@@ -594,234 +641,278 @@ ufs_fnv(uint64_t h, const uint8_t *p, uint64_t n)
return h;
}
+static uint32_t
+ufs_entry(const char *name)
+{
+ for (uint32_t i = 0; i < ufs_parts; i++) {
+ if (strcmp((const char *)ufs_page->part[i].name, name) == 0) {
+ return i;
+ }
+ }
+ return BLOCK_MAX_PARTS;
+}
+
/*
- * Each unit's logical unit number: the one its device path gives, if it gives one, else the first
- * of 0..7 not yet taken whose block 1 (the GPT header, with the unit's own GUID in it) is what the
- * loader reads as block 1 of that unit. Either way it is so only if the two are the same.
+ * Which logical unit each unit the loader lists is (their device paths do not say): unit k is
+ * taken to be LUN k -- with ufs_native=1 only the unit darwin is on, which must be LUN 0 -- and it
+ * is so only if block 1 (the GPT header, with the unit's own GUID in it) read through the loader
+ * is the same as block 1 of that LUN read here; no other LUN is read. The loader's reads are made
+ * first, with UEFI's list in; then this kernel's list goes in (with its proof) and stays in.
*/
static bool
-ufs_native_map(void)
+ufs_native_map(bool all_units)
{
- uint8_t *a = kalloc_data(2 * 4096, Z_WAITOK), *b = a + 4096;
- uint8_t unit_lun[BLOCK_MAX_UNITS];
- uint32_t taken = 0, units = 0;
- bool all = true;
+ uint8_t *mine = kalloc_data(4096, Z_WAITOK), *theirs = kalloc_data(BLOCK_MAX_UNITS * 4096, Z_WAITOK);
+ uint32_t units[BLOCK_MAX_UNITS], n = 0, mask = 0, darwin = ufs_entry("darwin");
+ bool good = mine != NULL && theirs != NULL;
- memset(unit_lun, 0xff, sizeof(unit_lun));
memset(ufs_lun, 0xff, sizeof(ufs_lun));
- if (a == NULL) {
- return false;
- }
- for (uint32_t i = 0; i < ufs_parts && all; i++) {
+ for (uint32_t i = 0; good && i < ufs_parts && n < BLOCK_MAX_UNITS; i++) {
volatile struct loader_block_part *e = &ufs_page->part[i];
- bool found = false;
- if ((e->flags & BLOCK_PART_UNIT) == 0) {
+ if ((e->flags & BLOCK_PART_UNIT) == 0 || e->unit >= 8) {
continue;
}
- units++;
- if (e->unit >= BLOCK_MAX_UNITS || e->block_size != 4096 || ufs_read_mailbox(i, 4096, a, 4096) != 0 ||
- memcmp(a, "EFI PART", 8) != 0) {
+ if (!all_units && (darwin == BLOCK_MAX_PARTS || ufs_page->part[darwin].unit != e->unit)) {
+ continue;
+ }
+ if (e->block_size != 4096 || ufs_read_mailbox(i, 4096, theirs + n * 4096, 4096) != 0 ||
+ memcmp(theirs + n * 4096, "EFI PART", 8) != 0) {
printf("ufs: native: %s: no GPT header to know it by (block size %u)\n", e->name, e->block_size);
- all = false;
+ good = false;
break;
}
- for (uint32_t k = 0; k < 8 && !found; k++) {
- uint32_t lun = e->lun != 0xff ? e->lun : (k == 0 ? e->unit : k - 1);
+ units[n++] = i;
+ mask |= 1u << e->unit;
+ }
+ if (good && !all_units && (n != 1 || ufs_page->part[units[0]].unit != 0)) {
+ printf("ufs: native: darwin is not on the first unit the loader lists: not LUN 0, not read here\n");
+ good = false;
+ }
+ ufshc_set_luns(mask);
+ if (good && ufs_native_in(true) != 0) {
+ good = false;
+ }
+ for (uint32_t k = 0; good && k < n; k++) {
+ volatile struct loader_block_part *e = &ufs_page->part[units[k]];
- if (lun >= 8 || (taken & (1u << lun)) || (e->lun != 0xff && k > 0)) {
- continue;
- }
- if (ufshc_read((uint8_t)lun, 4096, b, 4096) == 0 && memcmp(a, b, 4096) == 0) {
- unit_lun[e->unit] = (uint8_t)lun;
- taken |= 1u << lun;
- found = true;
- printf("ufs: native: %s is LUN %u (%s; its GPT header read both ways is the same)\n", e->name, lun,
- e->lun != 0xff ? "its device path says so" : "found by its GPT header");
- }
- }
- if (!found) {
- printf("ufs: native: %s: no logical unit reads as it does through the loader\n", e->name);
- all = false;
+ if (ufshc_read((uint8_t)e->unit, 4096, mine, 4096) != 0 || memcmp(mine, theirs + k * 4096, 4096) != 0) {
+ printf("ufs: native: %s: LUN %u's block 1 is NOT what the loader reads as its block 1\n", e->name, e->unit);
+ good = false;
+ } else {
+ printf("ufs: native: %s is LUN %u: its GPT header read both ways is the same\n", e->name, e->unit);
}
}
- kfree_data(a, 2 * 4096);
- if (!all || units == 0) {
+ if (!good) {
+ ufs_native_out("the units could not be told");
memset(ufs_lun, 0xff, sizeof(ufs_lun));
- return false;
- }
- for (uint32_t i = 0; i < ufs_parts; i++) {
- volatile struct loader_block_part *e = &ufs_page->part[i];
+ } else {
+ for (uint32_t i = 0; i < ufs_parts; i++) {
+ volatile struct loader_block_part *e = &ufs_page->part[i];
- if (e->unit < BLOCK_MAX_UNITS && unit_lun[e->unit] != 0xff && e->block_size == 4096) {
- ufs_lun[i] = unit_lun[e->unit];
- ufs_lun_offset[i] = e->start_lba * 4096;
+ if (e->unit < 8 && (mask & (1u << e->unit)) && e->block_size == 4096) {
+ ufs_lun[i] = (uint8_t)e->unit;
+ ufs_lun_offset[i] = e->start_lba * 4096;
+ }
}
}
- return true;
+ if (mine != NULL) {
+ kfree_data(mine, 4096);
+ }
+ if (theirs != NULL) {
+ kfree_data(theirs, BLOCK_MAX_UNITS * 4096);
+ }
+ return good;
}
-static uint64_t ufs_cmp_mailbox_ns, ufs_cmp_native_ns, ufs_cmp_bytes, ufs_cmp_regions, ufs_cmp_differ;
-
/*
- * Bytes [offset, offset + len) of entry part, read both ways a megabyte at a time and compared;
- * FNV-1a of what was read (each way), and of its first ref bytes if ref is not 0.
+ * ufs_native_check=1: what this kernel's own driver reads is what the loader reads, byte for
+ * byte. Spans are read a batch at a time, first all through the loader (UEFI's list in), then
+ * all here (this kernel's list in, proved), and compared.
*/
+struct ufs_span {
+ uint32_t part;
+ uint64_t offset, len;
+};
+
+#define UFS_CMP_BUFFER (32u << 20)
+
+static uint8_t *ufs_cmp_x, *ufs_cmp_y;
+static uint64_t ufs_cmp_mailbox_ns, ufs_cmp_native_ns, ufs_cmp_bytes, ufs_cmp_spans, ufs_cmp_differ, ufs_cmp_failed;
+
static bool
-ufs_compare(uint32_t part, uint64_t offset, uint64_t len, uint64_t ref, const char *what, bool quiet)
+ufs_compare_batch(const struct ufs_span *s, uint32_t n, const char *what, bool quiet)
{
- static uint8_t *x, *y;
- uint64_t hx = 0xcbf29ce484222325ULL, hy = hx, href = 0, done = 0, t, ns, mailbox_ns = 0, native_ns = 0;
- int ex = 0, ey = 0;
+ uint64_t at, t, ns, hx = 0xcbf29ce484222325ULL, hy = hx, bytes = 0, mailbox_ns, native_ns;
bool same = true;
+ int e = 0;
- if (x == NULL) {
- x = kalloc_data(2 << 20, Z_WAITOK);
- y = x + (1 << 20);
- }
- while (done < len && same && ex == 0 && ey == 0) {
- uint64_t n = len - done < (1 << 20) ? len - done : (1 << 20);
-
- t = mach_absolute_time();
- ex = ufs_read_mailbox(part, offset + done, x, n);
- absolutetime_to_nanoseconds(mach_absolute_time() - t, &ns);
- mailbox_ns += ns;
- t = mach_absolute_time();
- ey = ufs_read_native(part, offset + done, y, n);
- absolutetime_to_nanoseconds(mach_absolute_time() - t, &ns);
- native_ns += ns;
- if (ex == 0 && ey == 0 && memcmp(x, y, n) != 0) {
- uint64_t at = 0;
-
- while (x[at] == y[at]) {
- at++;
- }
- printf("ufs: native check: %s %s: DIFFERENT at byte %llu (%02x through the loader, %02x here)\n",
- ufs_page->part[part].name, what, offset + done + at, x[at], y[at]);
- same = false;
- }
- if (ref != 0 && done < ref) {
- uint64_t m = ref - done < n ? ref - done : n;
-
- href = ufs_fnv(done == 0 ? 0xcbf29ce484222325ULL : href, x, m);
- }
- hx = ufs_fnv(hx, x, n);
- hy = ufs_fnv(hy, y, n);
- done += n;
+ t = mach_absolute_time();
+ if (ufs_native_out("the loader reads, to compare") != 0) {
+ e = EIO;
+ }
+ at = 0;
+ for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {
+ e = ufs_read_mailbox(s[i].part, s[i].offset, ufs_cmp_x + at, s[i].len);
+ }
+ absolutetime_to_nanoseconds(mach_absolute_time() - t, &mailbox_ns);
+ t = mach_absolute_time();
+ if (e == 0 && ufs_native_in(true) != 0) {
+ e = EIO;
}
- if (ex != 0 || ey != 0) {
- printf("ufs: native check: %s %s: read failed (through the loader %d, here %d)\n", ufs_page->part[part].name,
- what, ex, ey);
- same = false;
+ at = 0;
+ for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {
+ e = ufs_read_native(s[i].part, s[i].offset, ufs_cmp_y + at, s[i].len);
}
- ufs_cmp_regions++;
- ufs_cmp_bytes += done;
+ absolutetime_to_nanoseconds(mach_absolute_time() - t, &native_ns);
ufs_cmp_mailbox_ns += mailbox_ns;
ufs_cmp_native_ns += native_ns;
- ufs_cmp_differ += !same;
- if (!quiet || !same) {
- printf("ufs: native check: %s %s: %llu bytes from %llu %s: fnv %llx / %llx; loader %llu ms, here %llu ms\n",
- ufs_page->part[part].name, what, done, offset, same ? "the same" : "NOT THE SAME", hx, hy,
- mailbox_ns / 1000000, native_ns / 1000000);
- if (ref != 0) {
- printf("ufs: native check: %s: fnv %llx over the first %llu bytes (the loader's and ufs_check's hash)\n",
- ufs_page->part[part].name, href, ref);
+ if (e != 0) {
+ printf("ufs: native check: %s: a read failed (%d): STOP\n", what, e);
+ ufs_cmp_failed++;
+ return false;
+ }
+ at = 0;
+ for (uint32_t i = 0; i < n; i++) {
+ if (memcmp(ufs_cmp_x + at, ufs_cmp_y + at, s[i].len) != 0) {
+ uint64_t k = 0;
+
+ while (ufs_cmp_x[at + k] == ufs_cmp_y[at + k]) {
+ k++;
+ }
+ printf("ufs: native check: %s %s: DIFFERENT at byte %llu (%02x through the loader, %02x here): STOP\n",
+ ufs_page->part[s[i].part].name, what, s[i].offset + k, ufs_cmp_x[at + k], ufs_cmp_y[at + k]);
+ same = false;
+ ufs_cmp_differ++;
}
+ hx = ufs_fnv(hx, ufs_cmp_x + at, s[i].len);
+ hy = ufs_fnv(hy, ufs_cmp_y + at, s[i].len);
+ at += s[i].len;
+ bytes += s[i].len;
+ }
+ ufs_cmp_spans += n;
+ ufs_cmp_bytes += bytes;
+ if (!quiet || !same) {
+ printf("ufs: native check: %s %s: %u span(s), %llu bytes from %llu: %s (fnv %llx / %llx); loader %llu ms, here "
+ "%llu ms (switches included)\n", ufs_page->part[s[0].part].name, what, n, bytes, s[0].offset,
+ same ? "the same" : "NOT THE SAME", hx, hy, mailbox_ns / 1000000, native_ns / 1000000);
}
return same;
}
-static uint32_t
-ufs_entry(const char *name)
+/* Bytes [offset, offset + len) of an entry, compared a buffer at a time; FNV-1a of the first ref bytes too */
+static bool
+ufs_compare(uint32_t part, uint64_t offset, uint64_t len, uint64_t ref, const char *what)
{
- for (uint32_t i = 0; i < ufs_parts; i++) {
- if (strcmp((const char *)ufs_page->part[i].name, name) == 0) {
- return i;
+ uint64_t done = 0, href = 0xcbf29ce484222325ULL;
+ bool same = true;
+
+ while (done < len && same) {
+ struct ufs_span s = { part, offset + done, len - done < UFS_CMP_BUFFER ? len - done : UFS_CMP_BUFFER };
+
+ same = ufs_compare_batch(&s, 1, what, true);
+ if (same && ref > done) {
+ href = ufs_fnv(href, ufs_cmp_x, ref - done < s.len ? ref - done : s.len);
}
+ done += s.len;
}
- return BLOCK_MAX_PARTS;
+ printf("ufs: native check: %s %s: %llu bytes from %llu: %s%s", ufs_page->part[part].name, what, done, offset,
+ same ? "the same" : "NOT THE SAME", ref ? "" : "\n");
+ if (ref) {
+ printf("; fnv %llx over the first %llu bytes (the loader's own survey hashes them)\n", href, ref);
+ }
+ return same;
}
-/*
- * ufs_native_check=1: what this kernel's own driver reads is what the loader reads, byte for
- * byte: every unit's GPT (protective MBR, header, table, and the backup table and header at the
- * end), the partitions the loader hashes (the ROM's images: xbl, tz, modem, ...) whole, the start
- * of super (its geometry and metadata), and of darwin 64 spread megabytes, 512 scattered blocks
- * and some reads that begin and end inside a block.
- */
static void
ufs_native_check(void)
{
uint8_t *h = kalloc_data(4096, Z_WAITOK);
uint32_t part;
- uint64_t seed = 0x9e3779b97f4a7c15ULL;
+ bool going = true;
+ ufs_cmp_x = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);
+ ufs_cmp_y = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);
+ if (h == NULL || ufs_cmp_x == NULL || ufs_cmp_y == NULL) {
+ printf("ufs: native check: no memory\n");
+ return;
+ }
printf("ufs: native check: through the loader and through this kernel's own driver, compared\n");
- for (uint32_t i = 0; h != NULL && i < ufs_parts; i++) {
+ /* every unit's GPT: protective MBR, header, table; the backup table and header at the end */
+ for (uint32_t i = 0; going && i < ufs_parts; i++) {
volatile struct loader_block_part *e = &ufs_page->part[i];
uint64_t entries_lba, count, size, table, alt, alt_entries;
+ uint32_t want, got;
- if ((e->flags & BLOCK_PART_UNIT) == 0 || ufs_read_mailbox(i, 4096, h, 4096) != 0) {
+ if ((e->flags & BLOCK_PART_UNIT) == 0 || ufs_lun[i] == 0xff) {
continue;
}
+ if (ufs_native_out("the loader reads, to compare") != 0 || ufs_read_mailbox(i, 4096, h, 4096) != 0) {
+ going = false;
+ break;
+ }
entries_lba = *(uint64_t *)(h + 72);
count = *(uint32_t *)(h + 80);
size = *(uint32_t *)(h + 84);
alt = *(uint64_t *)(h + 32);
table = (count * size + 4095) / 4096;
- {
- uint32_t want = *(uint32_t *)(h + 16), got;
-
- *(uint32_t *)(h + 16) = 0;
- got = crc32(0, h, *(uint32_t *)(h + 12));
- printf("ufs: native check: %s: GPT header crc %s, table at block %llu, %llu entries of %llu bytes, backup "
- "header at block %llu\n", e->name, got == want ? "right" : "WRONG", entries_lba, count, size, alt);
- }
- ufs_compare(i, 0, (entries_lba + table) * 4096, 0, "GPT (MBR, header, table)", false);
- if (alt < e->blocks && ufs_read_mailbox(i, alt * 4096, h, 4096) == 0 && memcmp(h, "EFI PART", 8) == 0) {
- alt_entries = *(uint64_t *)(h + 72);
- if (alt_entries <= alt) {
- ufs_compare(i, alt_entries * 4096, (alt + 1 - alt_entries) * 4096, 0, "backup GPT (table, header)",
- false);
- }
+ want = *(uint32_t *)(h + 16);
+ *(uint32_t *)(h + 16) = 0;
+ got = crc32(0, h, *(uint32_t *)(h + 12));
+ printf("ufs: native check: %s (LUN %u): GPT header crc %s (%x), table at block %llu, %llu entries of %llu bytes, "
+ "backup header at block %llu\n", e->name, ufs_lun[i], got == want ? "right" : "WRONG", got, entries_lba, count,
+ size, alt);
+ going = ufs_compare(i, 0, (entries_lba + table) * 4096, 0, "GPT (MBR, header, table)");
+ if (!going || alt >= e->blocks || ufs_native_out("the loader reads, to compare") != 0 ||
+ ufs_read_mailbox(i, alt * 4096, h, 4096) != 0 || memcmp(h, "EFI PART", 8) != 0 ||
+ (alt_entries = *(uint64_t *)(h + 72)) > alt) {
+ printf("ufs: native check: %s: no backup GPT header at block %llu: STOP\n", e->name, alt);
+ going = false;
+ break;
}
+ going = ufs_compare(i, alt_entries * 4096, (alt + 1 - alt_entries) * 4096, 0, "backup GPT (table, header)");
}
- for (part = 0; part < ufs_parts; part++) {
- if (ufs_page->part[part].flags & BLOCK_PART_UNIT) {
+ /* the partitions there are images of, whole */
+ for (part = 0; going && part < ufs_parts; part++) {
+ if ((ufs_page->part[part].flags & BLOCK_PART_UNIT) || ufs_lun[part] == 0xff) {
continue;
}
- for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
+ for (unsigned r = 0; going && r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
if (ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {
- ufs_compare(part, 0, ufs_part_bytes(part), ufs_references[r].bytes, "whole", false);
+ going = ufs_compare(part, 0, ufs_part_bytes(part), ufs_references[r].bytes, "whole");
}
}
}
- if ((part = ufs_entry("super")) < BLOCK_MAX_PARTS) {
- ufs_compare(part, 0, 8 << 20, 0, "first 8 MB (geometry, metadata)", false);
+ /* super's geometry and metadata */
+ if (going && (part = ufs_entry("super")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {
+ going = ufs_compare(part, 0, 8 << 20, 0, "first 8 MB (geometry, metadata)");
}
- if ((part = ufs_entry("darwin")) < BLOCK_MAX_PARTS) {
- uint64_t bytes = ufs_part_bytes(part), r0 = ufs_cmp_regions, d0 = ufs_cmp_differ;
+ /* darwin: 32 spread megabytes, 256 scattered blocks, 32 pieces that begin and end inside a block */
+ if (going && (part = ufs_entry("darwin")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {
+ struct ufs_span s[256];
+ uint64_t bytes = ufs_part_bytes(part), seed = 0x9e3779b97f4a7c15ULL, total = 0;
- for (uint64_t k = 0; k < 64; k++) {
- ufs_compare(part, (bytes - (1 << 20)) / 64 * k / 4096 * 4096, 1 << 20, 0, "spread megabyte", true);
+ for (uint32_t k = 0; k < 32; k++) {
+ s[k] = (struct ufs_span){ part, (bytes - (1 << 20)) / 32 * k / 4096 * 4096, 1 << 20 };
}
- for (uint64_t k = 0; k < 512; k++) {
+ going = ufs_compare_batch(s, 32, "spread megabytes", false);
+ for (uint32_t k = 0; k < 256; k++) {
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
- ufs_compare(part, (seed >> 16) % (bytes / 4096) * 4096, 4096, 0, "scattered block", true);
+ s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes / 4096) * 4096, 4096 };
}
- for (uint64_t k = 0; k < 64; k++) {
+ going = going && ufs_compare_batch(s, 256, "scattered blocks", false);
+ for (uint32_t k = 0; k < 32; k++) {
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
- ufs_compare(part, (seed >> 16) % (bytes - (2 << 20)), 1 + (seed >> 40) % (1 << 20), 0, "unaligned piece", true);
+ s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes - (2 << 20)), 1 + (seed >> 40) % (512 << 10) };
+ total += s[k].len;
}
- printf("ufs: native check: darwin: %llu spread megabytes, scattered blocks and unaligned pieces: %llu not the same\n",
- ufs_cmp_regions - r0, ufs_cmp_differ - d0);
+ going = going && ufs_compare_batch(s, 32, "unaligned pieces", false);
}
- if (h != NULL) {
- kfree_data(h, 4096);
- }
- printf("ufs: native check done: %llu regions, %llu bytes, %llu NOT THE SAME; through the loader %llu ms, "
- "through this kernel's own driver %llu ms\n", ufs_cmp_regions, ufs_cmp_bytes, ufs_cmp_differ,
- ufs_cmp_mailbox_ns / 1000000, ufs_cmp_native_ns / 1000000);
+ kfree_data(h, 4096);
+ printf("ufs: native check %s: %llu spans, %llu bytes, %llu NOT THE SAME, %llu failed reads; through the loader %llu ms, "
+ "through this kernel's own driver %llu ms (list switches included)\n", going ? "done" : "STOPPED", ufs_cmp_spans,
+ ufs_cmp_bytes, ufs_cmp_differ, ufs_cmp_failed, ufs_cmp_mailbox_ns / 1000000, ufs_cmp_native_ns / 1000000);
+ ufs_native_out("after the check");
ufshc_report("after the check");
}
@@ -835,10 +926,15 @@ ufs_sysctl_native(__unused struct sysctl_oid *oidp, __unused void *arg1, __unuse
if (error != 0 || !changed) {
return error;
}
- if (value != 0 && !(ufs_native_mapped && ufshc_available())) {
- return ENXIO;
+ if (value != 0) {
+ if (!(ufs_native_mapped && ufshc_available()) || ufs_native_in(true) != 0) {
+ return ENXIO;
+ }
+ ufs_native_on = true;
+ } else {
+ ufs_native_on = false;
+ ufs_native_out("kern.redwood.ufs_native=0");
}
- ufs_native_on = value != 0;
return 0;
}
SYSCTL_PROC(_kern_redwood, OID_AUTO, ufs_native, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
@@ -901,25 +997,53 @@ redwood_ufs_init(void)
printf("ufs: %u partitions of the phone's storage as /dev/ufs_NAME and /dev/rufs_NAME, read only, "
"through the loader's page at %llx and its %u KB buffer\n", made, pa, ufs_buffer_size >> 10);
- /* ufs_native=1: through this kernel's own driver instead, if the controller is as expected; 2: only look */
+ /*
+ * ufs_native=1: through this kernel's own driver instead, its own transfer list in the
+ * controller for the run, if all is as expected; 2: only look; 3: a dry run -- the list in and
+ * out again, and in once more for the end of the run to take out, no command sent.
+ */
check = 0;
memset(ufs_lun, 0xff, sizeof(ufs_lun));
if (PE_parse_boot_argn("ufs_native", &check, sizeof(check)) && check != 0) {
- if (ufshc_probe(check == 2) && ufs_native_map()) {
+ struct ufshc_loader offer = {
+ .hc_pa = ufs_page->native.hc_pa, .area_pa = ufs_page->native.area_pa,
+ .area_size = ufs_page->native.area_size, .ucd_offset = ufs_page->native.ucd_offset,
+ .ucd_size = ufs_page->native.ucd_size, .slots = ufs_page->native.slots,
+ .uefi_lo = ufs_page->native.uefi_utrlba, .uefi_hi = ufs_page->native.uefi_utrlbau,
+ .take_back = &ufs_page->loader_native.take_back, .list_in = &ufs_page->kernel_native.list_in,
+ .switched_in = (uint32_t *)&ufs_page->kernel_native.switched_in,
+ .switched_out = (uint32_t *)&ufs_page->kernel_native.switched_out,
+ };
+ uint32_t mode = check, compare = 0;
+
+ PE_parse_boot_argn("ufs_native_check", &compare, sizeof(compare));
+ if (!ufshc_probe(&offer, mode == 2)) {
+ if (mode != 2) {
+ printf("ufs: this kernel's own UFS driver is not used: through the loader\n");
+ }
+ } else if (mode == 3) {
+ printf("ufs: native: dry run: this kernel's list in and out, no command\n");
+ if (ufs_native_in(false) == 0) {
+ IOSleep(1000);
+ ufs_native_out("dry run");
+ IOSleep(500);
+ ufs_native_in(false);
+ printf("ufs: native: dry run: the list stays in for the end of the run\n");
+ }
+ } else if (ufs_native_map(compare != 0)) {
ufs_native_mapped = true;
+ if (compare != 0) {
+ /* right here, in bsd_init's thread, and then back to the loader */
+ ufs_native_check();
+ redwood_ufs_report("after the check");
+ loader_return();
+ }
ufs_native_on = true;
- printf("ufs: the partitions are read by this kernel's own UFS driver (kern.redwood.ufs_native=0: "
- "through the loader)\n");
- } else if (check != 2) {
+ printf("ufs: the partitions on LUN 0 are read by this kernel's own UFS driver, its list in the controller "
+ "(kern.redwood.ufs_native=0: through the loader)\n");
+ } else {
printf("ufs: this kernel's own UFS driver is not used: through the loader\n");
}
- check = 0;
- if (ufs_native_mapped && PE_parse_boot_argn("ufs_native_check", &check, sizeof(check)) && check != 0) {
- /* right here, in bsd_init's thread, and then back to the loader */
- ufs_native_check();
- redwood_ufs_report("after the check");
- loader_return();
- }
}
if (PE_parse_boot_argn("ufs_bench", &check, sizeof(check)) && check != 0) {
diff --git a/xnu/files/bsd/dev/redwood_ufshc.c b/xnu/files/bsd/dev/redwood_ufshc.c
index 636f9fc..acaf5d6 100644
--- a/xnu/files/bsd/dev/redwood_ufshc.c
+++ b/xnu/files/bsd/dev/redwood_ufshc.c
@@ -1,62 +1,57 @@
/*
- * The phone's UFS host controller, driven from this kernel -- read only, and shared with UEFI.
+ * The phone's UFS host controller, driven from this kernel -- read only, next to UEFI's driver.
*
* Reading the storage through the loader (redwood_ufs.c) is one request at a time: Qualcomm's
- * UFS driver in UEFI (UFSDxe) reads synchronously on core 0, and its asynchronous reads finish
- * only from a UEFI timer that does not tick there while the kernel runs. The controller itself
- * takes up to 32 requests at once. This file puts requests of its own into it, next to UEFI's,
- * without taking the controller away from UEFI: UEFI still reads for the loader and writes the
- * run's log at its end.
+ * UFSDxe reads synchronously on core 0, and its transfer request list has room for one request
+ * only (UTRLBA points at a single descriptor, with UFSDxe's command descriptor right behind it,
+ * where slots 8.. would be). The controller takes 32 at once. So for the length of a run this
+ * kernel puts a list of its own into the controller: with nothing in flight, the list is stopped
+ * (UTRLRSR 0), UTRLBA/UTRLBAU are pointed at the new one, the list is started again; at the end
+ * (redwood_ufs_quiesce, from loader_return), what is in flight is waited for, the list stopped,
+ * UEFI's address written back, the list started. The list and its command descriptors are in
+ * memory the loader owns (it offers it in its page, with UEFI's address); the loader checks the
+ * address on every way back from a run and puts UEFI's back itself if the kernel did not
+ * (xnuloader/loader.c, hc_take_back), and neither reads nor writes for the kernel or for its log
+ * while another list is in. While this kernel's list is in, it asks the loader for nothing.
*
- * What it does not do, ever: reset or enable the controller, start the link, send UIC commands
- * (DME_GET/DME_SET), change the request lists' addresses, the interrupt enables, aggregation,
- * clocks or power modes, clear a request (UTRLCLR), or touch the task management list. It reads
- * the registers that say where UEFI's request list is and whether the controller runs, writes
- * request descriptors into slots of that list that UEFI does not use, rings their doorbell bits
- * (UTRLDBR is write-one-to-set: the other bits are not affected), and polls UTRLDBR until the
- * controller clears them. Requests are made with the interrupt bit clear, so that they raise no
- * completion status of their own in IS.
+ * Nothing else of the controller is changed: no reset, no link startup, no UIC command, no
+ * interrupt enable, aggregation, clock or power setting; the task management list is not
+ * touched; IS and UTRLCNR are only read. UTRLCLR is written only to clear this kernel's own
+ * slots when they do not finish at the end.
*
- * Read only by construction: every command passes through ufshc_permitted() on its way to the
- * doorbell, a whitelist that lets through SCSI READ(10)/READ(16), TEST UNIT READY, REQUEST
- * SENSE, INQUIRY, READ CAPACITY(10/16) to a normal logical unit and query *read* requests (read
- * descriptor, attribute, flag), and nothing else; no data ever goes from host to device. What is
- * refused is logged and never reaches the doorbell. This file builds READ(10) only.
+ * Read only by construction: a command is made in a buffer of its own, and only one that passes
+ * ufshc_permitted() is ever written into a slot; what the slot then holds is read back and must
+ * be the same and pass again before the doorbell is rung. The whitelist checks three things of
+ * every command: the SCSI operation (READ(10)/READ(16), TEST UNIT READY, REQUEST SENSE, INQUIRY,
+ * READ CAPACITY(10/16), nothing else -- no query, no task management, no NOP), the data
+ * direction in the request descriptor (from the device or none, never to it), and the command
+ * UPIU's flags (never W); and the logical unit: never a well-known one (REPORT LUNS, BOOT, RPMB,
+ * the device), and only those reads are allowed to go to now (ufshc_set_luns: LUN 0, where the
+ * iOS volume is, unless a comparison with the loader's reads asks for more). A read's length in
+ * the CDB must be what the UPIU expects. It is tried at the probe on made-up commands that must
+ * pass or be refused. This file builds READ(10) and TEST UNIT READY only.
*
- * WHAT THE PROBE FOUND ON THIS PHONE, AND WHY THIS PATH IS NOT USED THERE: Qualcomm's UFSDxe sets
- * up a list of one. The controller has 32 slots, UTRLBA points at 0xccf96000, slot 0 there is
- * UEFI's request descriptor -- and its command descriptor is at 0xccf96100, inside the 1 KB the
- * other 31 would take: "slots" 8..14 are its command UPIU, response and PRD table (pointing into
- * the loader's buffer), and 1..7 and 15..17 hold other data of the driver. No slot but UEFI's own
- * is free memory, so the probe refuses (twice over: the command descriptor inside the list, and
- * slots 16..31 that are not descriptors) and the storage is read through the loader. What
- * follows is how the path works where the list does have room.
+ * Every slot of this list holds a whitelisted command at all times: TEST UNIT READY until it is
+ * used for a read. Slot 0 is kept for the proof that the controller does read this list: after
+ * every switch, before any read, slot 0's TEST UNIT READY is rung and must come back in this
+ * list (its status written here). Were the controller still reading UEFI's list, the doorbell
+ * would run UEFI's slot 0 instead -- so first UEFI's slot 0 must hold a read too (it does: the
+ * loader's last use of the storage before a run is reading back the log it wrote), and the
+ * proof costs nothing either way. Reads go in slots 1..31.
*
- * Which slots: UEFI's driver takes the first free slot for each request it makes, synchronously,
- * so it uses slot 0 (and during a run it reads only when the loader is asked to, and writes only
- * the log, after the run). This file keeps to the upper half of the list, 16..31, and makes its
- * requests in the even ones: a descriptor is 32 bytes and a cache line 64, and a line is only
- * ever written back here while its one request is not in flight, so nothing the controller
- * writes into a descriptor (its status) is overwritten by a stale copy of the line's other half.
+ * The doorbell is rung only with interrupts off, after looking that this kernel's list is the
+ * one in the controller and running, that the slot is free, and that the loader is not taking
+ * the controller back (it says so in its page, then waits before it does).
*
- * At the probe (ufshc_probe), anything unexpected leaves this path off and the loader's in use:
- * the controller not enabled and running, a version other than 2.x/3.x, fewer than 32 slots, no
- * 64-bit addressing, the list in memory this kernel manages, any of the slots 16..31 busy or
- * holding something that is not a request descriptor, a fatal error status. The slots' original
- * contents are kept and put back at the end of the run (redwood_ufs_quiesce, from
- * loader_return), after waiting until none of them is busy, so that the list is as UEFI left it
- * when it writes the run's log.
- *
- * Memory: each slot has a page for its command descriptor (command UPIU at 0, response UPIU at
- * 512, the PRD table at 1024) and a contiguous buffer the controller reads into; the descriptors
- * are cleaned to the point of coherency before the doorbell and invalidated before they are
- * read back, and so is the data, so it does not matter whether the controller's accesses are
- * coherent with the caches.
+ * The list and the command descriptors are mapped here without caching (nothing of them is ever
+ * in a cache line of this kernel's to be written back over what the controller writes); the data
+ * buffers are this kernel's memory, cleaned and invalidated around every read.
*/
#include <kern/clock.h>
#include <kern/locks.h>
#include <kern/thread.h>
#include <libkern/libkern.h>
+#include <stdarg.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <pexpert/pexpert.h>
@@ -67,14 +62,15 @@
#include <sys/sysctl.h>
#include <vm/vm_kern.h>
#include <vm/memory_types.h>
+#include "redwood_ufshc.h"
extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
extern uint64_t kvtophys(vm_offset_t va); /* pmap_paddr_t */
extern boolean_t pmap_valid_address(uint64_t addr);
extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length);
-extern void CleanPoC_DcacheRegion(vm_offset_t va, size_t length);
extern void IOSleep(unsigned int milliseconds);
+extern boolean_t ml_set_interrupts_enabled(boolean_t enable);
#define UFSHC_PHYS 0x1d84000ULL /* ufshc@1d84000, "ufs_mem", in the phone's Android device tree */
#define UFSHC_SIZE 0x3000
@@ -93,6 +89,7 @@ extern void IOSleep(unsigned int milliseconds);
#define UFSHC_UTRLBA 0x50
#define UFSHC_UTRLBAU 0x54
#define UFSHC_UTRLDBR 0x58
+#define UFSHC_UTRLCLR 0x5c
#define UFSHC_UTRLRSR 0x60
#define UFSHC_UTRLCNR 0x64
#define UFSHC_UTMRLBA 0x70
@@ -105,9 +102,9 @@ extern void IOSleep(unsigned int milliseconds);
#define HCS_DP (1u << 0)
#define HCS_UTRLRDY (1u << 1)
#define IS_FATAL ((1u << 2) | (1u << 11) | (1u << 12) | (1u << 16) | (1u << 17)) /* UE, DFES, UTPES, HCFES, SBFES */
-#define IS_UTRCS (1u << 0)
-/* the request descriptor's first word: UFS storage command type, data direction */
+/* the request descriptor's first word: command type, data direction */
+#define UTRD_CT_MASK 0xf0000000u
#define UTRD_CT_UFS (1u << 28)
#define UTRD_DD_MASK (3u << 25)
#define UTRD_DD_NONE (0u << 25)
@@ -117,44 +114,53 @@ extern void IOSleep(unsigned int milliseconds);
#define OCS_INVALID 0xf
#define UPIU_COMMAND 0x01
-#define UPIU_QUERY_REQUEST 0x16
#define UPIU_RESPONSE 0x21
#define UPIU_FLAG_READ 0x40
#define UPIU_FLAG_WRITE 0x20
+#define LUN_WELL_KNOWN 0x80
-#define UCD_SIZE 0x4000 /* a page per slot */
+#define SLOTS 32 /* this kernel's list */
+#define SLOT_PROOF 0
#define UCD_RESPONSE 512
#define UCD_PRDT 1024
#define PRD_MAX (256u << 10) /* the most one table entry describes */
-
-#define SLOT_FIRST 16 /* this kernel's part of the list: 16..31 */
-#define SLOTS_RESERVED 0xffff0000u
-#define SLOT_MAX 8
-#define SLOT_BUFFER_DEFAULT (512u << 10)
+#define SLOT_BUFFER_DEFAULT (256u << 10)
#define BLOCK 4096u /* the logical block size of every unit here */
#define SPIN_US 5000 /* look without sleeping this long, then a millisecond at a time */
#define TIMEOUT_MS 10000
-struct ufshc_slot {
- uint32_t number; /* in the list */
- volatile uint32_t *utrd; /* its descriptor, in UEFI's list */
- uint8_t *ucd;
+struct hc_cmd {
+ uint8_t upiu[32];
+ uint32_t dw0;
+};
+
+struct hc_slot {
+ uint32_t number; /* in this kernel's list */
+ volatile uint32_t *utrd; /* its descriptor (the loader's memory, not cached here) */
+ volatile uint32_t *ucd; /* its command descriptor (likewise) */
uint64_t ucd_pa;
- uint8_t *data;
+ uint8_t *data; /* this kernel's memory, cached */
uint64_t data_pa;
- uint32_t original[8]; /* what the descriptor held at the probe, put back at the end */
};
static vm_offset_t hc; /* the registers */
-static volatile uint32_t *hc_list; /* UEFI's request list, cacheable */
-static uint64_t hc_list_pa;
-static struct ufshc_slot hc_slots[SLOT_MAX];
-static uint32_t hc_nslots, hc_free, hc_slot_buffer = SLOT_BUFFER_DEFAULT;
-static uint32_t hc_is_at_probe;
-static bool hc_usable; /* the probe found it as expected and the memory is there */
-static volatile bool hc_closed, hc_dead;
-static uint64_t hc_requests, hc_errors, hc_bytes, hc_wait_us, hc_refused, hc_most_in_flight;
+static struct ufshc_loader hc_loader;
+static uint32_t hc_list_lo, hc_list_hi; /* this kernel's list's address */
+static volatile uint32_t *hc_uefi_list; /* UEFI's list, not cached here, only read */
+static uint64_t hc_uefi_map_pa, hc_uefi_map_size;
+static vm_offset_t hc_uefi_map_va;
+static struct hc_slot hc_slots[SLOTS];
+static uint32_t hc_nslots; /* slots 1..hc_nslots carry reads */
+static uint32_t hc_free; /* bit i: hc_slots[i] free */
+static uint32_t hc_slot_buffer = SLOT_BUFFER_DEFAULT;
+static uint32_t hc_lun_mask = 1; /* the logical units reads may go to: LUN 0 */
+static bool hc_usable;
+static uint32_t hc_leave; /* ufs_native_leave=1 (a test): the list left in at the end, for the loader */
+static volatile bool hc_in, hc_closed, hc_dead;
+static uint32_t hc_is_in, hc_cnr_in; /* IS and UTRLCNR when this kernel's list went in */
+static uint64_t hc_requests, hc_errors, hc_bytes, hc_wait_us, hc_refused, hc_most_in_flight, hc_proofs;
+static uint64_t hc_switch_in_count, hc_switch_out_count;
static uint32_t hc_in_flight;
static LCK_GRP_DECLARE(hc_lck_grp, "redwood-ufshc");
@@ -162,15 +168,59 @@ static LCK_MTX_DECLARE(hc_lock, &hc_lck_grp);
SYSCTL_DECL(_kern_redwood);
SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_requests,
- "requests this kernel's own UFS driver has had answered");
+ "reads this kernel's own UFS driver has had answered");
SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_errors,
- "requests of this kernel's own UFS driver that failed");
+ "reads of this kernel's own UFS driver that failed");
SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_bytes,
"bytes this kernel's own UFS driver has read");
SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_wait_us,
- "microseconds spent waiting for the controller, summed over the requests");
+ "microseconds spent waiting for the controller, summed over the reads");
SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_refused, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_refused,
"commands the whitelist refused (there should never be one)");
+SYSCTL_QUAD(_kern_redwood, OID_AUTO, ufs_native_most_in_flight, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_most_in_flight,
+ "the most reads this kernel's own UFS driver has had in flight at once");
+
+/*
+ * What this file says goes to the kernel's log, and the last lines are kept: the start of a long
+ * run's log does not come back to the loader (the buffer wraps), so they are said again at the
+ * end, into the loader's log directly (redwood_ufs_quiesce; printf is not for that moment).
+ */
+extern void loader_log_putc(char c);
+#define KEPT_LINES 64
+static char hc_kept[KEPT_LINES][208];
+static uint32_t hc_kept_count;
+
+static void
+hc_say(const char *format, ...) __printflike(1, 2);
+static void
+hc_say(const char *format, ...)
+{
+ char line[208];
+ va_list list;
+
+ va_start(list, format);
+ vsnprintf(line, sizeof(line), format, list);
+ va_end(list);
+ printf("%s", line);
+ strlcpy(hc_kept[hc_kept_count++ % KEPT_LINES], line, sizeof(hc_kept[0]));
+}
+
+/* the same, at the end: into the loader's log only */
+static void
+hc_say_late(const char *format, ...) __printflike(1, 2);
+static void
+hc_say_late(const char *format, ...)
+{
+ char line[208];
+ va_list list;
+
+ va_start(list, format);
+ vsnprintf(line, sizeof(line), format, list);
+ va_end(list);
+ for (const char *c = line; *c != 0; c++) {
+ loader_log_putc(*c);
+ }
+}
static inline uint32_t
hc_read(uint32_t reg)
@@ -178,14 +228,12 @@ hc_read(uint32_t reg)
return *(volatile uint32_t *)(hc + reg);
}
-/* Nothing more is sent from here on; whoever waits for a slot is told. */
-static void
-hc_off(void)
+static inline void
+hc_write(uint32_t reg, uint32_t value)
{
- hc_dead = true;
- lck_mtx_lock(&hc_lock);
- wakeup(&hc_free);
- lck_mtx_unlock(&hc_lock);
+ __asm__ volatile ("dsb sy" ::: "memory");
+ *(volatile uint32_t *)(hc + reg) = value;
+ __asm__ volatile ("dsb sy" ::: "memory");
}
static uint64_t
@@ -197,109 +245,167 @@ hc_us_since(uint64_t start)
return ns / 1000;
}
-/* ---- the whitelist: the one check every command passes on its way to the doorbell ---- */
+/* ---- the whitelist: the one check every command passes before it goes into a slot ---- */
static const char *
-hc_refusal(const uint8_t *upiu, uint32_t dw0)
+hc_refusal(const uint8_t *upiu, uint32_t dw0, uint32_t lun_mask, bool any_command_type)
{
const uint8_t *cdb = upiu + 16;
uint32_t dd = dw0 & UTRD_DD_MASK;
+ uint64_t expected = (uint64_t)upiu[12] << 24 | (uint64_t)upiu[13] << 16 | (uint64_t)upiu[14] << 8 | upiu[15];
+ uint8_t lun = upiu[2];
+ /* the data direction: from the device, or none -- never to it */
if (dd == UTRD_DD_OUT || dd == UTRD_DD_MASK) {
return "data from host to device";
}
- if ((dw0 & 0xf0000000u) != UTRD_CT_UFS) {
+ if (!any_command_type && (dw0 & UTRD_CT_MASK) != UTRD_CT_UFS) {
return "not a UFS storage command type";
}
- switch (upiu[0] & 0x3f) {
- case UPIU_COMMAND:
- if (upiu[1] & UPIU_FLAG_WRITE) {
- return "a command that writes";
- }
- if (upiu[2] >= 8) {
- return "a well-known logical unit";
- }
- if (upiu[4] & 0x0f) {
- return "not the SCSI command set";
- }
- switch (cdb[0]) {
- case 0x28: /* READ(10) */
- case 0x88: /* READ(16) */
- return dd == UTRD_DD_IN && (upiu[1] & UPIU_FLAG_READ) ? NULL : "a read without data in";
- case 0x00: /* TEST UNIT READY */
- return dd == UTRD_DD_NONE ? NULL : "TEST UNIT READY with data";
- case 0x03: /* REQUEST SENSE */
- case 0x12: /* INQUIRY */
- case 0x25: /* READ CAPACITY(10) */
- return NULL;
- case 0x9e: /* SERVICE ACTION IN(16): READ CAPACITY(16) only */
- return (cdb[1] & 0x1f) == 0x10 ? NULL : "a SERVICE ACTION IN other than READ CAPACITY(16)";
- default:
- return "a SCSI operation not on the list";
- }
- case UPIU_QUERY_REQUEST:
- if (upiu[5] != 0x01) {
- return "a query that is not a standard read request";
+ /* a SCSI command, and its flags: never W */
+ if ((upiu[0] & 0x3f) != UPIU_COMMAND) {
+ return "not a SCSI command (a query, task management, NOP, ...)";
+ }
+ if (upiu[1] & UPIU_FLAG_WRITE) {
+ return "the W flag";
+ }
+ if (upiu[4] & 0x0f) {
+ return "not the SCSI command set";
+ }
+ /* a normal logical unit, and one reads may go to now */
+ if (lun & LUN_WELL_KNOWN) {
+ return "a well-known logical unit";
+ }
+ if (lun >= 8 || (lun_mask & (1u << lun)) == 0) {
+ return "a logical unit reads do not go to now";
+ }
+ /* the operation */
+ switch (cdb[0]) {
+ case 0x28: /* READ(10) */
+ case 0x88: { /* READ(16) */
+ uint64_t blocks = cdb[0] == 0x28 ? ((uint64_t)cdb[7] << 8 | cdb[8]) :
+ ((uint64_t)cdb[10] << 24 | (uint64_t)cdb[11] << 16 | (uint64_t)cdb[12] << 8 | cdb[13]);
+
+ if (dd != UTRD_DD_IN || (upiu[1] & UPIU_FLAG_READ) == 0) {
+ return "a read without data from the device";
}
- if (upiu[12] != 0x01 && upiu[12] != 0x03 && upiu[12] != 0x05) {
- return "a query opcode other than read descriptor/attribute/flag";
+ if (blocks == 0 || expected != blocks * BLOCK) {
+ return "a read whose length is not what the UPIU expects";
}
- return dd == UTRD_DD_NONE ? NULL : "a query with data";
+ return NULL;
+ }
+ case 0x00: /* TEST UNIT READY */
+ return dd == UTRD_DD_NONE && expected == 0 && (upiu[1] & UPIU_FLAG_READ) == 0 ? NULL :
+ "TEST UNIT READY with data";
+ case 0x03: /* REQUEST SENSE */
+ case 0x12: /* INQUIRY */
+ case 0x25: /* READ CAPACITY(10) */
+ return NULL;
+ case 0x9e: /* SERVICE ACTION IN(16): READ CAPACITY(16) only */
+ return (cdb[1] & 0x1f) == 0x10 ? NULL : "a SERVICE ACTION IN other than READ CAPACITY(16)";
default:
- return "a transaction that is neither a command nor a query";
+ return "a SCSI operation not on the list";
}
}
static bool
ufshc_permitted(const uint8_t *upiu, uint32_t dw0)
{
- const char *why = hc_refusal(upiu, dw0);
+ const char *why = hc_refusal(upiu, dw0, hc_lun_mask, false);
if (why == NULL) {
return true;
}
hc_refused++;
- printf("ufs: native: REFUSED %s: UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x, "
- "descriptor %08x; nothing was sent\n", why, upiu[0], upiu[1], upiu[2], upiu[3], upiu[16], upiu[17], upiu[18],
+ hc_say("ufs: native: REFUSED (%s): UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x, "
+ "descriptor %08x; it goes into no slot\n", why, upiu[0], upiu[1], upiu[2], upiu[3], upiu[16], upiu[17], upiu[18],
upiu[19], upiu[20], upiu[21], upiu[22], upiu[23], upiu[24], upiu[25], dw0);
return false;
}
+static void
+hc_make_read(struct hc_cmd *c, uint8_t lun, uint8_t tag, uint32_t lba, uint32_t blocks)
+{
+ uint32_t bytes = blocks * BLOCK;
+
+ bzero(c, sizeof(*c));
+ c->upiu[0] = UPIU_COMMAND;
+ c->upiu[1] = UPIU_FLAG_READ; /* simple task attribute */
+ c->upiu[2] = lun;
+ c->upiu[3] = tag;
+ c->upiu[12] = (uint8_t)(bytes >> 24);
+ c->upiu[13] = (uint8_t)(bytes >> 16);
+ c->upiu[14] = (uint8_t)(bytes >> 8);
+ c->upiu[15] = (uint8_t)bytes;
+ c->upiu[16] = 0x28; /* READ(10) */
+ c->upiu[18] = (uint8_t)(lba >> 24);
+ c->upiu[19] = (uint8_t)(lba >> 16);
+ c->upiu[20] = (uint8_t)(lba >> 8);
+ c->upiu[21] = (uint8_t)lba;
+ c->upiu[23] = (uint8_t)(blocks >> 8);
+ c->upiu[24] = (uint8_t)blocks;
+ c->dw0 = UTRD_CT_UFS | UTRD_DD_IN; /* no interrupt, no inline encryption */
+}
+
+static void
+hc_make_tur(struct hc_cmd *c, uint8_t lun, uint8_t tag)
+{
+ bzero(c, sizeof(*c));
+ c->upiu[0] = UPIU_COMMAND;
+ c->upiu[2] = lun;
+ c->upiu[3] = tag;
+ c->upiu[16] = 0x00; /* TEST UNIT READY */
+ c->dw0 = UTRD_CT_UFS | UTRD_DD_NONE;
+}
+
/*
* The whitelist tried on commands made up here and never sent: those that must be refused,
- * and the read that must pass. Pure bytes; nothing touches the controller.
+ * and those that must pass. Pure bytes; nothing touches the controller.
*/
static bool
hc_whitelist_selftest(void)
{
static const struct {
- uint8_t type, flags, lun, cdb0, cdb1, query_function, query_opcode;
+ uint8_t type, flags, lun, op, cdb1;
+ uint16_t blocks; /* the CDB's length (READ(10)/(16)) */
+ uint32_t expected; /* the UPIU's */
uint32_t dd;
bool pass;
const char *what;
} cases[] = {
- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 0, 0, UTRD_DD_IN, true, "READ(10)" },
- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x88, 0, 0, 0, UTRD_DD_IN, true, "READ(16)" },
- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x25, 0, 0, 0, UTRD_DD_IN, true, "READ CAPACITY(10)" },
- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x01, 0x01, UTRD_DD_NONE, true, "QUERY READ DESCRIPTOR" },
- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x2a, 0, 0, 0, UTRD_DD_OUT, false, "WRITE(10)" },
- { UPIU_COMMAND, 0, 0, 0x2a, 0, 0, 0, UTRD_DD_NONE, false, "WRITE(10) without the W flag" },
- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x2a, 0, 0, 0, UTRD_DD_IN, false, "WRITE(10) dressed as a read" },
- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x8a, 0, 0, 0, UTRD_DD_OUT, false, "WRITE(16)" },
- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x42, 0, 0, 0, UTRD_DD_OUT, false, "UNMAP" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, true, "READ(10) of LUN 0" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x88, 0, 1, 4096, UTRD_DD_IN, true, "READ(16) of LUN 0" },
+ { UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_NONE, true, "TEST UNIT READY of LUN 0" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x25, 0, 0, 8, UTRD_DD_IN, true, "READ CAPACITY(10)" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x2a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(10)" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x8a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(16)" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x2a, 0, 1, 4096, UTRD_DD_IN, false, "WRITE(10) dressed as a read" },
+ { UPIU_COMMAND, 0, 0, 0x2a, 0, 0, 0, UTRD_DD_NONE, false, "WRITE(10) with no data" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x42, 0, 0, 24, UTRD_DD_OUT, false, "UNMAP" },
+ { UPIU_COMMAND, 0, 0, 0x42, 0, 0, 0, UTRD_DD_NONE, false, "UNMAP with no data" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_OUT, false, "READ(10) with data host to device" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_MASK, false, "READ(10) with direction 3" },
+ { UPIU_COMMAND, UPIU_FLAG_READ | UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with the W flag" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with only the W flag" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 8192, UTRD_DD_IN, false, "READ(10) of a length other than expected" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xc4, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the RPMB unit" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xb0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the BOOT unit" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0xd0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the device unit" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0x81, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of REPORT LUNS" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 3, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of LUN 3 (not allowed now)" },
{ UPIU_COMMAND, 0, 0, 0x04, 0, 0, 0, UTRD_DD_NONE, false, "FORMAT UNIT" },
{ UPIU_COMMAND, 0, 0, 0x1b, 0, 0, 0, UTRD_DD_NONE, false, "START STOP UNIT" },
{ UPIU_COMMAND, 0, 0, 0x35, 0, 0, 0, UTRD_DD_NONE, false, "SYNCHRONIZE CACHE(10)" },
- { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x3b, 0, 0, 0, UTRD_DD_OUT, false, "WRITE BUFFER" },
- { UPIU_COMMAND, UPIU_FLAG_READ, 0x44, 0x28, 0, 0, 0, UTRD_DD_IN, false, "READ(10) to the RPMB unit" },
- { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x9e, 0x11, 0, 0, UTRD_DD_IN, false, "SERVICE ACTION IN 0x11" },
- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x81, 0x02, UTRD_DD_NONE, false, "QUERY WRITE DESCRIPTOR" },
- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x81, 0x06, UTRD_DD_NONE, false, "QUERY SET FLAG" },
- { UPIU_QUERY_REQUEST, 0, 0, 0, 0, 0x01, 0x04, UTRD_DD_NONE, false, "QUERY WRITE ATTRIBUTE as a read" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x3b, 0, 0, 4096, UTRD_DD_OUT, false, "WRITE BUFFER" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0xb5, 0, 0, 512, UTRD_DD_OUT, false, "SECURITY PROTOCOL OUT" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0xa2, 0, 0, 512, UTRD_DD_IN, false, "SECURITY PROTOCOL IN" },
+ { UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x15, 0, 0, 24, UTRD_DD_OUT, false, "MODE SELECT" },
+ { UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x9e, 0x11, 0, 32, UTRD_DD_IN, false, "SERVICE ACTION IN 0x11" },
+ { UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_IN, false, "TEST UNIT READY with data in" },
+ { 0x16, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "a QUERY REQUEST" },
{ 0x04, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "TASK MANAGEMENT" },
{ 0x00, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "NOP OUT" },
};
- uint64_t refused = hc_refused;
bool right = true;
for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
@@ -308,53 +414,77 @@ hc_whitelist_selftest(void)
upiu[0] = cases[i].type;
upiu[1] = cases[i].flags;
- upiu[2] = cases[i].lun == 0x44 ? 0xc4 : cases[i].lun;
- upiu[5] = cases[i].query_function;
- upiu[12] = cases[i].query_opcode;
- upiu[16] = cases[i].cdb0;
+ upiu[2] = cases[i].lun;
+ upiu[12] = (uint8_t)(cases[i].expected >> 24);
+ upiu[13] = (uint8_t)(cases[i].expected >> 16);
+ upiu[14] = (uint8_t)(cases[i].expected >> 8);
+ upiu[15] = (uint8_t)cases[i].expected;
+ upiu[16] = cases[i].op;
upiu[17] = cases[i].cdb1;
- passed = hc_refusal(upiu, UTRD_CT_UFS | cases[i].dd) == NULL;
+ if (cases[i].op == 0x88) {
+ upiu[16 + 12] = (uint8_t)(cases[i].blocks >> 8);
+ upiu[16 + 13] = (uint8_t)cases[i].blocks;
+ } else {
+ upiu[16 + 7] = (uint8_t)(cases[i].blocks >> 8);
+ upiu[16 + 8] = (uint8_t)cases[i].blocks;
+ }
+ passed = hc_refusal(upiu, UTRD_CT_UFS | cases[i].dd, 1u << 0, false) == NULL;
if (passed != cases[i].pass) {
- printf("ufs: native: the whitelist %s %s\n", passed ? "LETS THROUGH" : "refuses", cases[i].what);
+ hc_say("ufs: native: the whitelist %s %s\n", passed ? "LETS THROUGH" : "refuses", cases[i].what);
right = false;
}
}
- hc_refused = refused;
- printf("ufs: native: whitelist tried on %u commands made up and never sent (READ(10/16), READ CAPACITY, "
- "QUERY READ DESCRIPTOR pass; WRITE(10/16), UNMAP, FORMAT UNIT, START STOP UNIT, SYNCHRONIZE CACHE, "
- "WRITE BUFFER, RPMB, query writes and set flag, task management, NOP OUT refused): %s\n",
- (unsigned)(sizeof(cases) / sizeof(cases[0])), right ? "as it should" : "WRONG");
+ /* and what this file makes */
+ {
+ struct hc_cmd c;
+
+ hc_make_read(&c, 0, 1, 12345, 64);
+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;
+ hc_make_read(&c, 2, 1, 12345, 64);
+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) != NULL;
+ hc_make_tur(&c, 0, 0);
+ right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;
+ }
+ hc_say("ufs: native: whitelist tried on %u commands made up and never sent -- READ(10/16), TEST UNIT READY, READ "
+ "CAPACITY of LUN 0 pass; WRITE(10/16), UNMAP, a READ with data host to device, a READ with the W flag, the "
+ "RPMB/BOOT/device/REPORT LUNS units, LUN 3, FORMAT, START STOP, SYNCHRONIZE CACHE, WRITE BUFFER, SECURITY "
+ "PROTOCOL, MODE SELECT, query, task management, NOP refused: %s\n",
+ (unsigned)(sizeof(cases) / sizeof(cases[0])) + 3, right ? "as it should" : "WRONG");
return right;
}
-/* ---- one request in one slot ---- */
+/* ---- one command in one slot ---- */
+static void
+hc_put(volatile uint32_t *to, const void *from, uint32_t bytes)
+{
+ for (uint32_t i = 0; i < bytes / 4; i++) {
+ uint32_t w;
+
+ memcpy(&w, (const uint8_t *)from + 4 * i, 4);
+ to[i] = w;
+ }
+}
+
+/*
+ * Into the slot: the command UPIU, an empty response, the PRD table for bytes of the slot's
+ * buffer, the descriptor -- only a command the whitelist passes, and only if what the slot then
+ * holds is the same and passes again.
+ */
static int
-hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)
+hc_place(struct hc_slot *s, const struct hc_cmd *c, uint32_t bytes)
{
- uint32_t bytes = blocks * BLOCK, n = 0, dw0 = UTRD_CT_UFS | UTRD_DD_IN;
- uint8_t *cmd = s->ucd;
-
- /* the command UPIU and an empty response */
- bzero(s->ucd, UCD_PRDT);
- cmd[0] = UPIU_COMMAND;
- cmd[1] = UPIU_FLAG_READ; /* simple task attribute */
- cmd[2] = lun;
- cmd[3] = (uint8_t)s->number; /* task tag */
- cmd[12] = (uint8_t)(bytes >> 24);
- cmd[13] = (uint8_t)(bytes >> 16);
- cmd[14] = (uint8_t)(bytes >> 8);
- cmd[15] = (uint8_t)bytes;
- cmd[16] = 0x28; /* READ(10) */
- cmd[18] = (uint8_t)(lba >> 24);
- cmd[19] = (uint8_t)(lba >> 16);
- cmd[20] = (uint8_t)(lba >> 8);
- cmd[21] = (uint8_t)lba;
- cmd[23] = (uint8_t)(blocks >> 8);
- cmd[24] = (uint8_t)blocks;
- /* the data's place, in pieces of at most 256 KB */
+ uint32_t n = 0, back[8];
+
+ if (!ufshc_permitted(c->upiu, c->dw0)) {
+ return EPERM;
+ }
+ hc_put(s->ucd, c->upiu, sizeof(c->upiu));
+ for (uint32_t i = sizeof(c->upiu) / 4; i < UCD_PRDT / 4; i++) {
+ s->ucd[i] = 0;
+ }
for (uint32_t at = 0; at < bytes; at += PRD_MAX, n++) {
- volatile uint32_t *prd = (volatile uint32_t *)(s->ucd + UCD_PRDT + 16 * n);
+ volatile uint32_t *prd = s->ucd + (UCD_PRDT + 16 * n) / 4;
uint64_t pa = s->data_pa + at;
uint32_t len = bytes - at < PRD_MAX ? bytes - at : PRD_MAX;
@@ -363,11 +493,7 @@ hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)
prd[2] = 0;
prd[3] = len - 1;
}
- FlushPoC_DcacheRegion((vm_offset_t)s->ucd, UCD_PRDT + 16 * n);
- /* nothing of the buffer is left in a cache to be written back over what the controller writes */
- FlushPoC_DcacheRegion((vm_offset_t)s->data, bytes);
-
- s->utrd[0] = dw0; /* no interrupt, no inline encryption */
+ s->utrd[0] = c->dw0;
s->utrd[1] = 0;
s->utrd[2] = OCS_INVALID;
s->utrd[3] = 0;
@@ -375,31 +501,56 @@ hc_submit(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)
s->utrd[5] = (uint32_t)(s->ucd_pa >> 32);
s->utrd[6] = ((UCD_RESPONSE / 4) << 16) | (UCD_RESPONSE / 4); /* response: offset, length in dwords */
s->utrd[7] = ((UCD_PRDT / 4) << 16) | n; /* PRD table: offset in dwords, entries */
- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);
-
- /* what the controller will read is what the whitelist sees */
- if (!ufshc_permitted(s->ucd, s->utrd[0])) {
- return EPERM;
- }
- if (hc_closed || hc_dead) {
- return ENXIO;
+ __asm__ volatile ("dsb sy" ::: "memory");
+ for (int i = 0; i < 8; i++) {
+ back[i] = s->ucd[i];
}
- if (hc_read(UFSHC_UTRLDBR) & (1u << s->number)) {
- printf("ufs: native: slot %u is busy before it is used: this path is off from now on\n", s->number);
- hc_off();
- return EIO;
+ if (memcmp(back, c->upiu, sizeof(back)) != 0 || s->utrd[0] != c->dw0 ||
+ !ufshc_permitted((const uint8_t *)back, s->utrd[0])) {
+ hc_say("ufs: native: slot %u does not hold what was put in it: not rung\n", s->number);
+ hc_refused++;
+ return EPERM;
}
- __asm__ volatile ("dsb sy" ::: "memory");
- *(volatile uint32_t *)(hc + UFSHC_UTRLDBR) = 1u << s->number;
return 0;
}
-/* Until the controller is done with the slot's request; then whether it went well. */
+/* The doorbell: only while this kernel's list is the one in, running, and the loader not taking it back */
static int
-hc_complete(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)
+hc_ring(struct hc_slot *s)
+{
+ uint32_t bit = 1u << s->number;
+ boolean_t istate;
+ bool ok;
+
+ lck_mtx_lock(&hc_lock);
+ istate = ml_set_interrupts_enabled(FALSE);
+ ok = hc_in && !hc_closed && !hc_dead && *hc_loader.take_back == 0 &&
+ hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi &&
+ (hc_read(UFSHC_UTRLRSR) & 1) != 0 && (hc_read(UFSHC_UTRLDBR) & bit) == 0;
+ if (ok) {
+ hc_write(UFSHC_UTRLDBR, bit);
+ }
+ ml_set_interrupts_enabled(istate);
+ lck_mtx_unlock(&hc_lock);
+ return ok ? 0 : ENXIO;
+}
+
+/* Nothing more is sent from here on; whoever waits for a slot is told. */
+static void
+hc_off(void)
+{
+ hc_dead = true;
+ lck_mtx_lock(&hc_lock);
+ wakeup(&hc_free);
+ lck_mtx_unlock(&hc_lock);
+}
+
+/* Until the controller is done with the slot's command; then whether it went well. */
+static int
+hc_complete(struct hc_slot *s, const char *what, bool tur)
{
uint64_t started = mach_absolute_time(), us;
- const uint8_t *r = s->ucd + UCD_RESPONSE;
+ volatile const uint8_t *r = (volatile const uint8_t *)s->ucd + UCD_RESPONSE;
uint32_t ocs;
while (hc_read(UFSHC_UTRLDBR) & (1u << s->number)) {
@@ -409,44 +560,43 @@ hc_complete(struct ufshc_slot *s, uint8_t lun, uint32_t lba, uint32_t blocks)
continue;
}
if (us > TIMEOUT_MS * 1000ULL) {
- printf("ufs: native: slot %u (LUN %u, blocks %u..%u) not done in %u s (doorbell %08x, IS %08x, HCS %08x): "
- "this path is off from now on\n", s->number, lun, lba, lba + blocks - 1, TIMEOUT_MS / 1000,
- hc_read(UFSHC_UTRLDBR), hc_read(UFSHC_IS), hc_read(UFSHC_HCS));
+ hc_say("ufs: native: slot %u (%s) not done in %u s (doorbell %08x, IS %08x, HCS %08x): "
+ "this path is off from now on\n", s->number, what, TIMEOUT_MS / 1000, hc_read(UFSHC_UTRLDBR),
+ hc_read(UFSHC_IS), hc_read(UFSHC_HCS));
hc_off();
return ETIMEDOUT;
}
IOSleep(1);
}
__asm__ volatile ("dsb sy" ::: "memory");
- hc_wait_us += hc_us_since(started);
- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);
- FlushPoC_DcacheRegion((vm_offset_t)r, 64);
+ if (!tur) {
+ hc_wait_us += hc_us_since(started);
+ }
ocs = s->utrd[2] & 0xff;
- if (ocs != OCS_SUCCESS || (r[0] & 0x3f) != UPIU_RESPONSE || r[3] != s->number || r[6] != 0 || r[7] != 0) {
- /* sense data follows the 32 bytes of the header: length (2), then the fixed format */
- FlushPoC_DcacheRegion((vm_offset_t)r + 64, 64);
- printf("ufs: native: slot %u, LUN %u, blocks %u..%u: OCS %x, response %02x tag %02x, response %02x status %02x, "
- "sense key %x asc %02x ascq %02x\n", s->number, lun, lba, lba + blocks - 1, ocs, r[0], r[3], r[6], r[7],
+ if (ocs != OCS_SUCCESS || (r[0] & 0x3f) != UPIU_RESPONSE || r[3] != s->number || r[6] != 0 ||
+ (r[7] != 0 && !(tur && r[7] == 0x02))) {
+ /* sense data follows the 32 bytes of the header: its length (2), then the fixed format */
+ hc_say("ufs: native: slot %u (%s): OCS %x, response %02x tag %02x, response %02x status %02x, "
+ "sense key %x asc %02x ascq %02x\n", s->number, what, ocs, r[0], r[3], r[6], r[7],
r[34 + 2] & 0x0f, r[34 + 12], r[34 + 13]);
- hc_errors++;
+ if (!tur) {
+ hc_errors++;
+ }
return EIO;
}
- FlushPoC_DcacheRegion((vm_offset_t)s->data, blocks * BLOCK);
- hc_requests++;
- hc_bytes += blocks * BLOCK;
return 0;
}
-static struct ufshc_slot *
+static struct hc_slot *
hc_take(bool wait)
{
- struct ufshc_slot *s = NULL;
+ struct hc_slot *s = NULL;
lck_mtx_lock(&hc_lock);
- while (hc_free == 0 && wait && !hc_dead) {
+ while (hc_free == 0 && wait && !hc_dead && hc_in) {
msleep(&hc_free, &hc_lock, PRIBIO, "ufshc", NULL);
}
- if (hc_free != 0) {
+ if (hc_free != 0 && hc_in && !hc_dead) {
uint32_t i = (uint32_t)__builtin_ctz(hc_free);
hc_free &= ~(1u << i);
@@ -460,7 +610,7 @@ hc_take(bool wait)
}
static void
-hc_give(struct ufshc_slot *s)
+hc_give(struct hc_slot *s)
{
lck_mtx_lock(&hc_lock);
hc_free |= 1u << (s - hc_slots);
@@ -478,27 +628,28 @@ int
ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)
{
struct {
- struct ufshc_slot *slot;
+ struct hc_slot *slot;
uint32_t lba, blocks, skip, take;
uint8_t *to;
- } q[SLOT_MAX];
+ } q[SLOTS];
uint32_t head = 0, tail = 0, per = hc_slot_buffer / BLOCK;
int error = 0;
- if (!hc_usable || hc_dead || hc_closed) {
+ if (!hc_usable || !hc_in || hc_dead || hc_closed) {
return ENXIO;
}
while ((len > 0 && error == 0) || head != tail) {
- /* as many more requests as there are free slots (and one, waiting for it, if none is in flight) */
- while (len > 0 && error == 0 && tail - head < SLOT_MAX) {
+ /* as many more reads as there are free slots (and one, waiting for it, if none is in flight) */
+ while (len > 0 && error == 0 && tail - head < SLOTS) {
uint64_t first = offset / BLOCK, skip = offset % BLOCK;
uint64_t blocks = (skip + len + BLOCK - 1) / BLOCK, take;
- struct ufshc_slot *s = hc_take(head == tail);
+ struct hc_slot *s = hc_take(head == tail);
+ struct hc_cmd c;
int e;
if (s == NULL) {
if (head == tail) {
- error = EIO; /* the path went off while waiting */
+ error = ENXIO; /* the path went off while waiting */
}
break;
}
@@ -514,18 +665,24 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)
if (take > len) {
take = len;
}
- e = hc_submit(s, lun, (uint32_t)first, (uint32_t)blocks);
+ hc_make_read(&c, lun, (uint8_t)s->number, (uint32_t)first, (uint32_t)blocks);
+ /* nothing of the buffer is left in a cache to be written back over what the controller writes */
+ FlushPoC_DcacheRegion((vm_offset_t)s->data, (size_t)blocks * BLOCK);
+ e = hc_place(s, &c, (uint32_t)blocks * BLOCK);
+ if (e == 0) {
+ e = hc_ring(s);
+ }
if (e != 0) {
hc_give(s);
error = e;
break;
}
- q[tail % SLOT_MAX].slot = s;
- q[tail % SLOT_MAX].lba = (uint32_t)first;
- q[tail % SLOT_MAX].blocks = (uint32_t)blocks;
- q[tail % SLOT_MAX].skip = (uint32_t)skip;
- q[tail % SLOT_MAX].take = (uint32_t)take;
- q[tail % SLOT_MAX].to = dst;
+ q[tail % SLOTS].slot = s;
+ q[tail % SLOTS].lba = (uint32_t)first;
+ q[tail % SLOTS].blocks = (uint32_t)blocks;
+ q[tail % SLOTS].skip = (uint32_t)skip;
+ q[tail % SLOTS].take = (uint32_t)take;
+ q[tail % SLOTS].to = dst;
tail++;
offset += take;
dst += take;
@@ -533,11 +690,19 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)
}
/* the oldest one done, copied out, its slot free again */
if (head != tail) {
- typeof(q[0]) *e = &q[head % SLOT_MAX];
- int r = hc_complete(e->slot, lun, e->lba, e->blocks);
+ typeof(q[0]) *e = &q[head % SLOTS];
+ char what[48];
+ int r;
- if (r == 0 && error == 0) {
- memcpy(e->to, e->slot->data + e->skip, e->take);
+ snprintf(what, sizeof(what), "LUN %u, blocks %u..%u", lun, e->lba, e->lba + e->blocks - 1);
+ r = hc_complete(e->slot, what, false);
+ if (r == 0) {
+ FlushPoC_DcacheRegion((vm_offset_t)e->slot->data, (size_t)e->blocks * BLOCK);
+ hc_requests++;
+ hc_bytes += (uint64_t)e->blocks * BLOCK;
+ if (error == 0) {
+ memcpy(e->to, e->slot->data + e->skip, e->take);
+ }
} else if (error == 0) {
error = r;
}
@@ -550,6 +715,253 @@ ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)
return error;
}
+/* ---- the proof, and the switches ---- */
+
+/* UEFI's slot 0 holds a read (what the controller would run were it to take no notice of this list) */
+static const char *
+hc_uefi_slot0(uint8_t *upiu, uint32_t *dw0)
+{
+ uint64_t ucd = (uint64_t)hc_uefi_list[5] << 32 | hc_uefi_list[4];
+ volatile const uint8_t *u;
+
+ *dw0 = hc_uefi_list[0];
+ if (ucd < hc_uefi_map_pa || ucd + 32 > hc_uefi_map_pa + hc_uefi_map_size) {
+ return "UEFI's command descriptor is not where it was";
+ }
+ u = (volatile const uint8_t *)(hc_uefi_map_va + (ucd - hc_uefi_map_pa));
+ for (int i = 0; i < 32; i++) {
+ upiu[i] = u[i];
+ }
+ return hc_refusal(upiu, *dw0, 0xff, true) == NULL ? NULL : "UEFI's slot 0 does not hold a read";
+}
+
+/* This kernel's slot 0: TEST UNIT READY rung, and its status must come back in this list */
+static int
+hc_prove(void)
+{
+ struct hc_slot *s = &hc_slots[SLOT_PROOF];
+ struct hc_cmd c;
+ uint32_t ocs;
+ int e;
+
+ hc_make_tur(&c, (uint8_t)__builtin_ctz(hc_lun_mask), SLOT_PROOF);
+ e = hc_place(s, &c, 0);
+ if (e == 0) {
+ e = hc_ring(s);
+ }
+ if (e == 0) {
+ e = hc_complete(s, "the proof's TEST UNIT READY", true);
+ }
+ ocs = s->utrd[2] & 0xff;
+ hc_proofs++;
+ if (e != 0 || ocs != OCS_SUCCESS) {
+ hc_say("ufs: native: proof FAILED (%d): this list's slot 0 has OCS %x after its doorbell -- the controller "
+ "did not run it from this list\n", e, ocs);
+ return e != 0 ? e : EIO;
+ }
+ hc_say("ufs: native: proof: TEST UNIT READY in this list's slot 0 came back here (OCS 0, response %02x, status %02x)\n",
+ ((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 6], ((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 7]);
+ return 0;
+}
+
+struct hc_put_back_result {
+ uint32_t db_before, db_after_clear, lo, hi, rsr, db, is, cnr;
+ bool cleared, ok;
+};
+
+/*
+ * UEFI's list back into the controller: what is in flight waited for (1 s), cleared if it does
+ * not finish (every slot of this list is this kernel's), the list stopped, UEFI's address
+ * written, the list started, read back. Interrupts off, no locks, no printing (the quiesce).
+ */
+static void
+hc_put_back(struct hc_put_back_result *r)
+{
+ uint64_t t;
+
+ bzero(r, sizeof(*r));
+ for (t = mach_absolute_time(); (r->db_before = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {
+ __builtin_arm_yield();
+ }
+ if (r->db_before != 0 && hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi) {
+ r->cleared = true;
+ hc_write(UFSHC_UTRLCLR, 0);
+ for (t = mach_absolute_time(); (r->db_after_clear = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {
+ __builtin_arm_yield();
+ }
+ }
+ hc_write(UFSHC_UTRLRSR, 0);
+ for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {
+ __builtin_arm_yield();
+ }
+ hc_write(UFSHC_UTRLBA, hc_loader.uefi_lo);
+ hc_write(UFSHC_UTRLBAU, hc_loader.uefi_hi);
+ hc_write(UFSHC_UTRLRSR, 1);
+ r->lo = hc_read(UFSHC_UTRLBA);
+ r->hi = hc_read(UFSHC_UTRLBAU);
+ r->rsr = hc_read(UFSHC_UTRLRSR);
+ r->db = hc_read(UFSHC_UTRLDBR);
+ r->is = hc_read(UFSHC_IS);
+ r->cnr = hc_read(UFSHC_UTRLCNR);
+ r->ok = r->lo == hc_loader.uefi_lo && r->hi == hc_loader.uefi_hi && (r->rsr & 1);
+ if (r->ok) {
+ *hc_loader.list_in = 0;
+ (*hc_loader.switched_out)++;
+ hc_switch_out_count++;
+ }
+}
+
+static void
+hc_put_back_print(const char *why, const struct hc_put_back_result *r, bool late)
+{
+ void (*say)(const char *, ...) = late ? hc_say_late : hc_say;
+
+ if (r->cleared) {
+ say("ufs: native: %s: doorbell %08x did not clear in 1 s: this kernel's slots cleared (UTRLCLR), doorbell "
+ "then %08x\n", why, r->db_before, r->db_after_clear);
+ }
+ say("ufs: native: %s: transfer list %s UEFI's %08x:%08x (run %u, doorbell %08x; IS %08x, at the switch %08x; "
+ "completion %08x, at the switch %08x)\n", why, r->ok ? "back to" : "NOT BACK TO", r->hi, r->lo, r->rsr, r->db,
+ r->is, hc_is_in, r->cnr, hc_cnr_in);
+}
+
+/*
+ * This kernel's list into the controller (the caller holds the loader's mailbox: nothing is asked
+ * of UEFI meanwhile). prove: then slot 0's TEST UNIT READY, before anything is read.
+ */
+int
+ufshc_switch_in(bool prove)
+{
+ uint32_t lo, hi, db, rsr, hcs, uefi_dw0 = 0;
+ uint8_t uefi_upiu[32] = { 0 };
+ const char *no = NULL;
+ struct hc_put_back_result undo;
+ boolean_t istate;
+ bool undone = false;
+
+ if (!hc_usable || hc_dead || hc_closed) {
+ return ENXIO;
+ }
+ lck_mtx_lock(&hc_lock);
+ if (hc_in) {
+ lck_mtx_unlock(&hc_lock);
+ return 0;
+ }
+ istate = ml_set_interrupts_enabled(FALSE);
+ if (*hc_loader.take_back != 0) {
+ no = "the loader has taken the controller back";
+ } else {
+ no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);
+ }
+ lo = hc_read(UFSHC_UTRLBA);
+ hi = hc_read(UFSHC_UTRLBAU);
+ db = hc_read(UFSHC_UTRLDBR);
+ rsr = hc_read(UFSHC_UTRLRSR);
+ hcs = hc_read(UFSHC_HCS);
+ if (no == NULL) {
+ if (lo != hc_loader.uefi_lo || hi != hc_loader.uefi_hi) {
+ no = "the controller's list is not UEFI's";
+ } else if (db != 0) {
+ no = "a request in flight";
+ } else if ((rsr & 1) == 0 || (hcs & HCS_UTRLRDY) == 0) {
+ no = "the list is not running, or not ready";
+ }
+ }
+ if (no == NULL) {
+ uint64_t t;
+
+ hc_is_in = hc_read(UFSHC_IS);
+ hc_cnr_in = hc_read(UFSHC_UTRLCNR);
+ hc_write(UFSHC_UTRLRSR, 0);
+ for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {
+ __builtin_arm_yield();
+ }
+ if (hc_read(UFSHC_UTRLRSR) & 1) {
+ no = "the list did not stop";
+ hc_write(UFSHC_UTRLRSR, 1); /* (UEFI's list, running, as it was) */
+ } else {
+ hc_write(UFSHC_UTRLBA, hc_list_lo);
+ hc_write(UFSHC_UTRLBAU, hc_list_hi);
+ hc_write(UFSHC_UTRLRSR, 1);
+ lo = hc_read(UFSHC_UTRLBA);
+ hi = hc_read(UFSHC_UTRLBAU);
+ rsr = hc_read(UFSHC_UTRLRSR);
+ if (lo != hc_list_lo || hi != hc_list_hi || (rsr & 1) == 0) {
+ no = "the controller did not take this list";
+ hc_put_back(&undo);
+ undone = true;
+ } else {
+ hc_in = true;
+ *hc_loader.list_in = 1;
+ (*hc_loader.switched_in)++;
+ hc_switch_in_count++;
+ }
+ }
+ }
+ ml_set_interrupts_enabled(istate);
+ lck_mtx_unlock(&hc_lock);
+ if (undone) {
+ hc_put_back_print("undoing", &undo, false);
+ }
+ if (no != NULL) {
+ hc_say("ufs: native: this kernel's list not put in: %s (list %08x:%08x, doorbell %08x, run %08x, HCS %08x; UEFI's "
+ "slot 0 %08x, CDB %02x)\n", no, hi, lo, db, rsr, hcs, uefi_dw0, uefi_upiu[16]);
+ return EIO;
+ }
+ hc_say("ufs: native: transfer list switched from UEFI's %08x:%08x to this kernel's %08x:%08x (doorbell 0, run %u; "
+ "IS %08x, completion %08x; UEFI's slot 0 holds a read: %08x, CDB %02x)\n", hc_loader.uefi_hi,
+ hc_loader.uefi_lo, hi, lo, rsr, hc_is_in, hc_cnr_in, uefi_dw0, uefi_upiu[16]);
+ if (prove) {
+ int e = hc_prove();
+
+ if (e != 0) {
+ ufshc_switch_out("after the failed proof");
+ hc_off();
+ return e;
+ }
+ }
+ return 0;
+}
+
+/* UEFI's list back (the caller holds the loader's mailbox) */
+int
+ufshc_switch_out(const char *why)
+{
+ struct hc_put_back_result r;
+ uint64_t started;
+ boolean_t istate;
+
+ lck_mtx_lock(&hc_lock);
+ if (!hc_in) {
+ lck_mtx_unlock(&hc_lock);
+ return 0;
+ }
+ hc_in = false; /* nothing more is rung */
+ wakeup(&hc_free);
+ lck_mtx_unlock(&hc_lock);
+ /* what is in flight finishes (whoever waits for it sees it done) */
+ for (started = mach_absolute_time(); hc_read(UFSHC_UTRLDBR) != 0 && hc_us_since(started) < 2000000;) {
+ IOSleep(1);
+ }
+ lck_mtx_lock(&hc_lock);
+ istate = ml_set_interrupts_enabled(FALSE);
+ hc_put_back(&r);
+ ml_set_interrupts_enabled(istate);
+ lck_mtx_unlock(&hc_lock);
+ hc_put_back_print(why, &r, false);
+ if (!r.ok) {
+ hc_off();
+ return EIO;
+ }
+ return 0;
+}
+
+bool
+ufshc_list_in(void)
+{
+ return hc_in;
+}
+
bool
ufshc_available(void)
{
@@ -557,49 +969,70 @@ ufshc_available(void)
}
void
-ufshc_report(const char *when)
+ufshc_set_luns(uint32_t mask)
{
- if (!hc_usable) {
- return;
- }
- printf("ufs: native: %s: %llu requests, %llu errors, %llu bytes, %llu ms waiting (summed over requests), "
- "at most %llu in flight, %llu refused%s\n", when, hc_requests, hc_errors, hc_bytes, hc_wait_us / 1000,
- hc_most_in_flight, hc_refused, hc_dead ? "; THE PATH IS OFF" : "");
+ hc_lun_mask = mask & 0xff;
}
-/* ---- the probe ---- */
-
-static bool
-hc_looks_like_descriptor(const volatile uint32_t *d)
+static void
+hc_report(const char *when, bool late)
{
- bool zero = true;
+ (late ? hc_say_late : hc_say)("ufs: native: %s: %llu reads, %llu errors, %llu bytes, %llu ms waiting (summed over reads), at most %llu "
+ "in flight, %llu refused, list in %llu times and out %llu times, %llu proofs%s\n", when, hc_requests, hc_errors,
+ hc_bytes, hc_wait_us / 1000, hc_most_in_flight, hc_refused, hc_switch_in_count, hc_switch_out_count, hc_proofs,
+ hc_dead ? "; THE PATH IS OFF" : "");
+}
- for (int i = 0; i < 8; i++) {
- zero &= d[i] == 0;
+void
+ufshc_report(const char *when)
+{
+ if (hc_usable) {
+ hc_report(when, false);
}
- /* never used, or a UFS storage request with its command descriptor 128-byte aligned */
- return zero || ((d[0] & 0xf0000000u) == UTRD_CT_UFS && (d[4] & 0x7f) == 0 && (d[0] & UTRD_DD_MASK) != UTRD_DD_MASK);
}
+/* ---- the probe ---- */
+
/*
- * What the controller looks like as UEFI left it, and whether this kernel may put requests of
- * its own into it (only == true: say so and do not touch anything but registers' reads).
+ * What the controller looks like as UEFI left it, and whether this kernel may put a list of its
+ * own in (only == true: say so and do not use it).
*/
bool
-ufshc_probe(bool only)
+ufshc_probe(const struct ufshc_loader *loader, bool only)
{
uint32_t cap, ver, hcs, hce, is, ie, iacr, ahit, base, baseu, db, rsr, cnr, tm_base, tm_baseu, tm_db, tm_rsr;
- uint64_t ucd0 = 0, list_end;
+ uint32_t uefi_dw0 = 0, slots = 31;
+ uint8_t uefi_upiu[32] = { 0 };
+ uint64_t uefi_pa;
const char *no = NULL;
- vm_offset_t list_va;
+ vm_offset_t area;
if (!hc_whitelist_selftest()) {
return false;
}
+ hc_loader = *loader;
PE_parse_boot_argn("ufs_native_buffer", &hc_slot_buffer, sizeof(hc_slot_buffer));
if (hc_slot_buffer < BLOCK || hc_slot_buffer > (4u << 20) || hc_slot_buffer % BLOCK) {
hc_slot_buffer = SLOT_BUFFER_DEFAULT;
}
+ PE_parse_boot_argn("ufs_native_slots", &slots, sizeof(slots));
+ if (slots < 1 || slots > SLOTS - 1) {
+ slots = SLOTS - 1;
+ }
+ PE_parse_boot_argn("ufs_native_leave", &hc_leave, sizeof(hc_leave));
+ if (loader->hc_pa != UFSHC_PHYS || loader->area_pa == 0 || loader->slots != SLOTS || (loader->area_pa & 0x3ff) ||
+ loader->ucd_size < UCD_PRDT + 16 * (hc_slot_buffer / PRD_MAX + 1) || (loader->ucd_offset & 0x7f) ||
+ (loader->ucd_size & 0x7f) || loader->ucd_offset < SLOTS * 32 ||
+ loader->area_size < loader->ucd_offset + SLOTS * loader->ucd_size) {
+ hc_say("ufs: native: not used: the loader offers no area for a list (or not one as expected: controller %llx, "
+ "area %llx of %u, command descriptors at %u of %u)\n", loader->hc_pa, loader->area_pa, loader->area_size,
+ loader->ucd_offset, loader->ucd_size);
+ return false;
+ }
+ if (pmap_valid_address(loader->area_pa) || pmap_valid_address(loader->area_pa + loader->area_size - 1)) {
+ hc_say("ufs: native: not used: the loader's area %llx is memory this kernel manages\n", loader->area_pa);
+ return false;
+ }
hc = ml_io_map(UFSHC_PHYS, UFSHC_SIZE);
cap = hc_read(UFSHC_CAP);
ver = hc_read(UFSHC_VER);
@@ -618,16 +1051,14 @@ ufshc_probe(bool only)
tm_baseu = hc_read(UFSHC_UTMRLBAU);
tm_db = hc_read(UFSHC_UTMRLDBR);
tm_rsr = hc_read(UFSHC_UTMRLRSR);
- hc_is_at_probe = is;
- printf("ufs: native: controller at %llx: CAP %08x (%u slots, 64-bit %s), VER %08x, HCDDID %08x, HCPMID %08x\n",
+ hc_say("ufs: native: controller at %llx: CAP %08x (%u slots, 64-bit %s), VER %08x, HCDDID %08x, HCPMID %08x\n",
UFSHC_PHYS, cap, CAP_NUTRS(cap), (cap & CAP_64AS) ? "yes" : "no", ver, hc_read(UFSHC_HCDDID),
hc_read(UFSHC_HCPMID));
- printf("ufs: native: HCE %08x, HCS %08x, IS %08x, IE %08x, UTRIACR %08x, AHIT %08x\n", hce, hcs, is, ie, iacr, ahit);
- printf("ufs: native: transfer list %08x:%08x, doorbell %08x, run %08x, completion %08x; "
- "task management list %08x:%08x, doorbell %08x, run %08x\n", baseu, base, db, rsr, cnr, tm_baseu, tm_base,
- tm_db, tm_rsr);
- hc_list_pa = (uint64_t)baseu << 32 | base;
- list_end = hc_list_pa + 32ULL * CAP_NUTRS(cap);
+ hc_say("ufs: native: HCE %08x, HCS %08x, IS %08x, IE %08x, UTRIACR %08x, AHIT %08x\n", hce, hcs, is, ie, iacr, ahit);
+ hc_say("ufs: native: transfer list %08x:%08x (the loader's note of UEFI's: %08x:%08x), doorbell %08x, run %08x, "
+ "completion %08x; task management list %08x:%08x, doorbell %08x, run %08x\n", baseu, base, loader->uefi_hi,
+ loader->uefi_lo, db, rsr, cnr, tm_baseu, tm_base, tm_db, tm_rsr);
+ uefi_pa = (uint64_t)baseu << 32 | base;
if ((hce & 1) == 0) {
no = "the controller is not enabled";
@@ -637,133 +1068,120 @@ ufshc_probe(bool only)
no = "the transfer list is not running";
} else if (ver < 0x200 || ver >= 0x400) {
no = "a controller version other than 2.x or 3.x";
- } else if (CAP_NUTRS(cap) < 32) {
+ } else if (CAP_NUTRS(cap) < SLOTS) {
no = "fewer than 32 slots";
} else if ((cap & CAP_64AS) == 0) {
no = "no 64-bit addressing";
} else if (is & IS_FATAL) {
no = "an error status";
- } else if (hc_list_pa == 0 || (hc_list_pa & 0x3ff) != 0) {
- no = "no transfer list, or one not 1 KB aligned";
- } else if (pmap_valid_address(hc_list_pa) || pmap_valid_address(list_end - 1)) {
- no = "the transfer list is in memory this kernel manages";
- } else if (db & SLOTS_RESERVED) {
- no = "a slot of 16..31 is busy";
+ } else if (base != loader->uefi_lo || baseu != loader->uefi_hi) {
+ no = "the transfer list is not the one the loader noted as UEFI's";
+ } else if (db != 0) {
+ no = "a request in flight";
+ } else if (pmap_valid_address(uefi_pa)) {
+ no = "UEFI's transfer list is in memory this kernel manages";
}
- if (no != NULL) {
- printf("ufs: native: not used: %s; the storage is read through the loader\n", no);
- return false;
- }
-
- /* UEFI's list: what each slot holds now (the ones that hold anything) */
- list_va = ml_io_map_unmappable((vm_offset_t)(hc_list_pa & ~(vm_offset_t)PAGE_MASK),
- (vm_size_t)round_page(list_end - (hc_list_pa & ~(uint64_t)PAGE_MASK)), VM_WIMG_DEFAULT);
- hc_list = (volatile uint32_t *)(list_va + (hc_list_pa & PAGE_MASK));
- FlushPoC_DcacheRegion((vm_offset_t)hc_list, 32 * 32);
- for (uint32_t i = 0; i < 32; i++) {
- const volatile uint32_t *d = hc_list + 8 * i;
-
- if ((d[0] | d[1] | d[2] | d[3] | d[4] | d[5] | d[6] | d[7]) != 0) {
- printf("ufs: native: slot %2u: %08x %08x %08x %08x %08x %08x %08x %08x\n", i, d[0], d[1], d[2], d[3], d[4],
- d[5], d[6], d[7]);
- }
- if (i >= SLOT_FIRST && !hc_looks_like_descriptor(d)) {
- no = "a slot of 16..31 holds something that is not a request descriptor";
- }
- if (i < SLOT_FIRST && ucd0 == 0 && (d[4] | d[5]) != 0) {
- ucd0 = (uint64_t)d[5] << 32 | d[4];
- }
- }
- if (ucd0 != 0) {
- /* UEFI's own command descriptor: the last command it sent, and where it is */
- vm_offset_t va = ml_io_map_unmappable((vm_offset_t)(ucd0 & ~(uint64_t)PAGE_MASK), PAGE_SIZE * 2, VM_WIMG_DEFAULT);
- const volatile uint8_t *u = (const volatile uint8_t *)(va + (ucd0 & PAGE_MASK));
-
- FlushPoC_DcacheRegion((vm_offset_t)u, 1024);
- printf("ufs: native: UEFI's command descriptor at %llx: command UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x "
- "%02x %02x %02x %02x %02x; response %02x .. %02x %02x\n", ucd0, u[0], u[1], u[2], u[3], u[16], u[17], u[18],
- u[19], u[20], u[21], u[22], u[23], u[24], u[25], u[512], u[518], u[519]);
- if (ucd0 < list_end && ucd0 + 1024 > hc_list_pa) {
- no = "UEFI's command descriptor is inside the list";
- }
+ if (no == NULL) {
+ /* UEFI's list and command descriptor (the page they are in), not cached here, only read */
+ hc_uefi_map_pa = uefi_pa & ~(uint64_t)PAGE_MASK;
+ hc_uefi_map_size = PAGE_SIZE;
+ hc_uefi_map_va = ml_io_map_unmappable((vm_offset_t)hc_uefi_map_pa, PAGE_SIZE, VM_WIMG_WCOMB);
+ hc_uefi_list = (volatile uint32_t *)(hc_uefi_map_va + (uefi_pa - hc_uefi_map_pa));
+ hc_say("ufs: native: UEFI's slot 0: %08x %08x %08x %08x %08x %08x %08x %08x\n", hc_uefi_list[0], hc_uefi_list[1],
+ hc_uefi_list[2], hc_uefi_list[3], hc_uefi_list[4], hc_uefi_list[5], hc_uefi_list[6], hc_uefi_list[7]);
+ no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);
+ hc_say("ufs: native: UEFI's last command: UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x "
+ "%02x %02x\n", uefi_upiu[0], uefi_upiu[1], uefi_upiu[2], uefi_upiu[3], uefi_upiu[16], uefi_upiu[17],
+ uefi_upiu[18], uefi_upiu[19], uefi_upiu[20], uefi_upiu[21], uefi_upiu[22], uefi_upiu[23], uefi_upiu[24],
+ uefi_upiu[25]);
}
if (no != NULL) {
- printf("ufs: native: not used: %s; the storage is read through the loader\n", no);
+ hc_say("ufs: native: not used: %s; the storage is read through the loader\n", no);
return false;
}
if (only) {
- printf("ufs: native: as expected; ufs_native=2 asks for no more than this look\n");
+ hc_say("ufs: native: as expected; ufs_native=2 asks for no more than this look\n");
return false;
}
- /* this kernel's slots: the even ones of 16..31, a page of descriptor and a buffer each */
- for (uint32_t i = 0; i < SLOT_MAX; i++) {
- struct ufshc_slot *s = &hc_slots[i];
- vm_offset_t ucd = 0, data = 0;
+ /* this kernel's list and command descriptors, in the loader's area; every slot TEST UNIT READY to begin with */
+ area = ml_io_map_unmappable((vm_offset_t)(loader->area_pa & ~(uint64_t)PAGE_MASK),
+ (vm_size_t)round_page((loader->area_pa & PAGE_MASK) + loader->area_size), VM_WIMG_WCOMB) +
+ (vm_offset_t)(loader->area_pa & PAGE_MASK);
+ hc_list_lo = (uint32_t)loader->area_pa;
+ hc_list_hi = (uint32_t)(loader->area_pa >> 32);
+ for (uint32_t i = 0; i < SLOTS; i++) {
+ struct hc_slot *s = &hc_slots[i];
+ struct hc_cmd c;
- if (kmem_alloc_contig(kernel_map, &ucd, UCD_SIZE, PAGE_MASK, 0, 0, KMA_KOBJECT | KMA_ZERO | KMA_DATA,
- VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS ||
- kmem_alloc_contig(kernel_map, &data, round_page(hc_slot_buffer), PAGE_MASK, 0, 0,
- KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
- printf("ufs: native: no memory for slot buffers; the storage is read through the loader\n");
- return false;
+ s->number = i;
+ s->utrd = (volatile uint32_t *)(area + 32 * i);
+ s->ucd = (volatile uint32_t *)(area + loader->ucd_offset + i * loader->ucd_size);
+ s->ucd_pa = loader->area_pa + loader->ucd_offset + i * loader->ucd_size;
+ if (i >= 1 && i <= slots) {
+ vm_offset_t data = 0;
+
+ if (kmem_alloc_contig(kernel_map, &data, round_page(hc_slot_buffer), PAGE_MASK, 0, 0,
+ KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
+ hc_say("ufs: native: not used: no memory for the slots' buffers\n");
+ return false;
+ }
+ s->data = (uint8_t *)data;
+ s->data_pa = kvtophys(data);
+ FlushPoC_DcacheRegion(data, hc_slot_buffer);
}
- s->number = SLOT_FIRST + 2 * i;
- s->utrd = hc_list + 8 * s->number;
- s->ucd = (uint8_t *)ucd;
- s->ucd_pa = kvtophys(ucd);
- s->data = (uint8_t *)data;
- s->data_pa = kvtophys(data);
- for (int w = 0; w < 8; w++) {
- s->original[w] = s->utrd[w];
+ hc_make_tur(&c, 0, (uint8_t)i);
+ if (hc_place(s, &c, 0) != 0) {
+ hc_say("ufs: native: not used: slot %u would not take its TEST UNIT READY\n", i);
+ return false;
}
- FlushPoC_DcacheRegion(ucd, UCD_SIZE);
- FlushPoC_DcacheRegion(data, hc_slot_buffer);
}
- hc_nslots = SLOT_MAX;
- hc_free = (1u << SLOT_MAX) - 1;
+ hc_nslots = slots;
+ hc_free = (uint32_t)(((1ULL << (slots + 1)) - 1) & ~1ULL); /* slots 1..n */
hc_usable = true;
- printf("ufs: native: in use: slots 16, 18, .. 30 of UEFI's list at %llx, %u KB a request; UEFI keeps the rest\n",
- hc_list_pa, hc_slot_buffer >> 10);
+ hc_say("ufs: native: usable: a list of %u slots at %llx (the loader's), reads in %u of them, %u KB each, LUN 0 only%s\n",
+ SLOTS, loader->area_pa, slots, hc_slot_buffer >> 10, hc_leave ? "; ufs_native_leave=1: the list is left in "
+ "at the end, for the loader to take back" : "");
return true;
}
/*
* Before the kernel gives the phone back to the loader (loader_return, the other processors
- * already stopped): nothing more is sent, what is in flight is waited for, and the slots'
- * descriptors are put back as they were, so that UEFI finds its list as it left it.
+ * already stopped): nothing more is sent, what is in flight is waited for, UEFI's list put back.
+ * No locks (a stopped processor may hold one).
*/
void
redwood_ufs_quiesce(void)
{
- uint64_t started = mach_absolute_time();
- uint32_t db = 0, is;
+ struct hc_put_back_result r;
+
+ if (hc_kept_count != 0) {
+ hc_say_late("ufs: native: what happened in this run, again:\n");
+ for (uint32_t i = hc_kept_count > KEPT_LINES ? hc_kept_count - KEPT_LINES : 0; i < hc_kept_count; i++) {
+ const char *line = hc_kept[i % KEPT_LINES];
+ hc_say_late("> %s%s", line, line[0] != 0 && line[strlen(line) - 1] == '\n' ? "" : "...\n");
+ }
+ }
if (!hc_usable) {
return;
}
hc_closed = true;
__asm__ volatile ("dsb sy" ::: "memory");
- while ((db = hc_read(UFSHC_UTRLDBR)) & SLOTS_RESERVED) {
- if (hc_us_since(started) > 2000000) {
- break;
- }
- }
- is = hc_read(UFSHC_IS);
- if (db & SLOTS_RESERVED) {
- printf("ufs: native: at the end, slots still busy after 2 s (doorbell %08x, IS %08x): the descriptors "
- "are left as they are\n", db, is);
+ if (!hc_in && hc_read(UFSHC_UTRLBA) == hc_loader.uefi_lo && hc_read(UFSHC_UTRLBAU) == hc_loader.uefi_hi &&
+ (hc_read(UFSHC_UTRLRSR) & 1)) {
+ hc_say_late("ufs: native: at the end: UEFI's list is in (doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));
+ hc_report("at the end", true);
return;
}
- for (uint32_t i = 0; i < hc_nslots; i++) {
- struct ufshc_slot *s = &hc_slots[i];
-
- for (int w = 0; w < 8; w++) {
- s->utrd[w] = s->original[w];
- }
- FlushPoC_DcacheRegion((vm_offset_t)s->utrd, 32);
+ hc_in = false;
+ if (hc_leave) {
+ hc_say_late("ufs: native: at the end: this kernel's list LEFT IN for the loader to take back "
+ "(ufs_native_leave=1; doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));
+ hc_report("at the end", true);
+ return;
}
- printf("ufs: native: at the end: none of slots 16..31 busy (doorbell %08x), their descriptors as they were; "
- "IS %08x (at the probe %08x); %llu requests, %llu errors, %llu refused\n", db, is, hc_is_at_probe,
- hc_requests, hc_errors, hc_refused);
+ hc_put_back(&r);
+ hc_put_back_print("at the end", &r, true);
+ hc_report("at the end", true);
}
diff --git a/xnu/files/bsd/dev/redwood_ufshc.h b/xnu/files/bsd/dev/redwood_ufshc.h
new file mode 100644
index 0000000..6703ac9
--- /dev/null
+++ b/xnu/files/bsd/dev/redwood_ufshc.h
@@ -0,0 +1,31 @@
+/*
+ * The kernel's own UFS driver (redwood_ufshc.c), as redwood_ufs.c uses it.
+ */
+#ifndef _REDWOOD_UFSHC_H_
+#define _REDWOOD_UFSHC_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/* what the loader's page says about it (struct loader_block's native sections) */
+struct ufshc_loader {
+ uint64_t hc_pa; /* the controller's registers; 0: not offered */
+ uint64_t area_pa; /* the loader's memory for this kernel's list and command descriptors */
+ uint32_t area_size, ucd_offset, ucd_size, slots;
+ uint32_t uefi_lo, uefi_hi; /* UEFI's list */
+ volatile uint32_t *take_back; /* the loader's: it takes the controller back, send nothing more */
+ volatile uint32_t *list_in; /* this kernel's: its list is in */
+ uint32_t *switched_in, *switched_out;
+};
+
+bool ufshc_probe(const struct ufshc_loader *loader, bool only);
+bool ufshc_available(void);
+void ufshc_set_luns(uint32_t mask);
+int ufshc_switch_in(bool prove);
+int ufshc_switch_out(const char *why);
+bool ufshc_list_in(void);
+int ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len);
+void ufshc_report(const char *when);
+void redwood_ufs_quiesce(void);
+
+#endif
diff --git a/xnuloader/loader.c b/xnuloader/loader.c
index ff02f56..22602a3 100644
--- a/xnuloader/loader.c
+++ b/xnuloader/loader.c
@@ -322,6 +322,11 @@ compose_logs(const char *klog, uint64_t klog_len)
staged_len = n;
}
+static int hc_list_is_uefis(void);
+static int hc_log_skips; /* logs not written this run (said twice at most) */
+static void hc_print_state(const char *when);
+static void hc_take_back(const char *when);
+
static void
write_logs(const char *klog, uint64_t klog_len)
{
@@ -341,9 +346,20 @@ write_logs(const char *klog, uint64_t klog_len)
}
uint64_t bs = bio->media->block_size;
uint64_t len = (staged_len + 1 + bs - 1) / bs * bs;
- efi_status ws = bio->write_blocks(bio, bio->media->media_id, 0, len, out_buf), rs = 0;
+ efi_status ws, rs = 0;
int same = 0;
+ /* only through UEFI's own transfer list (see hc_take_back) */
+ if (!hc_list_is_uefis()) {
+ if (hc_log_skips++ < 2) {
+ hc_print_state("log not written");
+ print("log: NOT WRITTEN to logdump: UEFI's transfer list is not the one in the controller\n");
+ }
+ compose_logs(klog, klog_len);
+ return;
+ }
+ ws = bio->write_blocks(bio, bio->media->media_id, 0, len, out_buf);
+
if (!EFI_ERROR(ws)) {
bio->flush_blocks(bio);
/*
@@ -540,6 +556,7 @@ static const struct platform platform_redwood = {
static void
reboot(void)
{
+ hc_take_back("before the reset");
flush_log();
ST->rt->reset_system(0 /* EfiResetCold */, 0, 0, NULL);
for (;;) {
@@ -1441,7 +1458,7 @@ ufs_survey(void)
* back over the data.
*/
#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */
-#define BLOCK_VERSION 2
+#define BLOCK_VERSION 3
#define BLOCK_MAX_PARTS 160
#define BLOCK_MAX_UNITS 16
#define BLOCK_PART_UNIT 1u /* the entry is a whole logical unit, not a partition */
@@ -1483,6 +1500,26 @@ struct loader_block {
uint64_t served, failed, bytes, ticks;
} answer __attribute__((aligned(128)));
struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));
+ /*
+ * Written by the loader before the kernel starts, with ufs_native= on the command line: where
+ * the kernel may put a transfer request list of its own into the UFS host controller for the
+ * run (hc_area, below), and UEFI's, which the loader puts back. hc_pa 0: it may not.
+ */
+ struct {
+ uint64_t hc_pa; /* the controller's registers */
+ uint64_t area_pa; /* the list (1 KB, at the start) and command descriptors */
+ uint32_t area_size, ucd_offset, ucd_size, slots;
+ uint32_t uefi_utrlba, uefi_utrlbau;
+ } native __attribute__((aligned(128)));
+ /* written by the loader during a run: it takes the controller back (a hung kernel); send nothing more */
+ struct {
+ volatile uint32_t take_back;
+ } loader_native __attribute__((aligned(128)));
+ /* written by the kernel: its list is in the controller (for the loader's log) */
+ struct {
+ volatile uint32_t list_in;
+ uint32_t switched_in, switched_out, reserved;
+ } kernel_native __attribute__((aligned(128)));
};
/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */
@@ -1539,6 +1576,174 @@ device_path_print(uint64_t unit, const efi_device_path *dp)
return lun;
}
+/*
+ * The UFS host controller, for the kernel's own driver (ufs_native= on the command line; without it
+ * this loader never touches the controller's registers). The kernel may, for the length of a run,
+ * stop the controller's transfer request list, point it at a list of its own in hc_area (memory of
+ * this loader, so that putting UEFI's back never depends on the kernel's memory), and start it
+ * again; UEFI's list is one slot and its command descriptor, with no room for another request.
+ * This loader keeps UEFI's list address (UTRLBA/UTRLBAU) from before the first run and checks it,
+ * and puts it back if need be, on every way back from a run (hc_take_back) before anything here
+ * touches the storage; it reads for the kernel (block_poll) and writes the log (write_logs) only
+ * while UEFI's list is the one in the controller.
+ */
+#define HC_PA 0x1d84000ULL /* ufshc@1d84000, "ufs_mem", in this phone's Android device tree */
+#define HC_CAP 0x00
+#define HC_VER 0x08
+#define HC_IS 0x20
+#define HC_IE 0x24
+#define HC_HCS 0x30
+#define HC_HCE 0x34
+#define HC_UTRLBA 0x50
+#define HC_UTRLBAU 0x54
+#define HC_UTRLDBR 0x58
+#define HC_UTRLCLR 0x5c
+#define HC_UTRLRSR 0x60
+#define HC_UTRLCNR 0x64
+#define HC_UTMRLDBR 0x78
+#define HC_SLOTS 32
+#define HC_UCD_OFFSET 0x1000
+#define HC_UCD_SIZE 0x800
+#define HC_AREA_SIZE (HC_UCD_OFFSET + HC_SLOTS * HC_UCD_SIZE)
+
+static uint8_t hc_area[HC_AREA_SIZE] __attribute__((aligned(0x2000)));
+static int hc_requested; /* ufs_native= on the command line */
+static int hc_known; /* UEFI's list address is kept here: check it, put it back */
+static uint32_t hc_uefi_lo, hc_uefi_hi;
+static int hc_refused_reads;
+
+static inline uint32_t
+hc_rd(uint32_t off)
+{
+ return *(volatile uint32_t *)(uintptr_t)(HC_PA + off);
+}
+
+static inline void
+hc_wr(uint32_t off, uint32_t v)
+{
+ __asm__ volatile ("dsb sy" ::: "memory");
+ *(volatile uint32_t *)(uintptr_t)(HC_PA + off) = v;
+ __asm__ volatile ("dsb sy" ::: "memory");
+}
+
+static uint64_t
+hc_ms_since(uint64_t then)
+{
+ return (READ_SYSREG(CNTVCT_EL0) - then) * 1000 / READ_SYSREG(CNTFRQ_EL0);
+}
+
+/* UEFI's list in, and running: what UEFI's driver needs to read or write */
+static int
+hc_list_is_uefis(void)
+{
+ return !hc_known || (hc_rd(HC_UTRLBA) == hc_uefi_lo && hc_rd(HC_UTRLBAU) == hc_uefi_hi &&
+ (hc_rd(HC_UTRLRSR) & 1) != 0);
+}
+
+static void
+hc_print_state(const char *when)
+{
+ print("ufs: %s: list %x:%x (UEFI's %x:%x), run %x, doorbell %x, HCS %x, IS %x, IE %x, completion %x, "
+ "task doorbell %x\n", when, (uint64_t)hc_rd(HC_UTRLBAU), (uint64_t)hc_rd(HC_UTRLBA), (uint64_t)hc_uefi_hi,
+ (uint64_t)hc_uefi_lo, (uint64_t)hc_rd(HC_UTRLRSR), (uint64_t)hc_rd(HC_UTRLDBR), (uint64_t)hc_rd(HC_HCS),
+ (uint64_t)hc_rd(HC_IS), (uint64_t)hc_rd(HC_IE), (uint64_t)hc_rd(HC_UTRLCNR), (uint64_t)hc_rd(HC_UTMRLDBR));
+}
+
+/*
+ * Whatever the kernel left: UEFI's list back in the controller. The kernel is told to send nothing
+ * more (it looks before every doorbell, with interrupts off), given 50 ms, then what is in flight
+ * is waited for (1 s), cleared if it does not finish (UTRLCLR: every slot of that list is the
+ * kernel's), the list stopped, UEFI's address written back, the list started, all read back.
+ */
+static void
+hc_take_back(const char *when)
+{
+ uint32_t lo, hi, rsr, db;
+ uint64_t t;
+
+ if (!hc_known) {
+ return;
+ }
+ lo = hc_rd(HC_UTRLBA);
+ hi = hc_rd(HC_UTRLBAU);
+ rsr = hc_rd(HC_UTRLRSR);
+ db = hc_rd(HC_UTRLDBR);
+ if (lo == hc_uefi_lo && hi == hc_uefi_hi && (rsr & 1)) {
+ print("ufs: %s: UEFI's transfer list is in the controller (%x), running, doorbell %x\n", when, (uint64_t)lo,
+ (uint64_t)db);
+ return;
+ }
+ print("ufs: %s: the controller has list %x:%x (run %x, doorbell %x), not UEFI's %x:%x: taking it back\n", when,
+ (uint64_t)hi, (uint64_t)lo, (uint64_t)rsr, (uint64_t)db, (uint64_t)hc_uefi_hi, (uint64_t)hc_uefi_lo);
+ block_shared.loader_native.take_back = 1;
+ clean_dcache((uint64_t)(uintptr_t)&block_shared.loader_native,
+ (uint64_t)(uintptr_t)&block_shared.loader_native + sizeof(block_shared.loader_native));
+ for (t = READ_SYSREG(CNTVCT_EL0); hc_ms_since(t) < 50;) {
+ }
+ for (t = READ_SYSREG(CNTVCT_EL0); (db = hc_rd(HC_UTRLDBR)) != 0 && hc_ms_since(t) < 1000;) {
+ }
+ if (db != 0) {
+ print("ufs: %s: doorbell %x still set after 1 s\n", when, (uint64_t)db);
+ if (hc_rd(HC_UTRLBA) != hc_uefi_lo || hc_rd(HC_UTRLBAU) != hc_uefi_hi) {
+ print("ufs: %s: clearing the kernel's slots (UTRLCLR)\n", when);
+ hc_wr(HC_UTRLCLR, 0);
+ for (t = READ_SYSREG(CNTVCT_EL0); (db = hc_rd(HC_UTRLDBR)) != 0 && hc_ms_since(t) < 1000;) {
+ }
+ print("ufs: %s: doorbell %x after the clear\n", when, (uint64_t)db);
+ }
+ }
+ hc_wr(HC_UTRLRSR, 0);
+ for (t = READ_SYSREG(CNTVCT_EL0); (hc_rd(HC_UTRLRSR) & 1) && hc_ms_since(t) < 100;) {
+ }
+ hc_wr(HC_UTRLBA, hc_uefi_lo);
+ hc_wr(HC_UTRLBAU, hc_uefi_hi);
+ hc_wr(HC_UTRLRSR, 1);
+ for (t = READ_SYSREG(CNTVCT_EL0); hc_ms_since(t) < 50;) {
+ }
+ hc_print_state(when);
+ print("ufs: %s: UEFI's list %s\n", when, hc_list_is_uefis() ? "put back" : "NOT PUT BACK: the storage is not used");
+}
+
+/*
+ * Before a run (ufs_native=): what the controller looks like, and UEFI's list address kept -- the
+ * first time; later it must be the same (or this loader's own area, which is then taken back).
+ */
+static int
+hc_init(void)
+{
+ uint32_t lo, hi, area = (uint32_t)(uintptr_t)hc_area;
+
+ if (!hc_requested) {
+ return 0;
+ }
+ lo = hc_rd(HC_UTRLBA);
+ hi = hc_rd(HC_UTRLBAU);
+ print("ufs: controller at %x: CAP %x, VER %x, HCE %x, HCS %x, IS %x, IE %x, list %x:%x, run %x, doorbell %x\n",
+ HC_PA, (uint64_t)hc_rd(HC_CAP), (uint64_t)hc_rd(HC_VER), (uint64_t)hc_rd(HC_HCE), (uint64_t)hc_rd(HC_HCS),
+ (uint64_t)hc_rd(HC_IS), (uint64_t)hc_rd(HC_IE), (uint64_t)hi, (uint64_t)lo, (uint64_t)hc_rd(HC_UTRLRSR),
+ (uint64_t)hc_rd(HC_UTRLDBR));
+ if (!hc_known) {
+ if (hi == 0 && lo == area) {
+ print("ufs: the controller has this loader's own area for its list before any run: left alone\n");
+ return 0;
+ }
+ hc_uefi_lo = lo;
+ hc_uefi_hi = hi;
+ hc_known = 1;
+ print("ufs: UEFI's transfer list is at %x:%x; it is put back after every run\n", (uint64_t)hi, (uint64_t)lo);
+ } else if (lo != hc_uefi_lo || hi != hc_uefi_hi) {
+ if (hi == 0 && lo == area) {
+ hc_take_back("before the run");
+ } else {
+ print("ufs: UEFI's list moved from %x:%x to %x:%x: that is UEFI's now\n", (uint64_t)hc_uefi_hi,
+ (uint64_t)hc_uefi_lo, (uint64_t)hi, (uint64_t)lo);
+ hc_uefi_lo = lo;
+ hc_uefi_hi = hi;
+ }
+ }
+ return hc_list_is_uefis();
+}
+
/*
* The firmware's configuration tables, by GUID (an ACPI or device tree table among them would say
* where the storage controller is).
@@ -1648,6 +1853,25 @@ block_start(void)
bp->flags = BLOCK_PART_UNIT;
block_bios[block_shared.parts++] = units[u].bio;
}
+ /* the kernel's own transfer list: this loader's area, if the controller is as UEFI keeps it */
+ if (hc_known && hc_list_is_uefis() && hc_rd(HC_UTRLDBR) == 0 && (hc_rd(HC_HCE) & 1)) {
+ memset(hc_area, 0, sizeof(hc_area));
+ flush_dcache((uint64_t)(uintptr_t)hc_area, (uint64_t)(uintptr_t)hc_area + sizeof(hc_area));
+ block_shared.native.hc_pa = HC_PA;
+ block_shared.native.area_pa = (uint64_t)(uintptr_t)hc_area;
+ block_shared.native.area_size = sizeof(hc_area);
+ block_shared.native.ucd_offset = HC_UCD_OFFSET;
+ block_shared.native.ucd_size = HC_UCD_SIZE;
+ block_shared.native.slots = HC_SLOTS;
+ block_shared.native.uefi_utrlba = hc_uefi_lo;
+ block_shared.native.uefi_utrlbau = hc_uefi_hi;
+ print("ufs: the kernel may put its own transfer list in, at %x (this loader's), for the run\n",
+ (uint64_t)(uintptr_t)hc_area);
+ } else if (hc_requested) {
+ print("ufs: the kernel's own transfer list is not offered: the controller is not as UEFI keeps it\n");
+ }
+ hc_refused_reads = 0;
+ hc_log_skips = 0;
block_shared.magic = BLOCK_MAGIC;
block_shared.version = BLOCK_VERSION;
block_shared.size = sizeof(block_shared);
@@ -1764,7 +1988,16 @@ block_poll(void)
efi_block_io *bio = block_bios[part];
bytes = count * block_shared.part[part].block_size;
- s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);
+ if (!hc_list_is_uefis()) {
+ /* the kernel's own list is in: UEFI's driver must not ring now (the kernel should not ask) */
+ if (hc_refused_reads++ < 4) {
+ print("ufs: a read asked for while the kernel's own transfer list is in the controller: refused\n");
+ }
+ s = 0x800000000000000FULL; /* EFI_ACCESS_DENIED */
+ bytes = 0;
+ } else {
+ s = bio->read_blocks(bio, bio->media->media_id, lba, bytes, block_buffer);
+ }
/* what the controller wrote is in memory; drop whatever this core held of the buffer */
flush_dcache((uint64_t)(uintptr_t)block_buffer, (uint64_t)(uintptr_t)block_buffer + bytes);
}
@@ -1951,7 +2184,8 @@ run_on_other_cpu(uint64_t mpidr, uint64_t entry, uint64_t boot_args_pa, const st
last_pet = READ_SYSREG(CNTVCT_EL0);
}
/* Qualcomm's UEFI charger stack owns the PMIC/GLINK side; keep XNU's cache live. */
- if (charger_enabled && seconds_since(last_battery) >= 5) {
+ if (charger_enabled && seconds_since(last_battery) >= 5 && hc_list_is_uefis()) {
+ /* (not while the kernel's own transfer list is in: Qualcomm's DXEs are not to touch the storage) */
charger_poll();
last_battery = READ_SYSREG(CNTVCT_EL0);
}
@@ -2481,10 +2715,15 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
ufs_requested = c[4] - '0';
}
block_probe_requested |= starts_with(c, "ufs_probe=1") && (c == pi.cmdline || c[-1] == ' ');
+ hc_requested |= starts_with(c, "ufs_native=") && (c == pi.cmdline || c[-1] == ' ') && c[11] >= '1' &&
+ c[11] <= '9';
}
if (battery_requested) {
charger_start();
}
+ if (ufs_requested >= 2) {
+ hc_init(); /* (ufs_native=: UEFI's transfer list kept, before anything reads) */
+ }
if (ufs_requested >= 1) {
ufs_survey();
}
@@ -2908,6 +3147,10 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
int64_t status = run_on_other_cpu(kernel_mpidr, entry_va + slide, ba_pa, kernel_live, slide, kernel_budget);
wdt_set_ms(WDT_BITE_MS);
+ if (status != 0) {
+ /* a kernel that hung may have left its own transfer list in the controller */
+ hc_take_back(status == OTHER_CPU_HUNG ? "the kernel hung" : "the kernel did not start");
+ }
if (status == OTHER_CPU_HUNG) {
/* the kernel still has that core; what it logged can be had all the same */
print("THE KERNEL HUNG on core %x: no sign of life for %d seconds; its log follows\n", kernel_mpidr,
@@ -2933,6 +3176,13 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
/* --- the kernel came back ------------------------------------------- */
wdt_set_ms(WDT_BITE_MS);
+ hc_take_back("the kernel came back"); /* before anything here touches the storage */
+ if (hc_known) {
+ flush_dcache((uint64_t)(uintptr_t)&block_shared.kernel_native,
+ (uint64_t)(uintptr_t)&block_shared.kernel_native + sizeof(block_shared.kernel_native));
+ print("ufs: the kernel put its own list in %d times and took it out %d times\n",
+ (uint64_t)block_shared.kernel_native.switched_in, (uint64_t)block_shared.kernel_native.switched_out);
+ }
/* ... and its other cores should all be off again: the next kernel cannot start one that is not */
for (int i = 1; i < cpu_count; i++) {
int64_t state = psci_call(PSCI_AFFINITY_INFO, cpu_mpidr[i], 0, 0);