Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
/*
* The phone's UFS host controller, driven from this kernel -- read only, next to UEFI's driver.
*
* Reading the storage through the loader (hakios.ufs.c) is one request at a time: Qualcomm's
* UFSDxe reads synchronously on core 0, and its transfer request list has room for one request
* only (UTRLBA points at a single descriptor, with UFSDxe's command descriptor right behind it,
* where slots 8.. would be). The controller takes 32 at once. So for the length of a run this
* kernel puts a list of its own into the controller: with nothing in flight, the list is stopped
* (UTRLRSR 0), UTRLBA/UTRLBAU are pointed at the new one, the list is started again; at the end
* (hakios_ufs_quiesce, from loader_return), what is in flight is waited for, the list stopped,
* UEFI's address written back, the list started. The list and its command descriptors are in
* memory the loader owns (it offers it in its page, with UEFI's address); the loader checks the
* address on every way back from a run and puts UEFI's back itself if the kernel did not
* (xnuloader/loader.c, hc_take_back), and neither reads nor writes for the kernel or for its log
* while another list is in. While this kernel's list is in, it asks the loader for nothing.
*
* Nothing else of the controller is changed: no reset, no link startup, no UIC command, no
* interrupt enable, aggregation, clock or power setting; the task management list is not
* touched; IS and UTRLCNR are only read. UTRLCLR is written only to clear this kernel's own
* slots when they do not finish at the end.
*
* Read only by construction: a command is made in a buffer of its own, and only one that passes
* ufshc_permitted() is ever written into a slot; what the slot then holds is read back and must
* be the same and pass again before the doorbell is rung. The whitelist checks three things of
* every command: the SCSI operation (READ(10)/READ(16), TEST UNIT READY, REQUEST SENSE, INQUIRY,
* READ CAPACITY(10/16), nothing else -- no query, no task management, no NOP), the data
* direction in the request descriptor (from the device or none, never to it), and the command
* UPIU's flags (never W); and the logical unit: never a well-known one (REPORT LUNS, BOOT, RPMB,
* the device), and only those reads are allowed to go to now (ufshc_set_luns: LUN 0, where the
* Darwin volume is, unless a comparison with the loader's reads asks for more). A read's length in
* the CDB must be what the UPIU expects. It is tried at the probe on made-up commands that must
* pass or be refused. This file builds READ(10) and TEST UNIT READY only.
*
* Every slot of this list holds a whitelisted command at all times: TEST UNIT READY until it is
* used for a read. Slot 0 is kept for the proof that the controller does read this list: after
* every switch, before any read, slot 0's TEST UNIT READY is rung and must come back in this
* list (its status written here). Were the controller still reading UEFI's list, the doorbell
* would run UEFI's slot 0 instead -- so first UEFI's slot 0 must hold a read too (it does: the
* loader's last use of the storage before a run is reading back the log it wrote), and the
* proof costs nothing either way. Reads go in slots 1..31.
*
* The doorbell is rung only with interrupts off, after looking that this kernel's list is the
* one in the controller and running, that the slot is free, and that the loader is not taking
* the controller back (it says so in its page, then waits before it does).
*
* The list and the command descriptors are mapped here without caching (nothing of them is ever
* in a cache line of this kernel's to be written back over what the controller writes); the data
* buffers are this kernel's memory, cleaned and invalidated around every read.
*/
#include <kern/clock.h>
#include <kern/locks.h>
#include <kern/thread.h>
#include <os/atomic_private.h>
#include <libkern/libkern.h>
#include <stdarg.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <pexpert/pexpert.h>
#include <sys/errno.h>
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <vm/vm_kern.h>
#include <vm/memory_types.h>
#include "hakios.ufshc.h"
extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
extern uint64_t kvtophys(vm_offset_t va); /* pmap_paddr_t */
extern boolean_t pmap_valid_address(uint64_t addr);
extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length);
extern void IOSleep(unsigned int milliseconds);
extern boolean_t ml_set_interrupts_enabled(boolean_t enable);
#define UFSHC_PHYS 0x1d84000ULL /* ufshc@1d84000, "ufs_mem", in the phone's Android device tree */
#define UFSHC_SIZE 0x3000
/* UFSHCI registers (JESD223). 0x38..0x48, the UIC error codes, clear when read: never read here. */
#define UFSHC_CAP 0x00
#define UFSHC_VER 0x08
#define UFSHC_HCDDID 0x10
#define UFSHC_HCPMID 0x14
#define UFSHC_AHIT 0x18
#define UFSHC_IS 0x20
#define UFSHC_IE 0x24
#define UFSHC_HCS 0x30
#define UFSHC_HCE 0x34
#define UFSHC_UTRIACR 0x4c
#define UFSHC_UTRLBA 0x50
#define UFSHC_UTRLBAU 0x54
#define UFSHC_UTRLDBR 0x58
#define UFSHC_UTRLCLR 0x5c
#define UFSHC_UTRLRSR 0x60
#define UFSHC_UTRLCNR 0x64
#define UFSHC_UTMRLBA 0x70
#define UFSHC_UTMRLBAU 0x74
#define UFSHC_UTMRLDBR 0x78
#define UFSHC_UTMRLRSR 0x80
#define CAP_NUTRS(c) (((c) & 0x1f) + 1)
#define CAP_64AS (1u << 24)
#define HCS_DP (1u << 0)
#define HCS_UTRLRDY (1u << 1)
#define IS_FATAL ((1u << 2) | (1u << 11) | (1u << 12) | (1u << 16) | (1u << 17)) /* UE, DFES, UTPES, HCFES, SBFES */
/* the request descriptor's first word: command type, data direction */
#define UTRD_CT_MASK 0xf0000000u
#define UTRD_CT_UFS (1u << 28)
#define UTRD_DD_MASK (3u << 25)
#define UTRD_DD_NONE (0u << 25)
#define UTRD_DD_OUT (1u << 25) /* host to device: never */
#define UTRD_DD_IN (2u << 25)
#define OCS_SUCCESS 0x0
#define OCS_INVALID 0xf
#define UPIU_COMMAND 0x01
#define UPIU_RESPONSE 0x21
#define UPIU_FLAG_READ 0x40
#define UPIU_FLAG_WRITE 0x20
#define LUN_WELL_KNOWN 0x80
#define SLOTS 32 /* this kernel's list */
#define SLOT_PROOF 0
#define UCD_RESPONSE 512
#define UCD_PRDT 1024
#define PRD_MAX (256u << 10) /* the most one table entry describes */
#define SLOT_BUFFER_DEFAULT (256u << 10)
#define BLOCK 4096u /* the logical block size of every unit here */
#define TIMEOUT_MS 10000
struct hc_cmd {
uint8_t upiu[32];
uint32_t dw0;
};
struct hc_slot {
uint32_t number; /* in this kernel's list */
volatile uint32_t *utrd; /* its descriptor (the loader's memory, not cached here) */
volatile uint32_t *ucd; /* its command descriptor (likewise) */
uint64_t ucd_pa;
uint8_t *data; /* this kernel's memory, cached */
uint64_t data_pa;
};
static vm_offset_t hc; /* the registers */
static struct ufshc_loader hc_loader;
static uint32_t hc_list_lo, hc_list_hi; /* this kernel's list's address */
static volatile uint32_t *hc_uefi_list; /* UEFI's list, not cached here, only read */
static uint64_t hc_uefi_map_pa, hc_uefi_map_size;
static vm_offset_t hc_uefi_map_va;
static struct hc_slot hc_slots[SLOTS];
static uint32_t hc_nslots; /* slots 1..hc_nslots carry reads */
static uint32_t hc_free; /* bit i: hc_slots[i] free */
static uint32_t hc_slot_buffer = SLOT_BUFFER_DEFAULT;
static uint32_t hc_lun_mask = 1; /* the logical units reads may go to: LUN 0 */
static bool hc_usable;
static uint32_t hc_leave; /* ufs_native_leave=1 (a test): the list left in at the end, for the loader */
static volatile bool hc_in, hc_closed, hc_dead;
static uint32_t hc_is_in, hc_cnr_in; /* IS and UTRLCNR when this kernel's list went in */
static uint64_t hc_requests, hc_errors, hc_bytes, hc_wait_us, hc_refused, hc_most_in_flight, hc_proofs;
static uint64_t hc_switch_in_count, hc_switch_out_count;
static uint32_t hc_in_flight;
/* where the time goes: doorbell reads while waiting, waits for a free slot, taking hc_lock, how many in flight */
static uint64_t hc_polls, hc_slot_waits, hc_slot_wait_ticks, hc_lock_ticks, hc_flight_hist[6];
static LCK_GRP_DECLARE(hc_lck_grp, "hakios-ufshc");
static LCK_MTX_DECLARE(hc_lock, &hc_lck_grp);
SYSCTL_DECL(_kern_hakios);
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_requests,
"reads this kernel's own UFS driver has had answered");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_errors,
"reads of this kernel's own UFS driver that failed");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_bytes,
"bytes this kernel's own UFS driver has read");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_wait_us,
"microseconds spent waiting for the controller, summed over the reads");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_refused, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_refused,
"commands the whitelist refused (there should never be one)");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_most_in_flight, CTLFLAG_RD | CTLFLAG_LOCKED, &hc_most_in_flight,
"the most reads this kernel's own UFS driver has had in flight at once");
/*
* What this file says goes to the kernel's log, and the last lines are kept: the start of a long
* run's log does not come back to the loader (the buffer wraps), so they are said again at the
* end, into the loader's log directly (hakios_ufs_quiesce; printf is not for that moment).
*/
extern void loader_log_putc(char c);
#define KEPT_LINES 64
static char hc_kept[KEPT_LINES][208];
static uint32_t hc_kept_count;
static void
hc_say(const char *format, ...) __printflike(1, 2);
static void
hc_say(const char *format, ...)
{
char line[208];
va_list list;
va_start(list, format);
vsnprintf(line, sizeof(line), format, list);
va_end(list);
printf("%s", line);
strlcpy(hc_kept[hc_kept_count++ % KEPT_LINES], line, sizeof(hc_kept[0]));
}
/* the same, at the end: into the loader's log only */
static void
hc_say_late(const char *format, ...) __printflike(1, 2);
static void
hc_say_late(const char *format, ...)
{
char line[208];
va_list list;
va_start(list, format);
vsnprintf(line, sizeof(line), format, list);
va_end(list);
for (const char *c = line; *c != 0; c++) {
loader_log_putc(*c);
}
}
static inline uint32_t
hc_read(uint32_t reg)
{
return *(volatile uint32_t *)(hc + reg);
}
static inline void
hc_write(uint32_t reg, uint32_t value)
{
__asm__ volatile ("dsb sy" ::: "memory");
*(volatile uint32_t *)(hc + reg) = value;
__asm__ volatile ("dsb sy" ::: "memory");
}
static uint64_t
hc_us_since(uint64_t start)
{
uint64_t ns;
absolutetime_to_nanoseconds(mach_absolute_time() - start, &ns);
return ns / 1000;
}
/* ---- the whitelist: the one check every command passes before it goes into a slot ---- */
static const char *
hc_refusal(const uint8_t *upiu, uint32_t dw0, uint32_t lun_mask, bool any_command_type)
{
const uint8_t *cdb = upiu + 16;
uint32_t dd = dw0 & UTRD_DD_MASK;
uint64_t expected = (uint64_t)upiu[12] << 24 | (uint64_t)upiu[13] << 16 | (uint64_t)upiu[14] << 8 | upiu[15];
uint8_t lun = upiu[2];
/* the data direction: from the device, or none -- never to it */
if (dd == UTRD_DD_OUT || dd == UTRD_DD_MASK) {
return "data from host to device";
}
if (!any_command_type && (dw0 & UTRD_CT_MASK) != UTRD_CT_UFS) {
return "not a UFS storage command type";
}
/* a SCSI command, and its flags: never W */
if ((upiu[0] & 0x3f) != UPIU_COMMAND) {
return "not a SCSI command (a query, task management, NOP, ...)";
}
if (upiu[1] & UPIU_FLAG_WRITE) {
return "the W flag";
}
if (upiu[4] & 0x0f) {
return "not the SCSI command set";
}
/* a normal logical unit, and one reads may go to now */
if (lun & LUN_WELL_KNOWN) {
return "a well-known logical unit";
}
if (lun >= 8 || (lun_mask & (1u << lun)) == 0) {
return "a logical unit reads do not go to now";
}
/* the operation */
switch (cdb[0]) {
case 0x28: /* READ(10) */
case 0x88: { /* READ(16) */
uint64_t blocks = cdb[0] == 0x28 ? ((uint64_t)cdb[7] << 8 | cdb[8]) :
((uint64_t)cdb[10] << 24 | (uint64_t)cdb[11] << 16 | (uint64_t)cdb[12] << 8 | cdb[13]);
if (dd != UTRD_DD_IN || (upiu[1] & UPIU_FLAG_READ) == 0) {
return "a read without data from the device";
}
if (blocks == 0 || expected != blocks * BLOCK) {
return "a read whose length is not what the UPIU expects";
}
return NULL;
}
case 0x00: /* TEST UNIT READY */
return dd == UTRD_DD_NONE && expected == 0 && (upiu[1] & UPIU_FLAG_READ) == 0 ? NULL :
"TEST UNIT READY with data";
case 0x03: /* REQUEST SENSE */
case 0x12: /* INQUIRY */
case 0x25: /* READ CAPACITY(10) */
return NULL;
case 0x9e: /* SERVICE ACTION IN(16): READ CAPACITY(16) only */
return (cdb[1] & 0x1f) == 0x10 ? NULL : "a SERVICE ACTION IN other than READ CAPACITY(16)";
default:
return "a SCSI operation not on the list";
}
}
static bool
ufshc_permitted(const uint8_t *upiu, uint32_t dw0)
{
const char *why = hc_refusal(upiu, dw0, hc_lun_mask, false);
if (why == NULL) {
return true;
}
hc_refused++;
hc_say("ufs: native: REFUSED (%s): UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x, "
"descriptor %08x; it goes into no slot\n", why, upiu[0], upiu[1], upiu[2], upiu[3], upiu[16], upiu[17], upiu[18],
upiu[19], upiu[20], upiu[21], upiu[22], upiu[23], upiu[24], upiu[25], dw0);
return false;
}
static void
hc_make_read(struct hc_cmd *c, uint8_t lun, uint8_t tag, uint32_t lba, uint32_t blocks)
{
uint32_t bytes = blocks * BLOCK;
bzero(c, sizeof(*c));
c->upiu[0] = UPIU_COMMAND;
c->upiu[1] = UPIU_FLAG_READ; /* simple task attribute */
c->upiu[2] = lun;
c->upiu[3] = tag;
c->upiu[12] = (uint8_t)(bytes >> 24);
c->upiu[13] = (uint8_t)(bytes >> 16);
c->upiu[14] = (uint8_t)(bytes >> 8);
c->upiu[15] = (uint8_t)bytes;
c->upiu[16] = 0x28; /* READ(10) */
c->upiu[18] = (uint8_t)(lba >> 24);
c->upiu[19] = (uint8_t)(lba >> 16);
c->upiu[20] = (uint8_t)(lba >> 8);
c->upiu[21] = (uint8_t)lba;
c->upiu[23] = (uint8_t)(blocks >> 8);
c->upiu[24] = (uint8_t)blocks;
c->dw0 = UTRD_CT_UFS | UTRD_DD_IN; /* no interrupt, no inline encryption */
}
static void
hc_make_tur(struct hc_cmd *c, uint8_t lun, uint8_t tag)
{
bzero(c, sizeof(*c));
c->upiu[0] = UPIU_COMMAND;
c->upiu[2] = lun;
c->upiu[3] = tag;
c->upiu[16] = 0x00; /* TEST UNIT READY */
c->dw0 = UTRD_CT_UFS | UTRD_DD_NONE;
}
/*
* The whitelist tried on commands made up here and never sent: those that must be refused,
* and those that must pass. Pure bytes; nothing touches the controller.
*/
static bool
hc_whitelist_selftest(void)
{
static const struct {
uint8_t type, flags, lun, op, cdb1;
uint16_t blocks; /* the CDB's length (READ(10)/(16)) */
uint32_t expected; /* the UPIU's */
uint32_t dd;
bool pass;
const char *what;
} cases[] = {
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, true, "READ(10) of LUN 0" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x88, 0, 1, 4096, UTRD_DD_IN, true, "READ(16) of LUN 0" },
{ UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_NONE, true, "TEST UNIT READY of LUN 0" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x25, 0, 0, 8, UTRD_DD_IN, true, "READ CAPACITY(10)" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x2a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(10)" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x8a, 0, 1, 4096, UTRD_DD_OUT, false, "WRITE(16)" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x2a, 0, 1, 4096, UTRD_DD_IN, false, "WRITE(10) dressed as a read" },
{ UPIU_COMMAND, 0, 0, 0x2a, 0, 0, 0, UTRD_DD_NONE, false, "WRITE(10) with no data" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x42, 0, 0, 24, UTRD_DD_OUT, false, "UNMAP" },
{ UPIU_COMMAND, 0, 0, 0x42, 0, 0, 0, UTRD_DD_NONE, false, "UNMAP with no data" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_OUT, false, "READ(10) with data host to device" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 4096, UTRD_DD_MASK, false, "READ(10) with direction 3" },
{ UPIU_COMMAND, UPIU_FLAG_READ | UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with the W flag" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) with only the W flag" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x28, 0, 1, 8192, UTRD_DD_IN, false, "READ(10) of a length other than expected" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0xc4, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the RPMB unit" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0xb0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the BOOT unit" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0xd0, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of the device unit" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0x81, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of REPORT LUNS" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 3, 0x28, 0, 1, 4096, UTRD_DD_IN, false, "READ(10) of LUN 3 (not allowed now)" },
{ UPIU_COMMAND, 0, 0, 0x04, 0, 0, 0, UTRD_DD_NONE, false, "FORMAT UNIT" },
{ UPIU_COMMAND, 0, 0, 0x1b, 0, 0, 0, UTRD_DD_NONE, false, "START STOP UNIT" },
{ UPIU_COMMAND, 0, 0, 0x35, 0, 0, 0, UTRD_DD_NONE, false, "SYNCHRONIZE CACHE(10)" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x3b, 0, 0, 4096, UTRD_DD_OUT, false, "WRITE BUFFER" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0xb5, 0, 0, 512, UTRD_DD_OUT, false, "SECURITY PROTOCOL OUT" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0xa2, 0, 0, 512, UTRD_DD_IN, false, "SECURITY PROTOCOL IN" },
{ UPIU_COMMAND, UPIU_FLAG_WRITE, 0, 0x15, 0, 0, 24, UTRD_DD_OUT, false, "MODE SELECT" },
{ UPIU_COMMAND, UPIU_FLAG_READ, 0, 0x9e, 0x11, 0, 32, UTRD_DD_IN, false, "SERVICE ACTION IN 0x11" },
{ UPIU_COMMAND, 0, 0, 0x00, 0, 0, 0, UTRD_DD_IN, false, "TEST UNIT READY with data in" },
{ 0x16, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "a QUERY REQUEST" },
{ 0x04, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "TASK MANAGEMENT" },
{ 0x00, 0, 0, 0, 0, 0, 0, UTRD_DD_NONE, false, "NOP OUT" },
};
bool right = true;
for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
uint8_t upiu[32] = { 0 };
bool passed;
upiu[0] = cases[i].type;
upiu[1] = cases[i].flags;
upiu[2] = cases[i].lun;
upiu[12] = (uint8_t)(cases[i].expected >> 24);
upiu[13] = (uint8_t)(cases[i].expected >> 16);
upiu[14] = (uint8_t)(cases[i].expected >> 8);
upiu[15] = (uint8_t)cases[i].expected;
upiu[16] = cases[i].op;
upiu[17] = cases[i].cdb1;
if (cases[i].op == 0x88) {
upiu[16 + 12] = (uint8_t)(cases[i].blocks >> 8);
upiu[16 + 13] = (uint8_t)cases[i].blocks;
} else {
upiu[16 + 7] = (uint8_t)(cases[i].blocks >> 8);
upiu[16 + 8] = (uint8_t)cases[i].blocks;
}
passed = hc_refusal(upiu, UTRD_CT_UFS | cases[i].dd, 1u << 0, false) == NULL;
if (passed != cases[i].pass) {
hc_say("ufs: native: the whitelist %s %s\n", passed ? "LETS THROUGH" : "refuses", cases[i].what);
right = false;
}
}
/* and what this file makes */
{
struct hc_cmd c;
hc_make_read(&c, 0, 1, 12345, 64);
right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;
hc_make_read(&c, 2, 1, 12345, 64);
right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) != NULL;
hc_make_tur(&c, 0, 0);
right &= hc_refusal(c.upiu, c.dw0, 1u << 0, false) == NULL;
}
hc_say("ufs: native: whitelist tried on %u commands made up and never sent -- READ(10/16), TEST UNIT READY, READ "
"CAPACITY of LUN 0 pass; WRITE(10/16), UNMAP, a READ with data host to device, a READ with the W flag, the "
"RPMB/BOOT/device/REPORT LUNS units, LUN 3, FORMAT, START STOP, SYNCHRONIZE CACHE, WRITE BUFFER, SECURITY "
"PROTOCOL, MODE SELECT, query, task management, NOP refused: %s\n",
(unsigned)(sizeof(cases) / sizeof(cases[0])) + 3, right ? "as it should" : "WRONG");
return right;
}
/* ---- one command in one slot ---- */
static void
hc_put(volatile uint32_t *to, const void *from, uint32_t bytes)
{
for (uint32_t i = 0; i < bytes / 4; i++) {
uint32_t w;
memcpy(&w, (const uint8_t *)from + 4 * i, 4);
to[i] = w;
}
}
/*
* Into the slot: the command UPIU, an empty response, the PRD table for bytes of the slot's
* buffer, the descriptor -- only a command the whitelist passes, and only if what the slot then
* holds is the same and passes again.
*/
static int
hc_place(struct hc_slot *s, const struct hc_cmd *c, uint32_t bytes)
{
uint32_t n = 0, back[8];
if (!ufshc_permitted(c->upiu, c->dw0)) {
return EPERM;
}
hc_put(s->ucd, c->upiu, sizeof(c->upiu));
for (uint32_t i = sizeof(c->upiu) / 4; i < UCD_PRDT / 4; i++) {
s->ucd[i] = 0;
}
for (uint32_t at = 0; at < bytes; at += PRD_MAX, n++) {
volatile uint32_t *prd = s->ucd + (UCD_PRDT + 16 * n) / 4;
uint64_t pa = s->data_pa + at;
uint32_t len = bytes - at < PRD_MAX ? bytes - at : PRD_MAX;
prd[0] = (uint32_t)pa;
prd[1] = (uint32_t)(pa >> 32);
prd[2] = 0;
prd[3] = len - 1;
}
s->utrd[0] = c->dw0;
s->utrd[1] = 0;
s->utrd[2] = OCS_INVALID;
s->utrd[3] = 0;
s->utrd[4] = (uint32_t)s->ucd_pa;
s->utrd[5] = (uint32_t)(s->ucd_pa >> 32);
s->utrd[6] = ((UCD_RESPONSE / 4) << 16) | (UCD_RESPONSE / 4); /* response: offset, length in dwords */
s->utrd[7] = ((UCD_PRDT / 4) << 16) | n; /* PRD table: offset in dwords, entries */
__asm__ volatile ("dsb sy" ::: "memory");
for (int i = 0; i < 8; i++) {
back[i] = s->ucd[i];
}
if (memcmp(back, c->upiu, sizeof(back)) != 0 || s->utrd[0] != c->dw0 ||
!ufshc_permitted((const uint8_t *)back, s->utrd[0])) {
hc_say("ufs: native: slot %u does not hold what was put in it: not rung\n", s->number);
hc_refused++;
return EPERM;
}
return 0;
}
/* The doorbell: only while this kernel's list is the one in, running, and the loader not taking it back */
static int
hc_ring(struct hc_slot *s)
{
uint32_t bit = 1u << s->number;
boolean_t istate;
bool ok;
lck_mtx_lock(&hc_lock);
istate = ml_set_interrupts_enabled(FALSE);
ok = hc_in && !hc_closed && !hc_dead && *hc_loader.take_back == 0 &&
hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi &&
(hc_read(UFSHC_UTRLRSR) & 1) != 0 && (hc_read(UFSHC_UTRLDBR) & bit) == 0;
if (ok) {
hc_write(UFSHC_UTRLDBR, bit);
}
ml_set_interrupts_enabled(istate);
lck_mtx_unlock(&hc_lock);
return ok ? 0 : ENXIO;
}
/*
* ufs_native_hang=N (a test of the loader's side): once N reads are done, the thread that has just
* rung a doorbell stops the kernel -- every processor spinning with interrupts off, this one
* included -- with reads in flight and this kernel's list in. The heartbeat stops, and the
* loader (or the watchdog) must take it from there.
*/
extern unsigned int cpu_broadcast_xcall(uint32_t *synch, boolean_t self_xcall, void (*func)(void *), void *parm);
static uint32_t hc_hang_after;
static volatile uint32_t hc_hanging;
static void
hc_spin_forever(__unused void *parm)
{
(void)ml_set_interrupts_enabled(FALSE);
for (;;) {
__builtin_arm_yield();
}
}
static void
hc_hang(uint32_t slot)
{
static int parm;
/* (into the loader's log directly as well: a hung kernel's own log never comes back to it) */
hc_say_late("ufs: native: ufs_native_hang=%u: the kernel stops HERE, every processor spinning with interrupts "
"off; slot %u just rung, doorbell %08x, %u reads in flight, this kernel's list in\n", hc_hang_after, slot,
hc_read(UFSHC_UTRLDBR), hc_in_flight);
cpu_broadcast_xcall(NULL, TRUE, hc_spin_forever, &parm);
}
/* Nothing more is sent from here on; whoever waits for a slot is told. */
static void
hc_off(void)
{
hc_dead = true;
lck_mtx_lock(&hc_lock);
wakeup(&hc_free);
/* those who sleep on a slot's answer look at hc_dead now, not after their own 1 s tick */
for (uint32_t i = 0; i < SLOTS; i++) {
wakeup(&hc_slots[i]);
}
lck_mtx_unlock(&hc_lock);
}
/*
* Waiting for the controller. Its interrupt stays off (UEFI's setting), so completions are seen
* by reading UTRLDBR. A waiter looks at its own bit for a short while (hc_spin_us: a small read
* is done by then), then sleeps; one poller thread looks for all who sleep and wakes each whose
* slot is done. So the processors are not all taken by threads spinning on the doorbell, which
* kept others -- one woken because the volume's lock came free, say -- from running.
*/
static volatile uint32_t hc_waiting; /* slots whose waiters sleep (hc_lock) */
static uint32_t hc_done; /* slots the poller has seen done, not yet collected (hc_lock) */
static bool hc_poller_asleep;
static uint32_t hc_spin_us = 40; /* boot-arg ufs_native_spin */
static uint64_t hc_sleeps, hc_poller_polls, hc_poller_rounds;
static void
hc_poller(__unused void *arg, __unused wait_result_t wr)
{
for (;;) {
uint32_t waiting, db, done;
lck_mtx_lock(&hc_lock);
while (hc_waiting == 0) {
hc_poller_asleep = true;
msleep((void *)&hc_waiting, &hc_lock, PRIBIO, "ufshc-poll", NULL);
}
hc_poller_asleep = false;
waiting = hc_waiting;
lck_mtx_unlock(&hc_lock);
hc_poller_rounds++;
/* until one of them is done, or another waiter comes */
while (((db = hc_read(UFSHC_UTRLDBR)) & waiting) == waiting && hc_waiting == waiting) {
hc_poller_polls++;
__builtin_arm_yield();
}
done = waiting & ~db;
if (done != 0) {
lck_mtx_lock(&hc_lock);
hc_done |= done;
hc_waiting &= ~done;
for (uint32_t m = done; m != 0; m &= m - 1) {
wakeup(&hc_slots[__builtin_ctz(m)]);
}
lck_mtx_unlock(&hc_lock);
}
}
}
/* Until the controller is done with the slot's command; then whether it went well. */
static int
hc_complete(struct hc_slot *s, const char *what, bool tur)
{
uint64_t started = mach_absolute_time(), us;
volatile const uint8_t *r = (volatile const uint8_t *)s->ucd + UCD_RESPONSE;
uint32_t ocs, bit = 1u << s->number;
uint64_t polls = 1;
while ((hc_read(UFSHC_UTRLDBR) & bit) && hc_us_since(started) < hc_spin_us) {
polls++;
__builtin_arm_yield();
}
if (hc_read(UFSHC_UTRLDBR) & bit) {
struct timespec second = { 1, 0 };
lck_mtx_lock(&hc_lock);
hc_sleeps++;
hc_waiting |= bit;
if (hc_poller_asleep) {
wakeup((void *)&hc_waiting);
}
while ((hc_done & bit) == 0) {
/*
* The path went off while this one waited (another slot of the same read timed out):
* do not wait out a fresh 10 s per slot (31 slots would hold the read for 310 s).
* A slot whose doorbell bit is already clear did finish, so it is read as done.
*/
if (hc_dead) {
hc_waiting &= ~bit;
if ((hc_read(UFSHC_UTRLDBR) & bit) == 0) {
break;
}
lck_mtx_unlock(&hc_lock);
hc_say("ufs: native: slot %u (%s) not waited for: this path is already off\n", s->number, what);
return ETIMEDOUT;
}
(void)msleep(s, &hc_lock, PRIBIO, "ufshc-io", &second);
us = hc_us_since(started);
if ((hc_done & bit) == 0 && us > TIMEOUT_MS * 1000ULL) {
hc_waiting &= ~bit;
lck_mtx_unlock(&hc_lock);
hc_say("ufs: native: slot %u (%s) not done in %u s (doorbell %08x, IS %08x, HCS %08x): "
"this path is off from now on\n", s->number, what, TIMEOUT_MS / 1000, hc_read(UFSHC_UTRLDBR),
hc_read(UFSHC_IS), hc_read(UFSHC_HCS));
hc_off();
return ETIMEDOUT;
}
}
hc_done &= ~bit;
lck_mtx_unlock(&hc_lock);
}
__asm__ volatile ("dsb sy" ::: "memory");
os_atomic_add(&hc_polls, polls, relaxed);
if (!tur) {
os_atomic_add(&hc_wait_us, hc_us_since(started), relaxed);
}
ocs = s->utrd[2] & 0xff;
if (ocs != OCS_SUCCESS || (r[0] & 0x3f) != UPIU_RESPONSE || r[3] != s->number || r[6] != 0 ||
(r[7] != 0 && !(tur && r[7] == 0x02))) {
/* sense data follows the 32 bytes of the header: its length (2), then the fixed format */
hc_say("ufs: native: slot %u (%s): OCS %x, response %02x tag %02x, response %02x status %02x, "
"sense key %x asc %02x ascq %02x\n", s->number, what, ocs, r[0], r[3], r[6], r[7],
r[34 + 2] & 0x0f, r[34 + 12], r[34 + 13]);
if (!tur) {
os_atomic_inc(&hc_errors, relaxed);
}
return EIO;
}
return 0;
}
static struct hc_slot *
hc_take(bool wait)
{
struct hc_slot *s = NULL;
uint64_t t0 = mach_absolute_time();
lck_mtx_lock(&hc_lock);
hc_lock_ticks += mach_absolute_time() - t0;
if (hc_free == 0 && wait && !hc_dead && hc_in) {
uint64_t w0 = mach_absolute_time();
hc_slot_waits++;
while (hc_free == 0 && wait && !hc_dead && hc_in) {
msleep(&hc_free, &hc_lock, PRIBIO, "ufshc", NULL);
}
hc_slot_wait_ticks += mach_absolute_time() - w0;
}
if (hc_free != 0 && hc_in && !hc_dead) {
uint32_t i = (uint32_t)__builtin_ctz(hc_free), f;
hc_free &= ~(1u << i);
s = &hc_slots[i];
f = ++hc_in_flight;
if (f > hc_most_in_flight) {
hc_most_in_flight = f;
}
hc_flight_hist[f <= 1 ? 0 : f <= 2 ? 1 : f <= 4 ? 2 : f <= 8 ? 3 : f <= 16 ? 4 : 5]++;
}
lck_mtx_unlock(&hc_lock);
return s;
}
static void
hc_give(struct hc_slot *s)
{
lck_mtx_lock(&hc_lock);
hc_free |= 1u << (s - hc_slots);
hc_in_flight--;
wakeup(&hc_free);
lck_mtx_unlock(&hc_lock);
}
/*
* Bytes [offset, offset + len) of logical unit lun into dst: the blocks that cover them, a slot's
* buffer at a time, with as many slots in flight at once as are free (at least one), the wanted
* bytes copied out of each as it is done.
*/
int
ufshc_read(uint8_t lun, uint64_t offset, uint8_t *dst, uint64_t len)
{
struct {
struct hc_slot *slot;
uint32_t lba, blocks, skip, take;
uint8_t *to;
} q[SLOTS];
uint32_t head = 0, tail = 0, per = hc_slot_buffer / BLOCK;
int error = 0;
if (!hc_usable || !hc_in || hc_dead || hc_closed) {
return ENXIO;
}
while ((len > 0 && error == 0) || head != tail) {
/* as many more reads as there are free slots (and one, waiting for it, if none is in flight) */
while (len > 0 && error == 0 && tail - head < SLOTS) {
uint64_t first = offset / BLOCK, skip = offset % BLOCK;
uint64_t blocks = (skip + len + BLOCK - 1) / BLOCK, take;
struct hc_slot *s = hc_take(head == tail);
struct hc_cmd c;
int e;
if (s == NULL) {
if (head == tail) {
error = ENXIO; /* the path went off while waiting */
}
break;
}
if (blocks > per) {
blocks = per;
}
if (first + blocks > 0xffffffffULL) {
hc_give(s);
error = EINVAL;
break;
}
take = blocks * BLOCK - skip;
if (take > len) {
take = len;
}
hc_make_read(&c, lun, (uint8_t)s->number, (uint32_t)first, (uint32_t)blocks);
/* nothing of the buffer is left in a cache to be written back over what the controller writes */
FlushPoC_DcacheRegion((vm_offset_t)s->data, (size_t)blocks * BLOCK);
e = hc_place(s, &c, (uint32_t)blocks * BLOCK);
if (e == 0) {
e = hc_ring(s);
}
if (e == 0 && hc_hang_after != 0 && hc_requests >= hc_hang_after &&
os_atomic_cmpxchg(&hc_hanging, 0, 1, relaxed)) {
hc_hang(s->number);
}
if (e != 0) {
hc_give(s);
error = e;
break;
}
q[tail % SLOTS].slot = s;
q[tail % SLOTS].lba = (uint32_t)first;
q[tail % SLOTS].blocks = (uint32_t)blocks;
q[tail % SLOTS].skip = (uint32_t)skip;
q[tail % SLOTS].take = (uint32_t)take;
q[tail % SLOTS].to = dst;
tail++;
offset += take;
dst += take;
len -= take;
}
/* the oldest one done, copied out, its slot free again */
if (head != tail) {
typeof(q[0]) *e = &q[head % SLOTS];
char what[48];
int r;
snprintf(what, sizeof(what), "LUN %u, blocks %u..%u", lun, e->lba, e->lba + e->blocks - 1);
r = hc_complete(e->slot, what, false);
if (r == 0) {
FlushPoC_DcacheRegion((vm_offset_t)e->slot->data, (size_t)e->blocks * BLOCK);
os_atomic_inc(&hc_requests, relaxed);
os_atomic_add(&hc_bytes, (uint64_t)e->blocks * BLOCK, relaxed);
if (error == 0) {
memcpy(e->to, e->slot->data + e->skip, e->take);
}
} else if (error == 0) {
error = r;
}
if (r != ETIMEDOUT) {
hc_give(e->slot); /* (a slot that never came back is not used again) */
}
head++;
}
}
return error;
}
/* ---- the proof, and the switches ---- */
/* UEFI's slot 0 holds a read (what the controller would run were it to take no notice of this list) */
static const char *
hc_uefi_slot0(uint8_t *upiu, uint32_t *dw0)
{
uint64_t ucd = (uint64_t)hc_uefi_list[5] << 32 | hc_uefi_list[4];
volatile const uint8_t *u;
*dw0 = hc_uefi_list[0];
if (ucd < hc_uefi_map_pa || ucd + 32 > hc_uefi_map_pa + hc_uefi_map_size) {
return "UEFI's command descriptor is not where it was";
}
u = (volatile const uint8_t *)(hc_uefi_map_va + (ucd - hc_uefi_map_pa));
for (int i = 0; i < 32; i++) {
upiu[i] = u[i];
}
return hc_refusal(upiu, *dw0, 0xff, true) == NULL ? NULL : "UEFI's slot 0 does not hold a read";
}
/* This kernel's slot 0: TEST UNIT READY rung, and its status must come back in this list */
static int
hc_prove(void)
{
struct hc_slot *s = &hc_slots[SLOT_PROOF];
struct hc_cmd c;
uint32_t ocs;
int e;
hc_make_tur(&c, (uint8_t)__builtin_ctz(hc_lun_mask), SLOT_PROOF);
e = hc_place(s, &c, 0);
if (e == 0) {
e = hc_ring(s);
}
if (e == 0) {
e = hc_complete(s, "the proof's TEST UNIT READY", true);
}
ocs = s->utrd[2] & 0xff;
hc_proofs++;
if (e != 0 || ocs != OCS_SUCCESS) {
hc_say("ufs: native: proof FAILED (%d): this list's slot 0 has OCS %x after its doorbell -- the controller "
"did not run it from this list\n", e, ocs);
return e != 0 ? e : EIO;
}
hc_say("ufs: native: proof: TEST UNIT READY in this list's slot 0 came back here (OCS 0, response %02x, status %02x)\n",
((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 6], ((volatile const uint8_t *)s->ucd)[UCD_RESPONSE + 7]);
return 0;
}
struct hc_put_back_result {
uint32_t db_before, db_after_clear, lo, hi, rsr, db, is, cnr, cnr_ours;
bool cleared, ok;
};
/*
* UEFI's list back into the controller: what is in flight waited for (1 s), cleared if it does
* not finish (every slot of this list is this kernel's), the list stopped, UEFI's address
* written, the list started, read back. Interrupts off, no locks, no printing (the quiesce).
*/
static void
hc_put_back(struct hc_put_back_result *r)
{
uint64_t t;
bzero(r, sizeof(*r));
for (t = mach_absolute_time(); (r->db_before = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {
__builtin_arm_yield();
}
if (r->db_before != 0 && hc_read(UFSHC_UTRLBA) == hc_list_lo && hc_read(UFSHC_UTRLBAU) == hc_list_hi) {
r->cleared = true;
hc_write(UFSHC_UTRLCLR, 0);
for (t = mach_absolute_time(); (r->db_after_clear = hc_read(UFSHC_UTRLDBR)) != 0 && hc_us_since(t) < 1000000;) {
__builtin_arm_yield();
}
}
/* the completion notifications this kernel's requests left: exactly those bits, written with ones */
r->cnr_ours = hc_read(UFSHC_UTRLCNR) & ~hc_cnr_in;
if (r->cnr_ours != 0) {
hc_write(UFSHC_UTRLCNR, r->cnr_ours);
}
hc_write(UFSHC_UTRLRSR, 0);
for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {
__builtin_arm_yield();
}
hc_write(UFSHC_UTRLBA, hc_loader.uefi_lo);
hc_write(UFSHC_UTRLBAU, hc_loader.uefi_hi);
hc_write(UFSHC_UTRLRSR, 1);
r->lo = hc_read(UFSHC_UTRLBA);
r->hi = hc_read(UFSHC_UTRLBAU);
r->rsr = hc_read(UFSHC_UTRLRSR);
r->db = hc_read(UFSHC_UTRLDBR);
r->is = hc_read(UFSHC_IS);
r->cnr = hc_read(UFSHC_UTRLCNR);
r->ok = r->lo == hc_loader.uefi_lo && r->hi == hc_loader.uefi_hi && (r->rsr & 1);
if (r->ok) {
*hc_loader.list_in = 0;
(*hc_loader.switched_out)++;
hc_switch_out_count++;
}
}
static void
hc_put_back_print(const char *why, const struct hc_put_back_result *r, bool late)
{
void (*say)(const char *, ...) = late ? hc_say_late : hc_say;
if (r->cleared) {
say("ufs: native: %s: doorbell %08x did not clear in 1 s: this kernel's slots cleared (UTRLCLR), doorbell "
"then %08x\n", why, r->db_before, r->db_after_clear);
}
say("ufs: native: %s: transfer list %s UEFI's %08x:%08x (run %u, doorbell %08x; IS %08x, at the switch %08x; "
"completion %08x after clearing this kernel's %08x, at the switch %08x%s)\n", why, r->ok ? "back to" :
"NOT BACK TO", r->hi, r->lo, r->rsr, r->db, r->is, hc_is_in, r->cnr, r->cnr_ours, hc_cnr_in,
r->cnr == hc_cnr_in ? ", as it was" : ", NOT AS IT WAS");
}
/*
* This kernel's list into the controller (the caller holds the loader's mailbox: nothing is asked
* of UEFI meanwhile). prove: then slot 0's TEST UNIT READY, before anything is read.
*/
int
ufshc_switch_in(bool prove)
{
uint32_t lo, hi, db, rsr, hcs, uefi_dw0 = 0;
uint8_t uefi_upiu[32] = { 0 };
const char *no = NULL;
struct hc_put_back_result undo;
boolean_t istate;
bool undone = false;
if (!hc_usable || hc_dead || hc_closed) {
return ENXIO;
}
lck_mtx_lock(&hc_lock);
if (hc_in) {
lck_mtx_unlock(&hc_lock);
return 0;
}
istate = ml_set_interrupts_enabled(FALSE);
if (*hc_loader.take_back != 0) {
no = "the loader has taken the controller back";
} else {
no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);
}
lo = hc_read(UFSHC_UTRLBA);
hi = hc_read(UFSHC_UTRLBAU);
db = hc_read(UFSHC_UTRLDBR);
rsr = hc_read(UFSHC_UTRLRSR);
hcs = hc_read(UFSHC_HCS);
if (no == NULL) {
if (lo != hc_loader.uefi_lo || hi != hc_loader.uefi_hi) {
no = "the controller's list is not UEFI's";
} else if (db != 0) {
no = "a request in flight";
} else if ((rsr & 1) == 0 || (hcs & HCS_UTRLRDY) == 0) {
no = "the list is not running, or not ready";
}
}
if (no == NULL) {
uint64_t t;
hc_is_in = hc_read(UFSHC_IS);
hc_cnr_in = hc_read(UFSHC_UTRLCNR);
hc_write(UFSHC_UTRLRSR, 0);
for (t = mach_absolute_time(); (hc_read(UFSHC_UTRLRSR) & 1) && hc_us_since(t) < 5000;) {
__builtin_arm_yield();
}
if (hc_read(UFSHC_UTRLRSR) & 1) {
no = "the list did not stop";
hc_write(UFSHC_UTRLRSR, 1); /* (UEFI's list, running, as it was) */
} else {
hc_write(UFSHC_UTRLBA, hc_list_lo);
hc_write(UFSHC_UTRLBAU, hc_list_hi);
hc_write(UFSHC_UTRLRSR, 1);
lo = hc_read(UFSHC_UTRLBA);
hi = hc_read(UFSHC_UTRLBAU);
rsr = hc_read(UFSHC_UTRLRSR);
if (lo != hc_list_lo || hi != hc_list_hi || (rsr & 1) == 0) {
no = "the controller did not take this list";
hc_put_back(&undo);
undone = true;
} else {
hc_in = true;
*hc_loader.cnr_at_switch = hc_cnr_in;
*hc_loader.list_in = 1;
(*hc_loader.switched_in)++;
hc_switch_in_count++;
}
}
}
ml_set_interrupts_enabled(istate);
lck_mtx_unlock(&hc_lock);
if (undone) {
hc_put_back_print("undoing", &undo, false);
}
if (no != NULL) {
hc_say("ufs: native: this kernel's list not put in: %s (list %08x:%08x, doorbell %08x, run %08x, HCS %08x; UEFI's "
"slot 0 %08x, CDB %02x)\n", no, hi, lo, db, rsr, hcs, uefi_dw0, uefi_upiu[16]);
return EIO;
}
hc_say("ufs: native: transfer list switched from UEFI's %08x:%08x to this kernel's %08x:%08x (doorbell 0, run %u; "
"IS %08x, completion %08x; UEFI's slot 0 holds a read: %08x, CDB %02x)\n", hc_loader.uefi_hi,
hc_loader.uefi_lo, hi, lo, rsr, hc_is_in, hc_cnr_in, uefi_dw0, uefi_upiu[16]);
if (prove) {
int e = hc_prove();
if (e != 0) {
ufshc_switch_out("after the failed proof");
hc_off();
return e;
}
}
return 0;
}
/* UEFI's list back (the caller holds the loader's mailbox) */
int
ufshc_switch_out(const char *why)
{
struct hc_put_back_result r;
uint64_t started;
boolean_t istate;
lck_mtx_lock(&hc_lock);
if (!hc_in) {
lck_mtx_unlock(&hc_lock);
return 0;
}
hc_in = false; /* nothing more is rung */
wakeup(&hc_free);
lck_mtx_unlock(&hc_lock);
/* what is in flight finishes (whoever waits for it sees it done) */
for (started = mach_absolute_time(); hc_read(UFSHC_UTRLDBR) != 0 && hc_us_since(started) < 2000000;) {
IOSleep(1);
}
lck_mtx_lock(&hc_lock);
istate = ml_set_interrupts_enabled(FALSE);
hc_put_back(&r);
ml_set_interrupts_enabled(istate);
lck_mtx_unlock(&hc_lock);
hc_put_back_print(why, &r, false);
if (!r.ok) {
hc_off();
return EIO;
}
return 0;
}
bool
ufshc_list_in(void)
{
return hc_in;
}
bool
ufshc_available(void)
{
return hc_usable && !hc_dead && !hc_closed;
}
void
ufshc_set_luns(uint32_t mask)
{
hc_lun_mask = mask & 0xff;
}
static void
hc_report(const char *when, bool late)
{
(late ? hc_say_late : hc_say)("ufs: native: %s: %llu reads, %llu errors, %llu bytes, %llu ms waiting (summed over reads), at most %llu "
"in flight, %llu refused, list in %llu times and out %llu times, %llu proofs%s\n", when, hc_requests, hc_errors,
hc_bytes, hc_wait_us / 1000, hc_most_in_flight, hc_refused, hc_switch_in_count, hc_switch_out_count, hc_proofs,
hc_dead ? "; THE PATH IS OFF" : "");
{
uint64_t wait_ns, lock_ns;
absolutetime_to_nanoseconds(hc_slot_wait_ticks, &wait_ns);
absolutetime_to_nanoseconds(hc_lock_ticks, &lock_ns);
(late ? hc_say_late : hc_say)("ufs: native: %s: %llu doorbell reads while waiting (%llu a read), %llu waits for a "
"free slot (%llu ms), %llu ms taking the slots' lock; reads put in with 1 in flight %llu, 2 %llu, 3-4 %llu, "
"5-8 %llu, 9-16 %llu, more %llu\n", when, hc_polls, hc_requests ? hc_polls / hc_requests : 0, hc_slot_waits,
wait_ns / 1000000, lock_ns / 1000000, hc_flight_hist[0], hc_flight_hist[1], hc_flight_hist[2],
hc_flight_hist[3], hc_flight_hist[4], hc_flight_hist[5]);
(late ? hc_say_late : hc_say)("ufs: native: %s: waiters spin %u us, then sleep: %llu slept; the poller %llu "
"rounds, %llu doorbell reads\n", when, hc_spin_us, hc_sleeps, hc_poller_rounds, hc_poller_polls);
}
}
void
ufshc_report(const char *when)
{
if (hc_usable) {
hc_report(when, false);
}
}
/* ---- the probe ---- */
/*
* What the controller looks like as UEFI left it, and whether this kernel may put a list of its
* own in (only == true: say so and do not use it).
*/
bool
ufshc_probe(const struct ufshc_loader *loader, bool only)
{
uint32_t cap, ver, hcs, hce, is, ie, iacr, ahit, base, baseu, db, rsr, cnr, tm_base, tm_baseu, tm_db, tm_rsr;
uint32_t uefi_dw0 = 0, slots = 31;
uint8_t uefi_upiu[32] = { 0 };
uint64_t uefi_pa;
const char *no = NULL;
vm_offset_t area;
if (!hc_whitelist_selftest()) {
return false;
}
hc_loader = *loader;
PE_parse_boot_argn("ufs_native_buffer", &hc_slot_buffer, sizeof(hc_slot_buffer));
if (hc_slot_buffer < BLOCK || hc_slot_buffer > (4u << 20) || hc_slot_buffer % BLOCK) {
hc_slot_buffer = SLOT_BUFFER_DEFAULT;
}
PE_parse_boot_argn("ufs_native_slots", &slots, sizeof(slots));
if (slots < 1 || slots > SLOTS - 1) {
slots = SLOTS - 1;
}
PE_parse_boot_argn("ufs_native_leave", &hc_leave, sizeof(hc_leave));
PE_parse_boot_argn("ufs_native_hang", &hc_hang_after, sizeof(hc_hang_after));
if (loader->hc_pa != UFSHC_PHYS || loader->area_pa == 0 || loader->slots != SLOTS || (loader->area_pa & 0x3ff) ||
loader->ucd_size < UCD_PRDT + 16 * (hc_slot_buffer / PRD_MAX + 1) || (loader->ucd_offset & 0x7f) ||
(loader->ucd_size & 0x7f) || loader->ucd_offset < SLOTS * 32 ||
loader->area_size < loader->ucd_offset + SLOTS * loader->ucd_size) {
hc_say("ufs: native: not used: the loader offers no area for a list (or not one as expected: controller %llx, "
"area %llx of %u, command descriptors at %u of %u)\n", loader->hc_pa, loader->area_pa, loader->area_size,
loader->ucd_offset, loader->ucd_size);
return false;
}
if (pmap_valid_address(loader->area_pa) || pmap_valid_address(loader->area_pa + loader->area_size - 1)) {
hc_say("ufs: native: not used: the loader's area %llx is memory this kernel manages\n", loader->area_pa);
return false;
}
hc = ml_io_map(UFSHC_PHYS, UFSHC_SIZE);
cap = hc_read(UFSHC_CAP);
ver = hc_read(UFSHC_VER);
hcs = hc_read(UFSHC_HCS);
hce = hc_read(UFSHC_HCE);
is = hc_read(UFSHC_IS);
ie = hc_read(UFSHC_IE);
iacr = hc_read(UFSHC_UTRIACR);
ahit = hc_read(UFSHC_AHIT);
base = hc_read(UFSHC_UTRLBA);
baseu = hc_read(UFSHC_UTRLBAU);
db = hc_read(UFSHC_UTRLDBR);
rsr = hc_read(UFSHC_UTRLRSR);
cnr = hc_read(UFSHC_UTRLCNR);
tm_base = hc_read(UFSHC_UTMRLBA);
tm_baseu = hc_read(UFSHC_UTMRLBAU);
tm_db = hc_read(UFSHC_UTMRLDBR);
tm_rsr = hc_read(UFSHC_UTMRLRSR);
hc_say("ufs: native: controller at %llx: CAP %08x (%u slots, 64-bit %s), VER %08x, HCDDID %08x, HCPMID %08x\n",
UFSHC_PHYS, cap, CAP_NUTRS(cap), (cap & CAP_64AS) ? "yes" : "no", ver, hc_read(UFSHC_HCDDID),
hc_read(UFSHC_HCPMID));
hc_say("ufs: native: HCE %08x, HCS %08x, IS %08x, IE %08x, UTRIACR %08x, AHIT %08x\n", hce, hcs, is, ie, iacr, ahit);
hc_say("ufs: native: transfer list %08x:%08x (the loader's note of UEFI's: %08x:%08x), doorbell %08x, run %08x, "
"completion %08x; task management list %08x:%08x, doorbell %08x, run %08x\n", baseu, base, loader->uefi_hi,
loader->uefi_lo, db, rsr, cnr, tm_baseu, tm_base, tm_db, tm_rsr);
uefi_pa = (uint64_t)baseu << 32 | base;
if ((hce & 1) == 0) {
no = "the controller is not enabled";
} else if ((hcs & (HCS_DP | HCS_UTRLRDY)) != (HCS_DP | HCS_UTRLRDY)) {
no = "no device present or the transfer list not ready";
} else if ((rsr & 1) == 0) {
no = "the transfer list is not running";
} else if (ver < 0x200 || ver >= 0x400) {
no = "a controller version other than 2.x or 3.x";
} else if (CAP_NUTRS(cap) < SLOTS) {
no = "fewer than 32 slots";
} else if ((cap & CAP_64AS) == 0) {
no = "no 64-bit addressing";
} else if (is & IS_FATAL) {
no = "an error status";
} else if (base != loader->uefi_lo || baseu != loader->uefi_hi) {
no = "the transfer list is not the one the loader noted as UEFI's";
} else if (db != 0) {
no = "a request in flight";
} else if (pmap_valid_address(uefi_pa)) {
no = "UEFI's transfer list is in memory this kernel manages";
}
if (no == NULL) {
/* UEFI's list and command descriptor (the page they are in), not cached here, only read */
hc_uefi_map_pa = uefi_pa & ~(uint64_t)PAGE_MASK;
hc_uefi_map_size = PAGE_SIZE;
hc_uefi_map_va = ml_io_map_unmappable((vm_offset_t)hc_uefi_map_pa, PAGE_SIZE, VM_WIMG_WCOMB);
hc_uefi_list = (volatile uint32_t *)(hc_uefi_map_va + (uefi_pa - hc_uefi_map_pa));
hc_say("ufs: native: UEFI's slot 0: %08x %08x %08x %08x %08x %08x %08x %08x\n", hc_uefi_list[0], hc_uefi_list[1],
hc_uefi_list[2], hc_uefi_list[3], hc_uefi_list[4], hc_uefi_list[5], hc_uefi_list[6], hc_uefi_list[7]);
no = hc_uefi_slot0(uefi_upiu, &uefi_dw0);
hc_say("ufs: native: UEFI's last command: UPIU %02x %02x %02x %02x, CDB %02x %02x %02x %02x %02x %02x %02x %02x "
"%02x %02x\n", uefi_upiu[0], uefi_upiu[1], uefi_upiu[2], uefi_upiu[3], uefi_upiu[16], uefi_upiu[17],
uefi_upiu[18], uefi_upiu[19], uefi_upiu[20], uefi_upiu[21], uefi_upiu[22], uefi_upiu[23], uefi_upiu[24],
uefi_upiu[25]);
}
if (no != NULL) {
hc_say("ufs: native: not used: %s; the storage is read through the loader\n", no);
return false;
}
if (only) {
hc_say("ufs: native: as expected; ufs_native=2 asks for no more than this look\n");
return false;
}
/* this kernel's list and command descriptors, in the loader's area; every slot TEST UNIT READY to begin with */
area = ml_io_map_unmappable((vm_offset_t)(loader->area_pa & ~(uint64_t)PAGE_MASK),
(vm_size_t)round_page((loader->area_pa & PAGE_MASK) + loader->area_size), VM_WIMG_WCOMB) +
(vm_offset_t)(loader->area_pa & PAGE_MASK);
hc_list_lo = (uint32_t)loader->area_pa;
hc_list_hi = (uint32_t)(loader->area_pa >> 32);
for (uint32_t i = 0; i < SLOTS; i++) {
struct hc_slot *s = &hc_slots[i];
struct hc_cmd c;
s->number = i;
s->utrd = (volatile uint32_t *)(area + 32 * i);
s->ucd = (volatile uint32_t *)(area + loader->ucd_offset + i * loader->ucd_size);
s->ucd_pa = loader->area_pa + loader->ucd_offset + i * loader->ucd_size;
if (i >= 1 && i <= slots) {
vm_offset_t data = 0;
if (kmem_alloc_contig(kernel_map, &data, round_page(hc_slot_buffer), PAGE_MASK, 0, 0,
KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
hc_say("ufs: native: not used: no memory for the slots' buffers\n");
return false;
}
s->data = (uint8_t *)data;
s->data_pa = kvtophys(data);
FlushPoC_DcacheRegion(data, hc_slot_buffer);
}
hc_make_tur(&c, 0, (uint8_t)i);
if (hc_place(s, &c, 0) != 0) {
hc_say("ufs: native: not used: slot %u would not take its TEST UNIT READY\n", i);
return false;
}
}
PE_parse_boot_argn("ufs_native_spin", &hc_spin_us, sizeof(hc_spin_us));
if (hc_spin_us > 5000) {
hc_spin_us = 5000;
}
{
thread_t th;
if (kernel_thread_start(hc_poller, NULL, &th) != KERN_SUCCESS) {
hc_say("ufs: native: not used: no poller thread\n");
return false;
}
thread_deallocate(th);
}
hc_nslots = slots;
hc_free = (uint32_t)(((1ULL << (slots + 1)) - 1) & ~1ULL); /* slots 1..n */
hc_usable = true;
hc_say("ufs: native: usable: a list of %u slots at %llx (the loader's), reads in %u of them, %u KB each (the logical units reads may go to are set by hakios.ufs.c)%s\n",
SLOTS, loader->area_pa, slots, hc_slot_buffer >> 10, hc_leave ? "; ufs_native_leave=1: the list is left in "
"at the end, for the loader to take back" : "");
return true;
}
/*
* Before the kernel gives the phone back to the loader (loader_return, the other processors
* already stopped): nothing more is sent, what is in flight is waited for, UEFI's list put back.
* No locks (a stopped processor may hold one).
*/
void
hakios_ufs_quiesce(void)
{
struct hc_put_back_result r;
if (hc_kept_count != 0) {
hc_say_late("ufs: native: what happened in this run, again:\n");
for (uint32_t i = hc_kept_count > KEPT_LINES ? hc_kept_count - KEPT_LINES : 0; i < hc_kept_count; i++) {
const char *line = hc_kept[i % KEPT_LINES];
hc_say_late("> %s%s", line, line[0] != 0 && line[strlen(line) - 1] == '\n' ? "" : "...\n");
}
}
if (!hc_usable) {
return;
}
hc_closed = true;
__asm__ volatile ("dsb sy" ::: "memory");
hc_say_late("ufs: native: at the end: entered with doorbell %08x, %u reads of this kernel's in flight, its list %s\n",
hc_read(UFSHC_UTRLDBR), hc_in_flight, hc_in ? "in" : "out");
if (!hc_in && hc_read(UFSHC_UTRLBA) == hc_loader.uefi_lo && hc_read(UFSHC_UTRLBAU) == hc_loader.uefi_hi &&
(hc_read(UFSHC_UTRLRSR) & 1)) {
hc_say_late("ufs: native: at the end: UEFI's list is in (doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));
hc_report("at the end", true);
return;
}
hc_in = false;
if (hc_leave) {
hc_say_late("ufs: native: at the end: this kernel's list LEFT IN for the loader to take back "
"(ufs_native_leave=1; doorbell %08x)\n", hc_read(UFSHC_UTRLDBR));
hc_report("at the end", true);
return;
}
hc_put_back(&r);
hc_put_back_print("at the end", &r, true);
hc_report("at the end", true);
}