Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
/*
* The phone's storage: one block device and one character device for every GPT partition of its
* UFS (/dev/ufs_NAME and /dev/rufs_NAME, NAME as the partition table has it). Read only, but for
* the one place the data of Darwin lives (ufs_write=1, below).
*
* There is no UFS driver here, and none is needed. UEFI never exits on this port: Qualcomm's own
* UFS driver (UFSDxe) has brought up the controller, its PHY, its clocks and power, and stays
* resident, and xnuloader keeps serving the kernel from core 0 while the kernel runs on another
* core (it already does for the battery). So with ufs=2 on its command line the loader publishes a
* page (ufs_shared_pa=) with the table of partitions, a request and an answer, and a buffer; this
* driver writes a request -- which partition, which blocks -- and core 0 reads them with that
* driver's ReadBlocks into the buffer (xnuloader/loader.c, block_poll). Nothing in this kernel
* touches the storage controller.
*
* Read only wherever a write could do harm, by construction. The block devices never write: the
* strategy routine refuses, so no filesystem mounted from one can write, and opening one for
* writing is refused. With ufs_write=1 one character device alone takes writes -- /dev/rufs_darwin,
* the port's own partition -- and in it only whole blocks from UFS_WRITE_FIRST on, past the Darwin
* volume and both cryptexes that the build writes from the computer (tools/darwin-write.py); that
* is where the data volume of Darwin is kept between runs (userland/src/var_store.c). The partition
* has to be the one the port was set up with (its name, size, place and logical unit here; its
* GUIDs as well on the loader's side), and the loader refuses the same things over again and reads
* every write back before it answers. Without ufs_write=1 nothing writes at all.
*
* One request at a time (the page has one), so requests are serialized by a mutex and split into
* pieces of the buffer's size. The kernel writes only the request's cache lines and core 0 only
* the answer's (see the loader), both through a mapping that does not cache here, so neither side
* ever overwrites the other's. The data buffer, which only core 0's reads fill, is mapped cacheable
* and the part of it an answer covers is invalidated here before it is copied out.
*
* The data volume of Darwin (HFS Plus, built into this kernel: bsd/hfs) is /dev/ufs_darwin_data: darwin from
* UFS_WRITE_FIRST to its end, a block device of its own that reads and writes, there with ufs_write=1 only. Its
* writes go the same way as /dev/rufs_darwin's -- through the loader, which flushes and reads every one back before it
* answers -- so a write that has come back is on the flash, which is what the file system's journal counts on. The
* one and the other are not used in the same run: once the data volume has been opened for writing, the packed store
* (var_store) is refused, and while a saver holds the store the data volume does not open for writing.
*
* ufs_check=1 reads, once at start, the partitions there are images of in the ROM's
* firmware-update/ (bluetooth, xbl, tz, ...) through the same path and prints an FNV-1a hash of
* as many bytes as each image has: tools/ufs-survey.py compares those with the images and with
* what the loader read itself before the kernel started.
*/
#include <kern/clock.h>
#include <kern/locks.h>
#include <kern/thread.h>
#include <os/atomic_private.h>
#include <libkern/libkern.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <miscfs/devfs/devfs.h>
#include <pexpert/pexpert.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/disk.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/uio.h>
#include <sys/hakios.input.h>
#include <libkern/crc.h>
#include <vm/memory_types.h>
#include "hakios.ufshc.h"
extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
extern void IOSleep(unsigned int milliseconds);
extern void loader_return(void); /* osfmk/arm64/loader_return.c */
extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length); /* osfmk/arm64/caches_asm.s */
/* the page the loader publishes: the same layout as struct loader_block in xnuloader/loader.c */
#define BLOCK_MAGIC 0x4b434f4c42574452ULL /* "RDWBLOCK" */
#define BLOCK_VERSION 3
#define BLOCK_MAX_PARTS 160
#define BLOCK_MAX_UNITS 16
#define BLOCK_PART_UNIT 1u /* a whole logical unit, not a partition */
#define BLOCK_OP_READ 0u
#define BLOCK_OP_WRITE 1u
#define BLOCK_OP_WRITE_FAST 2u /* written only: the data volume's (the loader, BLOCK_OP_WRITE_FAST) */
#define BLOCK_OP_FLUSH 3u
#define BLOCK_CAN_FAST 1u
/*
* Where a write may go, and nowhere else: darwin, the port's own 20 GiB at the end of logical
* unit 0, and in it only past what the build puts there from the computer -- the Darwin volume
* (blocks 0..1374207) and the two cryptexes behind it (..2042367). The loader keeps the same
* bounds and checks the partition's type besides (xnuloader/loader.c, BLOCK_WRITE_FIRST). Where darwin
* begins and its exact size are the phone's (tools/userdata-carve.py puts it after userdata, whatever
* that is): at least the layout's 20 GiB.
*/
#define UFS_WRITE_FIRST 2042368ULL /* darwin's blocks of 4096 */
#define UFS_DARWIN_MIN 5242875ULL /* the first phone's darwin, in blocks of 4096 */
struct loader_block_part {
char name[36];
uint32_t block_size;
uint64_t blocks;
uint64_t start_lba; /* where it begins on its unit, in the unit's blocks */
uint16_t unit; /* the unit it is on, its number among the units; 0xffff: not known */
uint8_t lun; /* a unit's logical unit number as its device path says; 0xff: it does not */
uint8_t flags; /* BLOCK_PART_UNIT */
uint32_t reserved;
};
struct loader_block {
uint64_t magic;
uint32_t version, size;
uint32_t parts, buffer_size;
uint64_t buffer_pa;
/*
* Which partition the loader will take writes for this run (its index plus one; 0: none) and
* from which of its blocks. They sit in what was padding, so the page is the size and the
* version it was and one phone goes on being shared: a loader built before writes existed
* leaves them zero, which is no write, and a kernel built before them leaves ask.op zero,
* which is a read.
*/
uint32_t write_part_plus_one;
uint32_t write_caps; /* BLOCK_CAN_FAST; zero from a loader that has only BLOCK_OP_WRITE */
uint64_t write_first;
struct {
volatile uint32_t request;
uint32_t part;
uint64_t lba, count;
uint32_t op; /* BLOCK_OP_READ, BLOCK_OP_WRITE */
uint32_t reserved;
} ask __attribute__((aligned(128)));
struct {
volatile uint32_t answered;
uint32_t reserved;
int64_t status;
uint64_t served, failed, bytes, ticks;
} answer __attribute__((aligned(128)));
struct loader_block_part part[BLOCK_MAX_PARTS] __attribute__((aligned(128)));
/* the loader's offer of an area for this kernel's own transfer request list, and UEFI's list */
struct {
uint64_t hc_pa;
uint64_t area_pa;
uint32_t area_size, ucd_offset, ucd_size, slots;
uint32_t uefi_utrlba, uefi_utrlbau;
} native __attribute__((aligned(128)));
struct {
volatile uint32_t take_back; /* the loader's: it takes the controller back, send nothing more */
} loader_native __attribute__((aligned(128)));
struct {
volatile uint32_t list_in; /* this kernel's: its list is in */
uint32_t switched_in, switched_out;
volatile uint32_t cnr_at_switch; /* UTRLCNR when its list went in (for the loader's take-back) */
} kernel_native __attribute__((aligned(128)));
};
_Static_assert(__builtin_offsetof(struct loader_block, ask) == 128,
"the loader's page must stay the size it was: what is before ask has to fit in its padding");
#define UFS_PAGE 0x4000UL
#define UFS_TIMEOUT_MS 20000 /* core 0 answers in well under a second; this is for a dead loader */
static volatile struct loader_block *ufs_page;
static uint8_t *ufs_buffer; /* the loader's data buffer: filled by it for a read, here for a write */
static uint32_t ufs_buffer_size, ufs_parts, ufs_seq;
static uint32_t ufs_bsize[BLOCK_MAX_PARTS]; /* the block size each device is counted in (spec_open sets 512) */
static int ufs_bmajor = -1, ufs_cmajor = -1;
static bool ufs_ready, ufs_dead;
static uint64_t ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us;
static uint64_t ufs_copy_us; /* copying out of the loader's buffer */
static uint64_t ufs_small, ufs_medium, ufs_large; /* requests of <= 4 KB, <= 64 KB, more */
static uint64_t ufs_sequential; /* requests that start where the one before ended */
static uint32_t ufs_last_part = ~0u;
static uint64_t ufs_last_end;
/*
* This kernel's own UFS driver (hakios.ufshc.c), with ufs_native=1: each partition's logical
* unit and where it begins there, in bytes; reads go that way unless it fails, and then through
* the loader again. While its list is in the controller the loader is asked for nothing.
*/
static uint8_t ufs_lun[BLOCK_MAX_PARTS]; /* 0xff: not known, read through the loader */
static uint64_t ufs_lun_offset[BLOCK_MAX_PARTS];
static bool ufs_native_mapped;
static uint32_t ufs_native_luns; /* how many units are mapped (LUN 0..n-1) */
static volatile bool ufs_native_on;
static uint64_t ufs_native_fallbacks, ufs_mailbox_refused;
static LCK_GRP_DECLARE(ufs_lck_grp, "hakios-ufs");
static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);
/* One writer at a time; never held while the mailbox is, nor while user memory is touched. */
static LCK_MTX_DECLARE(ufs_write_mtx, &ufs_lck_grp);
/*
* Saving the data volume without being asked (userland/src/var_store.c).
*
* The volume is packed and written by a program, not by this kernel, so the kernel's part is to
* say when and to let only one program at a time do it:
*
* kern.hakios.var_save the handshake. The saver writes VAR_SAVE_IDLE when it starts and is
* then the one that will be asked; on the orderly way back to the loader
* (loader_return_after expired, or "xl stop") the kernel writes
* VAR_SAVE_WANTED and waits, and the saver writes VAR_SAVE_DONE when the
* volume is in the flash. Nobody registered: nothing is asked and
* nothing waited for, so a run without a saver returns as it always did.
* A panic does not come this way at all -- there is no program left to
* ask -- and then the store simply keeps what it had.
* kern.hakios.var_busy the claim. A write to /dev/rufs_darwin is refused unless the writer
* holds it, so two savers cannot interleave into one slot. It is held by
* process number, and a claimant may take it from a process that is gone
* (a saver killed in the middle leaves no lock behind for the rest of the
* run). What the one that was killed half-wrote is not lost either: the
* slot it was writing is not the one holding the newest good archive.
*/
#define VAR_SAVE_NONE 0
#define VAR_SAVE_IDLE 1
#define VAR_SAVE_WANTED 2
#define VAR_SAVE_DONE 3
#define VAR_SAVE_MS 20000 /* at most this long on the way out (var_save_ms=N) */
static volatile int ufs_var_save = VAR_SAVE_NONE;
static int ufs_var_busy_pid;
/* ufs_write=1 and the one partition it is for: everything else answers EROFS as it always did */
static int ufs_write_part = -1;
static volatile bool ufs_write_busy; /* a write of ours has this kernel's list out: not a failure */
static bool ufs_write_out, ufs_write_was_native; /* the list is out for writing, and was in before */
static uint64_t ufs_writes, ufs_write_bytes, ufs_write_errors, ufs_write_refused;
/* /dev/ufs_darwin_data: the minor number past every partition's, and the block size it is counted in */
#define UFS_DATA_MINOR BLOCK_MAX_PARTS
static uint32_t ufs_data_bsize = 4096;
static bool ufs_data_used; /* opened for writing this run: the packed store is not */
static LCK_MTX_DECLARE(ufs_data_mtx, &ufs_lck_grp); /* one write of the data volume at a time */
SYSCTL_DECL(_kern_hakios);
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_requests, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_requests,
"requests the phone's storage has answered through the loader");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_errors,
"requests the phone's storage has refused or not answered");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_bytes,
"bytes read from the phone's storage");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_wait_us, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_wait_us,
"microseconds spent waiting for core 0's answers");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_copy_us, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_copy_us,
"microseconds spent copying out of the loader's buffer");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_writes, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_writes,
"requests written into darwin and read back by the loader");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_write_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_write_bytes,
"bytes written into darwin");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_write_errors, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_write_errors,
"writes the loader refused or did not finish");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_write_refused, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_write_refused,
"writes refused here: not darwin, or not past what the build writes");
/* The handshake: 1 to register as the saver, 3 to say the volume is in the flash. 2 is the kernel's. */
static int
ufs_sysctl_var_save(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
{
int value = ufs_var_save, error;
error = sysctl_io_number(req, value, sizeof(value), &value, NULL);
if (error != 0 || req->newptr == 0) {
return error;
}
if (value == VAR_SAVE_IDLE && ufs_var_save != VAR_SAVE_WANTED) {
ufs_var_save = VAR_SAVE_IDLE; /* a saver is there; it is the one that will be asked */
} else if (value == VAR_SAVE_DONE && ufs_var_save == VAR_SAVE_WANTED) {
ufs_var_save = VAR_SAVE_DONE; /* and it has finished: the way out may go on */
} else if (value == VAR_SAVE_NONE) {
ufs_var_save = VAR_SAVE_NONE;
} else {
return EINVAL;
}
return 0;
}
SYSCTL_PROC(_kern_hakios, OID_AUTO, var_save, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
ufs_sysctl_var_save, "I", "the data volume's saver: 1 registers, 2 is the kernel asking, 3 is done");
/* The claim on the store: 1 takes it, 0 gives it back; reading says which process holds it. */
static int
ufs_sysctl_var_busy(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
{
int value = ufs_var_busy_pid, error, me = proc_selfpid();
error = sysctl_io_number(req, value, sizeof(value), &value, NULL);
if (error != 0 || req->newptr == 0) {
return error;
}
lck_mtx_lock(&ufs_write_mtx);
if (value == 1 && ufs_data_used) {
error = EBUSY; /* the data volume is mounted from darwin this run: no packed store into it */
} else if (value == 1) {
if (ufs_var_busy_pid != 0 && ufs_var_busy_pid != me) {
proc_t held = proc_find(ufs_var_busy_pid);
if (held != NULL) {
proc_rele(held);
lck_mtx_unlock(&ufs_write_mtx);
return EBUSY; /* another saver has it and is alive: one at a time */
}
printf("var_store: the claim on the store was left by process %d, which is gone\n",
ufs_var_busy_pid);
}
ufs_var_busy_pid = me;
} else if (value == 0) {
if (ufs_var_busy_pid == me || ufs_var_busy_pid == 0) {
ufs_var_busy_pid = 0;
} else {
error = EPERM;
}
} else {
error = EINVAL;
}
lck_mtx_unlock(&ufs_write_mtx);
return error;
}
SYSCTL_PROC(_kern_hakios, OID_AUTO, var_busy, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
ufs_sysctl_var_busy, "I", "the claim a writer of /dev/rufs_darwin must hold: 1 takes it, 0 gives it back");
/*
* Native blocks [lba, lba + count) of partition part, one request of at most the buffer's size:
* read into dst, or, with src, written from there. Called with ufs_lock held.
*/
static int
ufs_request(uint32_t op, uint32_t part, uint64_t lba, uint64_t count, uint8_t *dst, const uint8_t *src)
{
uint64_t bytes = count * ufs_page->part[part].block_size, started, waited_ms = 0;
uint32_t seq = ++ufs_seq;
int64_t status;
if (ufs_dead) {
return EIO;
}
if (op == BLOCK_OP_READ) {
if (part == ufs_last_part && lba == ufs_last_end) {
ufs_sequential++;
}
ufs_last_part = part;
ufs_last_end = lba + count;
} else if (op != BLOCK_OP_FLUSH) {
/*
* The data into the buffer and out of this core's cache before the request is raised:
* core 0 drops its own lines of the buffer and hands it to UEFI's driver.
*/
memcpy(ufs_buffer, src, (size_t)bytes);
FlushPoC_DcacheRegion((vm_offset_t)ufs_buffer, (size_t)bytes);
ufs_last_part = ~0u;
}
ufs_page->ask.part = part;
ufs_page->ask.lba = lba;
ufs_page->ask.count = count;
ufs_page->ask.op = op;
__asm__ volatile ("dmb sy" ::: "memory");
ufs_page->ask.request = seq;
__asm__ volatile ("dmb sy" ::: "memory");
started = mach_absolute_time();
while (ufs_page->answer.answered != seq) {
/* spin briefly for a quick answer, then sleep a millisecond at a time */
uint64_t us;
absolutetime_to_nanoseconds(mach_absolute_time() - started, &us);
us /= 1000;
if (us < 2000) {
/* the answer to a small request comes in tens of microseconds: look often */
__builtin_arm_yield();
continue;
}
IOSleep(1);
if (++waited_ms > UFS_TIMEOUT_MS) {
/* no answer: the loader is gone or stuck; nothing more is asked of it */
printf("ufs: no answer from the loader to request %u in %u s: the storage is off from now on\n",
seq, UFS_TIMEOUT_MS / 1000);
ufs_dead = true;
ufs_errors++;
return EIO;
}
}
__asm__ volatile ("dmb sy" ::: "memory");
{
uint64_t ns;
absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
ufs_wait_us += ns / 1000;
}
status = ufs_page->answer.status;
if (status != 0) {
printf("ufs: %s: blocks %llu..%llu: the loader's %s failed (status %llx)\n",
ufs_page->part[part].name, lba, lba + count - 1,
op == BLOCK_OP_READ ? "read" : op == BLOCK_OP_FLUSH ? "flush" : "write", (uint64_t)status);
ufs_errors++;
return EIO;
}
if (op != BLOCK_OP_READ) {
/* BLOCK_OP_WRITE: the loader has flushed, read the blocks back and found them the same */
if (op != BLOCK_OP_FLUSH) {
ufs_writes++;
ufs_write_bytes += bytes;
}
return 0;
}
started = mach_absolute_time();
/*
* The buffer is mapped cacheable here (copying out of an uncached mapping went at 870 MB/s,
* as slow as the storage itself). Core 0 cleans and invalidates it after every read, and this
* side only ever fills it for a write of its own and cleans its lines then; whatever this
* core may still hold of an earlier request -- a line brought in by a prefetch, say -- goes
* before the copy.
*/
FlushPoC_DcacheRegion((vm_offset_t)ufs_buffer, (size_t)bytes);
memcpy(dst, ufs_buffer, (size_t)bytes);
{
uint64_t ns;
absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
ufs_copy_us += ns / 1000;
}
ufs_requests++;
ufs_bytes += bytes;
if (bytes <= 4096) {
ufs_small++;
} else if (bytes <= 65536) {
ufs_medium++;
} else {
ufs_large++;
}
return 0;
}
/* One of this driver's devices (block or character): a request to it costs tens of microseconds */
bool
hakios_ufs_dev(dev_t dev)
{
return ufs_ready && ((ufs_bmajor >= 0 && major(dev) == ufs_bmajor) ||
(ufs_cmajor >= 0 && major(dev) == ufs_cmajor));
}
/*
* Where the time of reading the storage went, so far: waiting for core 0's answer (of which
* the loader's own clock says how long ReadBlocks itself took) and copying out of its buffer.
*/
void
hakios_ufs_report(const char *when)
{
uint64_t loader_ns = 0;
if (!ufs_ready) {
return;
}
ufshc_report(when);
absolutetime_to_nanoseconds(ufs_page->answer.ticks, &loader_ns);
printf("ufs: %s: %llu requests (%llu of 4 KB or less, %llu to 64 KB, %llu more; %llu right after "
"the one before), %llu errors, %llu bytes; waiting %llu ms, of it in ReadBlocks %llu ms; "
"copying %llu ms\n", when, ufs_requests, ufs_small, ufs_medium, ufs_large, ufs_sequential,
ufs_errors, ufs_bytes, ufs_wait_us / 1000, loader_ns / 1000000, ufs_copy_us / 1000);
if (ufs_write_part >= 0 || ufs_writes != 0 || ufs_write_refused != 0) {
printf("ufs: %s: %llu writes into darwin, %llu bytes, read back by the loader; %llu did not "
"finish, %llu refused here\n", when, ufs_writes, ufs_write_bytes, ufs_write_errors,
ufs_write_refused);
}
}
static uint64_t
ufs_part_bytes(uint32_t part)
{
return ufs_page->part[part].blocks * ufs_page->part[part].block_size;
}
/*
* Bytes [offset, offset + len) of partition part into dst, whatever their alignment, through the
* loader: the native blocks that cover them are read a buffer at a time and the wanted bytes
* copied out.
*/
static int
ufs_read_mailbox(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
{
uint64_t nbs = ufs_page->part[part].block_size, per = ufs_buffer_size / nbs;
static uint8_t *bounce;
int error = 0;
lck_mtx_lock(&ufs_lock);
/*
* Allocated under the lock: two first readers at once would both allocate (one 4 MB leak),
* and one could copy out of a buffer the other has just replaced and never filled.
*/
if (bounce == NULL) {
bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);
if (bounce == NULL) {
lck_mtx_unlock(&ufs_lock);
return ENOMEM;
}
}
if (ufshc_list_in()) {
/* this kernel's own transfer list is in the controller: UEFI's driver must not be asked to ring */
if (ufs_mailbox_refused++ < 4) {
printf("ufs: %s: not asked of the loader while this kernel's own transfer list is in\n",
ufs_page->part[part].name);
}
lck_mtx_unlock(&ufs_lock);
return EIO;
}
while (len > 0 && error == 0) {
uint64_t first = offset / nbs, skip = offset % nbs;
uint64_t count = (skip + len + nbs - 1) / nbs, take;
if (count > per) {
count = per;
}
take = count * nbs - skip;
if (take > len) {
take = len;
}
if (skip == 0 && take == count * nbs) {
error = ufs_request(BLOCK_OP_READ, part, first, count, dst, NULL); /* whole blocks: straight in */
} else {
error = ufs_request(BLOCK_OP_READ, part, first, count, bounce, NULL);
if (error == 0) {
memcpy(dst, bounce + skip, (size_t)take);
}
}
offset += take;
dst += take;
len -= take;
}
lck_mtx_unlock(&ufs_lock);
return error;
}
/* The same, through this kernel's own driver: the partition's unit, from where it begins there */
static int
ufs_read_native(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
{
if (part >= ufs_parts || ufs_lun[part] == 0xff) {
return ENXIO;
}
return ufshc_read(ufs_lun[part], ufs_lun_offset[part] + offset, dst, len);
}
/* The switches of lists, with the loader's mailbox held: nothing is asked of UEFI meanwhile */
static int
ufs_native_in(bool prove)
{
int error;
lck_mtx_lock(&ufs_lock);
/* (a loader that did not answer may still be inside UEFI's driver) */
error = ufs_dead ? EIO : ufshc_switch_in(prove);
lck_mtx_unlock(&ufs_lock);
return error;
}
static int
ufs_native_out(const char *why)
{
int error;
lck_mtx_lock(&ufs_lock);
error = ufshc_switch_out(why);
lck_mtx_unlock(&ufs_lock);
return error;
}
/* The way chosen (ufs_native=, kern.hakios.ufs_native); if this kernel's own fails, the loader's */
static int
ufs_read(uint32_t part, uint64_t offset, uint8_t *dst, uint64_t len)
{
/*
* No read leaves its partition on the native path either (the loader's path refuses it already):
* a short partition would otherwise be read together with the one that follows it. Written
* without a sum that can wrap.
*/
uint64_t part_size = ufs_part_bytes(part);
if (offset > part_size || len > part_size - offset) {
return EINVAL;
}
if (ufs_native_on && ufs_lun[part] != 0xff) {
int error = ufs_read_native(part, offset, dst, len);
if (error == 0) {
return 0;
}
if (ufs_write_busy) {
/* the list is out for a write of ours, not broken: this read goes the other way */
return ufs_read_mailbox(part, offset, dst, len);
}
if (ufs_native_on) {
ufs_native_on = false;
ufs_native_fallbacks++;
printf("ufs: %s: bytes %llu..%llu: this kernel's own read failed (%d); through the loader from now on\n",
ufs_page->part[part].name, offset, offset + len - 1, error);
}
ufs_native_out("a read failed");
}
return ufs_read_mailbox(part, offset, dst, len);
}
/*
* Whole blocks [offset, offset + len) of the one partition that takes writes, from src, through
* the loader. This kernel's own driver never writes -- its whitelist of commands has no write in
* it -- so for the length of a write its transfer list goes out and UEFI's driver, which the
* loader calls, has the controller again; reads meanwhile go through the loader too (ufs_read),
* which is slower and right. Nothing of Darwin waits on this but the writer itself: the data volume
* is in memory (ramfs) and only its copy for the next boot is written here.
*/
static int
ufs_write(uint32_t op, uint32_t part, uint64_t offset, const uint8_t *src, uint64_t len)
{
uint64_t nbs, per;
int error = 0;
if (!ufs_ready || ufs_write_part < 0 || part != (uint32_t)ufs_write_part) {
ufs_write_refused++;
return EROFS;
}
nbs = ufs_page->part[part].block_size;
if (len == 0 || offset % nbs != 0 || len % nbs != 0) {
return EINVAL; /* whole blocks only: nothing is read before a write */
}
if (offset / nbs < UFS_WRITE_FIRST || offset > ufs_part_bytes(part) ||
len > ufs_part_bytes(part) - offset) {
/* the Darwin volume, the cryptexes, and anything past the partition's end: never */
ufs_write_refused++;
return EROFS;
}
per = ufs_buffer_size / nbs;
lck_mtx_lock(&ufs_write_mtx);
/*
* The list goes out once, at the first write, and comes back when the device is closed: a
* whole data volume is written a bufferful at a time, and switching the controller's list
* for each of those would say so a hundred times over and buy nothing. ufs_native_on is
* cleared first, so that a reader about to start goes the other way of its own accord; one
* already inside this kernel's driver gets an error and is sent the other way by
* ufs_write_busy.
*/
if (!ufs_write_out) {
ufs_write_was_native = ufs_native_on;
ufs_write_busy = true;
ufs_write_out = true;
if (ufs_write_was_native) {
ufs_native_on = false;
ufs_native_out("writing the data volume into darwin");
}
}
while (len > 0 && error == 0) {
uint64_t count = len / nbs;
if (count > per) {
count = per;
}
lck_mtx_lock(&ufs_lock);
if (ufshc_list_in()) {
/* it did not go out after all: UEFI's driver must not be asked to ring */
error = EIO;
} else {
error = ufs_request(op, part, offset / nbs, count, NULL, src);
}
lck_mtx_unlock(&ufs_lock);
offset += count * nbs;
src += count * nbs;
len -= count * nbs;
}
lck_mtx_unlock(&ufs_write_mtx);
if (error != 0) {
ufs_write_errors++;
}
return error;
}
/* The writing over: this kernel's own transfer list back in, and reads through it again. */
static void
ufs_write_restore(void)
{
lck_mtx_lock(&ufs_write_mtx);
if (ufs_write_out) {
if (ufs_write_was_native && ufs_native_in(false) == 0) {
ufs_native_on = true;
}
ufs_write_busy = false;
ufs_write_out = false;
}
lck_mtx_unlock(&ufs_write_mtx);
}
/* ---- the data volume: darwin from UFS_WRITE_FIRST on ---- */
static bool
ufs_is_data(dev_t dev)
{
return ufs_ready && ufs_write_part >= 0 && (uint32_t)minor(dev) == UFS_DATA_MINOR;
}
static uint64_t
ufs_data_bytes(void)
{
return ufs_part_bytes((uint32_t)ufs_write_part) - UFS_WRITE_FIRST * 4096;
}
/*
* How the data volume writes: only written, when the loader can (BLOCK_CAN_FAST), the flash's cache flushed when the
* file system asks (DKIOCSYNCHRONIZE: HFS Plus's journal, where the order of what reaches the flash matters); else as
* /dev/rufs_darwin writes, each flushed and read back, which is safe and slow.
*/
static uint32_t
ufs_data_op(void)
{
return (ufs_page->write_caps & BLOCK_CAN_FAST) ? BLOCK_OP_WRITE_FAST : BLOCK_OP_WRITE;
}
static int
ufs_data_flush(void)
{
int error;
if (ufs_data_op() != BLOCK_OP_WRITE_FAST) {
return 0; /* every write was flushed already */
}
lck_mtx_lock(&ufs_write_mtx);
lck_mtx_lock(&ufs_lock);
/* one block in bounds, for the loader's checks; nothing of it is touched */
error = ufshc_list_in() ? EIO : ufs_request(BLOCK_OP_FLUSH, (uint32_t)ufs_write_part, UFS_WRITE_FIRST, 1, NULL, NULL);
lck_mtx_unlock(&ufs_lock);
lck_mtx_unlock(&ufs_write_mtx);
return error;
}
/*
* Bytes [offset, offset + len) of the data volume, read into p or written from it. A write that is not whole blocks
* of 4096 (the device may be counted in 512-byte ones) is made whole here: the blocks around it read, patched,
* written.
*/
static int
ufs_data_io(bool write, uint64_t offset, uint8_t *p, uint64_t len)
{
uint64_t base = UFS_WRITE_FIRST * 4096, size = ufs_data_bytes();
uint8_t *bounce;
int error = 0;
if (offset > size || len > size - offset) {
return EINVAL;
}
if (!write) {
return ufs_read((uint32_t)ufs_write_part, base + offset, p, len);
}
lck_mtx_lock(&ufs_data_mtx);
if (offset % 4096 == 0 && len % 4096 == 0) {
error = ufs_write(ufs_data_op(), (uint32_t)ufs_write_part, base + offset, p, len);
lck_mtx_unlock(&ufs_data_mtx);
return error;
}
bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);
if (bounce == NULL) {
lck_mtx_unlock(&ufs_data_mtx);
return ENOMEM;
}
while (len > 0 && error == 0) {
uint64_t first = offset & ~4095ULL, skip = offset - first;
uint64_t span = (skip + len + 4095) & ~4095ULL, take;
if (span > ufs_buffer_size) {
span = ufs_buffer_size;
}
take = MIN(span - skip, len);
error = ufs_read((uint32_t)ufs_write_part, base + first, bounce, span);
if (error == 0) {
memcpy(bounce + skip, p, (size_t)take);
error = ufs_write(ufs_data_op(), (uint32_t)ufs_write_part, base + first, bounce, span);
}
offset += take;
p += take;
len -= take;
}
kfree_data(bounce, ufs_buffer_size);
lck_mtx_unlock(&ufs_data_mtx);
return error;
}
static void
ufs_data_strategy(buf_t bp)
{
uint64_t offset = (uint64_t)buf_blkno(bp) * ufs_data_bsize, len = buf_count(bp), size = ufs_data_bytes();
caddr_t mapped;
int error;
if (offset >= size) {
buf_seterror(bp, offset == size ? 0 : EINVAL);
buf_setresid(bp, buf_count(bp));
buf_biodone(bp);
return;
}
len = MIN(len, size - offset);
/* HFS Plus's cluster I/O hands over pages with no kernel address yet: buf_map() gives them one */
if ((error = buf_map(bp, &mapped)) == 0) {
error = ufs_data_io(!(buf_flags(bp) & B_READ), offset, (uint8_t *)mapped, len);
buf_unmap(bp);
}
buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : len)));
if (error) {
buf_seterror(bp, error);
}
buf_biodone(bp);
}
static int
ufs_data_ioctl(u_long cmd, caddr_t data)
{
switch (cmd) {
case DKIOCGETBLOCKSIZE:
*(uint32_t *)data = ufs_data_bsize;
return 0;
case DKIOCSETBLOCKSIZE:
if (*(uint32_t *)data < 512 || *(uint32_t *)data > 4096 ||
(*(uint32_t *)data & (*(uint32_t *)data - 1)) != 0) {
return EINVAL;
}
ufs_data_bsize = *(uint32_t *)data;
return 0;
case DKIOCGETPHYSICALBLOCKSIZE:
*(uint32_t *)data = 4096;
return 0;
case DKIOCGETBLOCKCOUNT:
*(uint64_t *)data = ufs_data_bytes() / ufs_data_bsize;
return 0;
case DKIOCISWRITABLE:
*(uint32_t *)data = 1;
return 0;
case DKIOCGETMAXBLOCKCOUNTREAD:
case DKIOCGETMAXBLOCKCOUNTWRITE:
*(uint64_t *)data = ufs_buffer_size / ufs_data_bsize;
return 0;
case DKIOCSYNCHRONIZE:
case DKIOCSYNCHRONIZECACHE:
return ufs_data_flush();
default:
return ENOTTY;
}
}
/* ---- the devices ---- */
static int
ufs_open(dev_t dev, int flags, __unused int devtype, __unused proc_t p)
{
if (ufs_is_data(dev)) {
if (flags & FWRITE) {
lck_mtx_lock(&ufs_write_mtx);
if (ufs_var_busy_pid != 0) {
lck_mtx_unlock(&ufs_write_mtx);
return EBUSY; /* a saver is writing the packed store into darwin */
}
ufs_data_used = true;
lck_mtx_unlock(&ufs_write_mtx);
}
return 0;
}
if (!ufs_ready || (uint32_t)minor(dev) >= ufs_parts || (ufs_page->part[minor(dev)].flags & BLOCK_PART_UNIT)) {
return ENXIO;
}
if (flags & FWRITE) {
return EROFS; /* a block device of this driver never writes */
}
return 0;
}
/* The same, for the character devices: the one partition that takes writes may be opened for them. */
static int
ufs_copen(dev_t dev, int flags, int devtype, proc_t p)
{
if ((flags & FWRITE) && ufs_write_part >= 0 && (uint32_t)minor(dev) == (uint32_t)ufs_write_part) {
if (ufs_data_used) {
return EBUSY; /* the data volume is in darwin this run */
}
flags &= ~FWRITE;
}
return ufs_open(dev, flags, devtype, p);
}
static int
ufs_close(__unused dev_t dev, __unused int flags, __unused int devtype, __unused proc_t p)
{
return 0;
}
static void ufs_write_restore(void);
/* The last close of /dev/rufs_darwin ends the writing: this kernel's own transfer list goes back in. */
static int
ufs_cclose(dev_t dev, int flags, int devtype, proc_t p)
{
if (ufs_write_part >= 0 && (uint32_t)minor(dev) == (uint32_t)ufs_write_part) {
ufs_write_restore();
}
return ufs_close(dev, flags, devtype, p);
}
static void
ufs_strategy(buf_t bp)
{
uint32_t part = (uint32_t)minor(buf_device(bp));
uint64_t offset, len, size;
int error;
if (ufs_is_data(buf_device(bp))) {
ufs_data_strategy(bp);
return;
}
if (!ufs_ready || part >= ufs_parts) {
buf_seterror(bp, ENXIO);
buf_biodone(bp);
return;
}
if (!(buf_flags(bp) & B_READ)) {
buf_seterror(bp, EROFS); /* this device never writes */
buf_biodone(bp);
return;
}
size = ufs_part_bytes(part);
offset = (uint64_t)buf_blkno(bp) * ufs_bsize[part];
len = buf_count(bp);
if (offset >= size) {
buf_seterror(bp, offset == size ? 0 : EINVAL);
buf_setresid(bp, buf_count(bp));
buf_biodone(bp);
return;
}
if (offset + len > size) {
len = size - offset;
}
error = ufs_read(part, offset, (uint8_t *)buf_dataptr(bp), len);
buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : len)));
if (error) {
buf_seterror(bp, error);
}
buf_biodone(bp);
}
static int
ufs_ioctl(dev_t dev, u_long cmd, caddr_t data, __unused int fflag, __unused proc_t p)
{
uint32_t part = (uint32_t)minor(dev), native;
if (ufs_is_data(dev)) {
return ufs_data_ioctl(cmd, data);
}
if (!ufs_ready || part >= ufs_parts) {
return ENXIO;
}
native = ufs_page->part[part].block_size;
switch (cmd) {
case DKIOCGETBLOCKSIZE:
*(uint32_t *)data = ufs_bsize[part];
return 0;
case DKIOCSETBLOCKSIZE:
/*
* spec_open() sets the block size to 512 to count the device in 512-byte blocks and
* then back, and treats a refusal as a broken device; reads are by byte here, so any
* power of two from 512 to the native block size will do.
*/
if (*(uint32_t *)data < 512 || *(uint32_t *)data > native ||
(*(uint32_t *)data & (*(uint32_t *)data - 1)) != 0) {
return EINVAL;
}
ufs_bsize[part] = *(uint32_t *)data;
return 0;
case DKIOCGETPHYSICALBLOCKSIZE:
*(uint32_t *)data = native;
return 0;
case DKIOCGETBLOCKCOUNT:
*(uint64_t *)data = ufs_part_bytes(part) / ufs_bsize[part];
return 0;
case DKIOCISWRITABLE:
*(uint32_t *)data = 0; /* the block device, which is what a filesystem would ask */
return 0;
case DKIOCGETMAXBLOCKCOUNTREAD:
*(uint64_t *)data = ufs_buffer_size / ufs_bsize[part];
return 0;
default:
return ENOTTY;
}
}
static int
ufs_psize(dev_t dev)
{
uint32_t part = (uint32_t)minor(dev);
if (ufs_is_data(dev)) {
return (int)ufs_data_bsize;
}
return ufs_ready && part < ufs_parts ? (int)ufs_bsize[part] : -1;
}
/*
* A read of /dev/rufs_NAME: a bufferful at a time into memory of this kernel's and out to the
* reader with uiomove. Not physio, which hands the driver the reader's own address to copy to --
* and this kernel may not touch user memory itself (PAN): the copy has to go through uiomove.
* Nothing of this driver is held while it does, for a fault in the reader's memory can come back
* here as a read of the storage.
*/
static int
ufs_cread(dev_t dev, struct uio *uio, __unused int ioflag)
{
uint32_t part = (uint32_t)minor(dev);
uint64_t size;
uint8_t *bounce;
int error = 0;
if (!ufs_ready || part >= ufs_parts) {
return ENXIO;
}
size = ufs_part_bytes(part);
if (uio_offset(uio) < 0) {
return EINVAL;
}
bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);
if (bounce == NULL) {
return ENOMEM;
}
while (error == 0 && uio_resid(uio) > 0 && (uint64_t)uio_offset(uio) < size) {
uint64_t at = (uint64_t)uio_offset(uio), want = (uint64_t)uio_resid(uio);
if (want > ufs_buffer_size) {
want = ufs_buffer_size;
}
if (want > size - at) {
want = size - at;
}
error = ufs_read(part, at, bounce, want);
if (error == 0) {
error = uiomove((caddr_t)bounce, (int)want, uio);
}
}
kfree_data(bounce, ufs_buffer_size);
return error;
}
/*
* A write to /dev/rufs_darwin: whole blocks, out of user memory a bufferful at a time and into the
* partition. Nothing of this driver is held while user memory is touched -- a fault there can come
* back as a read of the storage -- so the bytes are taken first and written after.
*/
static int
ufs_cwrite(dev_t dev, struct uio *uio, __unused int ioflag)
{
uint32_t part = (uint32_t)minor(dev);
uint64_t nbs;
uint8_t *bounce;
int error = 0;
if (!ufs_ready || part >= ufs_parts || ufs_write_part < 0 || part != (uint32_t)ufs_write_part) {
return EROFS;
}
if (ufs_var_busy_pid != proc_selfpid()) {
/* one writer of the store at a time, and it says so first (kern.hakios.var_busy) */
ufs_write_refused++;
return EPERM;
}
nbs = ufs_page->part[part].block_size;
if (uio_offset(uio) < 0 || (uint64_t)uio_offset(uio) % nbs != 0 || (uint64_t)uio_resid(uio) % nbs != 0) {
return EINVAL;
}
bounce = kalloc_data(ufs_buffer_size, Z_WAITOK);
if (bounce == NULL) {
return ENOMEM;
}
while (error == 0 && uio_resid(uio) > 0) {
user_ssize_t want = uio_resid(uio);
off_t at = uio_offset(uio);
if (want > (user_ssize_t)ufs_buffer_size) {
want = (user_ssize_t)ufs_buffer_size;
}
error = uiomove((caddr_t)bounce, (int)want, uio);
if (error == 0) {
error = ufs_write(BLOCK_OP_WRITE, part, (uint64_t)at, bounce, (uint64_t)want);
if (error != 0) {
/* these bytes did not go in: the write is to say so, not to count them */
uio_setoffset(uio, at);
uio_setresid(uio, uio_resid(uio) + want);
}
}
}
kfree_data(bounce, ufs_buffer_size);
return error;
}
static const struct bdevsw ufs_bdevsw = {
.d_open = ufs_open,
.d_close = ufs_close,
.d_strategy = ufs_strategy,
.d_ioctl = ufs_ioctl,
.d_dump = eno_dump,
.d_psize = ufs_psize,
.d_type = D_DISK,
};
static const struct cdevsw ufs_cdevsw = {
.d_open = ufs_copen,
.d_close = ufs_cclose,
.d_read = ufs_cread,
.d_write = ufs_cwrite,
.d_ioctl = ufs_ioctl,
.d_stop = eno_stop,
.d_reset = eno_reset,
.d_ttys = NULL,
.d_select = eno_select,
.d_mmap = eno_mmap,
.d_strategy = eno_strat,
.d_reserved_1 = eno_getc,
.d_reserved_2 = eno_putc,
.d_type = D_DISK,
};
/*
* The orderly end of a run, from osfmk/arm64/loader_return.c, in the thread that is about to hand
* control back: ask the saver for the data volume and wait for it, so that a run need not have
* been told to save. Bounded, and only when a saver registered -- the loader goes on feeding the
* watchdog for a minute past the run's own budget, and the host waits a hundred seconds past it,
* so twenty seconds here is well inside both. Nothing else of the way out has happened yet: the
* GPU, the DSPs and this kernel's transfer list are still as the run had them.
*/
void
hakios_var_save_on_return(void)
{
uint32_t ms = VAR_SAVE_MS;
uint64_t waited = 0;
if (!ufs_ready || ufs_write_part < 0 || ufs_var_save != VAR_SAVE_IDLE) {
return;
}
PE_parse_boot_argn("var_save_ms", &ms, sizeof(ms));
printf("var_store: the run is ending: the data volume is asked for\n");
ufs_var_save = VAR_SAVE_WANTED;
while (ufs_var_save != VAR_SAVE_DONE && waited < ms) {
IOSleep(20);
waited += 20;
}
if (ufs_var_save == VAR_SAVE_DONE) {
/* what it made of it is in its own lines: this only says it answered */
printf("var_store: the saver has finished after %llu ms\n", waited);
} else {
printf("var_store: the saver did not answer in %u ms: the store keeps what it had\n", ms);
}
}
/* ---- the check: the partitions there are images of, hashed as the loader hashes them ---- */
static const struct {
const char *name;
uint64_t bytes;
} ufs_references[] = {
{ "bluetooth", 503808 }, { "abl", 208896 }, { "aop", 208896 }, { "xbl", 3706880 },
{ "tz", 3710976 }, { "hyp", 3653632 }, { "dsp", 67108864 }, { "modem", 214265856 },
};
static bool
ufs_is_reference(const char *name, const char *ref)
{
size_t n = strlen(ref);
return strncmp(name, ref, n) == 0 &&
(name[n] == 0 || (name[n] == '_' && (name[n + 1] == 'a' || name[n + 1] == 'b') && name[n + 2] == 0));
}
static void
ufs_check(void *arg, __unused wait_result_t wr)
{
uint8_t *chunk = kalloc_data(1 << 20, Z_WAITOK);
/*
* Not at once: what the kernel prints in its first minute or so does not come back with the
* log on the phone (the first line that does is from long after this driver has started), so
* the check waits until it will be read -- unless it runs right at the start (ufs_check=2, then
* back to the loader: the log is short).
*/
if (arg == NULL) {
IOSleep(40000);
}
printf("ufs: check: %u partitions of the phone's storage, read through %s\n", ufs_parts,
ufs_native_on ? "this kernel's own UFS driver" : "the loader's page");
for (uint32_t part = 0; chunk != NULL && part < ufs_parts; part++) {
for (unsigned r = 0; r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
uint64_t h = 0xcbf29ce484222325ULL, done = 0, started = mach_absolute_time(), ns;
uint64_t bytes = ufs_references[r].bytes;
int error = 0;
if (!ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {
continue;
}
while (done < bytes && error == 0) {
uint64_t n = bytes - done < (1 << 20) ? bytes - done : (1 << 20);
error = ufs_read(part, done, chunk, n);
for (uint64_t i = 0; error == 0 && i < n; i++) {
h = (h ^ chunk[i]) * 0x100000001b3ULL;
}
done += n;
}
absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
if (error) {
printf("ufs: check %s: read failed (%d)\n", ufs_page->part[part].name, error);
} else {
printf("ufs: check %s: fnv %llx over %llu bytes, %llu ms\n", ufs_page->part[part].name,
h, bytes, ns / 1000000);
}
}
}
if (chunk != NULL) {
kfree_data(chunk, 1 << 20);
}
printf("ufs: check done: %llu requests, %llu errors, %llu bytes, %llu ms waiting\n",
ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us / 1000);
}
/*
* ufs_bench=1: what one request through the loader costs, by size. The partition "darwin" is
* read from the front, a different stretch for every size so that nothing the storage itself may
* have kept is read twice; for each size, MB/s and microseconds a request, split into the time
* core 0 spent in ReadBlocks (its clock in the page, read before and after), the rest of waiting
* for its answer (noticing the request, answering), and copying out of its buffer.
*/
static void
ufs_bench(__unused void *arg, __unused wait_result_t wr)
{
static const uint32_t sizes[] = { 4096, 16384, 65536, 262144, 1048576, 4194304 };
uint8_t *dst = kalloc_data(ufs_buffer_size, Z_WAITOK);
uint32_t part = BLOCK_MAX_PARTS;
uint64_t base = 0;
for (uint32_t i = 0; i < ufs_parts; i++) {
if (strcmp((const char *)ufs_page->part[i].name, "darwin") == 0) {
part = i;
}
}
if (dst == NULL || part == BLOCK_MAX_PARTS) {
printf("ufs: bench: no partition darwin, or no memory\n");
return;
}
for (unsigned z = 0; z < sizeof(sizes) / sizeof(sizes[0]); z++) {
uint64_t total = sizes[z] <= 16384 ? (16ULL << 20) : (128ULL << 20), done = 0, n = 0;
uint64_t wait0 = ufs_wait_us, copy0 = ufs_copy_us, loader0 = ufs_page->answer.ticks;
uint64_t started = mach_absolute_time(), ns, loader_ns;
int error = 0;
while (done < total && error == 0) {
error = ufs_read(part, base + done, dst, sizes[z]);
done += sizes[z];
n++;
}
absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
absolutetime_to_nanoseconds(ufs_page->answer.ticks - loader0, &loader_ns);
printf("ufs: bench: %7u bytes a request: %4llu MB/s, %llu requests, %llu us each: "
"ReadBlocks %llu, the rest of waiting %llu, copying %llu%s\n", sizes[z],
ns ? (done * 1000ULL) / ns : 0, n, ns / 1000 / n, loader_ns / 1000 / n,
(ufs_wait_us - wait0 - loader_ns / 1000) / n, (ufs_copy_us - copy0) / n,
error ? " (an error)" : "");
base += total;
}
kfree_data(dst, ufs_buffer_size);
hakios_ufs_report("after the bench");
}
/* ---- this kernel's own driver: which unit is which, and whether it reads what the loader reads ---- */
static uint64_t
ufs_fnv(uint64_t h, const uint8_t *p, uint64_t n)
{
for (uint64_t i = 0; i < n; i++) {
h = (h ^ p[i]) * 0x100000001b3ULL;
}
return h;
}
static uint32_t
ufs_entry(const char *name)
{
for (uint32_t i = 0; i < ufs_parts; i++) {
if (strcmp((const char *)ufs_page->part[i].name, name) == 0) {
return i;
}
}
return BLOCK_MAX_PARTS;
}
/*
* For the kernel's own drivers (the modem's file systems, drivers/modem/hakios.mpss.c): bytes
* [offset, offset + len) of the partition of that name (or of its _a or _b slot), and how big it
* is. ENXIO without the storage, ENOENT without the partition; nothing is written.
*/
int
hakios_ufs_read_partition(const char *name, uint64_t offset, uint8_t *dst, uint64_t len, uint64_t *bytes)
{
uint32_t part;
if (ufs_page == NULL) {
return ENXIO;
}
part = ufs_entry(name);
for (const char *slot = "_a"; part == BLOCK_MAX_PARTS && slot[0] != 0; slot = slot[1] == 'a' ? "_b" : "") {
char slotted[48];
snprintf(slotted, sizeof(slotted), "%s%s", name, slot);
part = ufs_entry(slotted);
}
if (part == BLOCK_MAX_PARTS) {
return ENOENT;
}
if (bytes != NULL) {
*bytes = ufs_part_bytes(part);
}
if (len == 0) {
return 0;
}
/* (written without a sum: offset + len wraps for an offset near 2^64) */
if (offset > ufs_part_bytes(part) || len > ufs_part_bytes(part) - offset) {
return EINVAL;
}
return ufs_read(part, offset, dst, len);
}
/*
* Which logical unit each unit the loader lists is (their device paths do not say): unit k is
* taken to be LUN k (ordinary logical units only: never a well-known one), and it is so only if
* block 1 (the GPT header, with the unit's own GUID in it) read through the loader
* is the same as block 1 of that LUN read here; no other LUN is read. The loader's reads are made
* first, with UEFI's list in; then this kernel's list goes in (with its proof) and stays in.
*/
static bool
ufs_native_map(bool all_units)
{
uint8_t *mine = kalloc_data(4096, Z_WAITOK), *theirs = kalloc_data(BLOCK_MAX_UNITS * 4096, Z_WAITOK);
uint32_t units[BLOCK_MAX_UNITS], n = 0, mask = 0, darwin = ufs_entry("darwin");
bool good = mine != NULL && theirs != NULL;
memset(ufs_lun, 0xff, sizeof(ufs_lun));
for (uint32_t i = 0; good && i < ufs_parts && n < BLOCK_MAX_UNITS; i++) {
volatile struct loader_block_part *e = &ufs_page->part[i];
if ((e->flags & BLOCK_PART_UNIT) == 0 || e->unit >= 8) {
continue;
}
if (!all_units && (darwin == BLOCK_MAX_PARTS || ufs_page->part[darwin].unit != e->unit)) {
continue;
}
if (e->block_size != 4096 || ufs_read_mailbox(i, 4096, theirs + n * 4096, 4096) != 0 ||
memcmp(theirs + n * 4096, "EFI PART", 8) != 0) {
printf("ufs: native: %s: no GPT header to know it by (block size %u)\n", e->name, e->block_size);
good = false;
break;
}
units[n++] = i;
mask |= 1u << e->unit;
ufs_native_luns = n;
}
if (good && !all_units && (n != 1 || ufs_page->part[units[0]].unit != 0)) {
printf("ufs: native: darwin is not on the first unit the loader lists: not LUN 0, not read here\n");
good = false;
}
ufshc_set_luns(mask);
if (good && ufs_native_in(true) != 0) {
good = false;
}
for (uint32_t k = 0; good && k < n; k++) {
volatile struct loader_block_part *e = &ufs_page->part[units[k]];
if (ufshc_read((uint8_t)e->unit, 4096, mine, 4096) != 0 || memcmp(mine, theirs + k * 4096, 4096) != 0) {
printf("ufs: native: %s: LUN %u's block 1 is NOT what the loader reads as its block 1\n", e->name, e->unit);
good = false;
} else {
printf("ufs: native: %s is LUN %u: its GPT header read both ways is the same\n", e->name, e->unit);
}
}
if (!good) {
ufs_native_out("the units could not be told");
memset(ufs_lun, 0xff, sizeof(ufs_lun));
} else {
for (uint32_t i = 0; i < ufs_parts; i++) {
volatile struct loader_block_part *e = &ufs_page->part[i];
if (e->unit < 8 && (mask & (1u << e->unit)) && e->block_size == 4096) {
ufs_lun[i] = (uint8_t)e->unit;
ufs_lun_offset[i] = e->start_lba * 4096;
}
}
}
if (mine != NULL) {
kfree_data(mine, 4096);
}
if (theirs != NULL) {
kfree_data(theirs, BLOCK_MAX_UNITS * 4096);
}
return good;
}
/*
* ufs_native_check=1: what this kernel's own driver reads is what the loader reads, byte for
* byte. Spans are read a batch at a time, first all through the loader (UEFI's list in), then
* all here (this kernel's list in, proved), and compared.
*/
struct ufs_span {
uint32_t part;
uint64_t offset, len;
};
#define UFS_CMP_BUFFER (32u << 20)
static uint8_t *ufs_cmp_x, *ufs_cmp_y;
static uint64_t ufs_cmp_mailbox_ns, ufs_cmp_native_ns, ufs_cmp_bytes, ufs_cmp_spans, ufs_cmp_differ, ufs_cmp_failed;
static bool
ufs_compare_batch(const struct ufs_span *s, uint32_t n, const char *what, bool quiet)
{
uint64_t at, t, ns, hx = 0xcbf29ce484222325ULL, hy = hx, bytes = 0, mailbox_ns, native_ns;
bool same = true;
int e = 0;
t = mach_absolute_time();
if (ufs_native_out("the loader reads, to compare") != 0) {
e = EIO;
}
at = 0;
for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {
e = ufs_read_mailbox(s[i].part, s[i].offset, ufs_cmp_x + at, s[i].len);
}
absolutetime_to_nanoseconds(mach_absolute_time() - t, &mailbox_ns);
t = mach_absolute_time();
if (e == 0 && ufs_native_in(true) != 0) {
e = EIO;
}
at = 0;
for (uint32_t i = 0; e == 0 && i < n; at += s[i].len, i++) {
e = ufs_read_native(s[i].part, s[i].offset, ufs_cmp_y + at, s[i].len);
}
absolutetime_to_nanoseconds(mach_absolute_time() - t, &native_ns);
ufs_cmp_mailbox_ns += mailbox_ns;
ufs_cmp_native_ns += native_ns;
if (e != 0) {
printf("ufs: native check: %s: a read failed (%d): STOP\n", what, e);
ufs_cmp_failed++;
return false;
}
at = 0;
for (uint32_t i = 0; i < n; i++) {
if (memcmp(ufs_cmp_x + at, ufs_cmp_y + at, s[i].len) != 0) {
uint64_t k = 0;
while (ufs_cmp_x[at + k] == ufs_cmp_y[at + k]) {
k++;
}
printf("ufs: native check: %s %s: DIFFERENT at byte %llu (%02x through the loader, %02x here): STOP\n",
ufs_page->part[s[i].part].name, what, s[i].offset + k, ufs_cmp_x[at + k], ufs_cmp_y[at + k]);
same = false;
ufs_cmp_differ++;
}
hx = ufs_fnv(hx, ufs_cmp_x + at, s[i].len);
hy = ufs_fnv(hy, ufs_cmp_y + at, s[i].len);
at += s[i].len;
bytes += s[i].len;
}
ufs_cmp_spans += n;
ufs_cmp_bytes += bytes;
if (!quiet || !same) {
printf("ufs: native check: %s %s: %u span(s), %llu bytes from %llu: %s (fnv %llx / %llx); loader %llu ms, here "
"%llu ms (switches included)\n", ufs_page->part[s[0].part].name, what, n, bytes, s[0].offset,
same ? "the same" : "NOT THE SAME", hx, hy, mailbox_ns / 1000000, native_ns / 1000000);
}
return same;
}
/* Bytes [offset, offset + len) of an entry, compared a buffer at a time; FNV-1a of the first ref bytes too */
static bool
ufs_compare(uint32_t part, uint64_t offset, uint64_t len, uint64_t ref, const char *what)
{
uint64_t done = 0, href = 0xcbf29ce484222325ULL;
bool same = true;
while (done < len && same) {
struct ufs_span s = { part, offset + done, len - done < UFS_CMP_BUFFER ? len - done : UFS_CMP_BUFFER };
same = ufs_compare_batch(&s, 1, what, true);
if (same && ref > done) {
href = ufs_fnv(href, ufs_cmp_x, ref - done < s.len ? ref - done : s.len);
}
done += s.len;
}
printf("ufs: native check: %s %s: %llu bytes from %llu: %s%s", ufs_page->part[part].name, what, done, offset,
same ? "the same" : "NOT THE SAME", ref ? "" : "\n");
if (ref) {
printf("; fnv %llx over the first %llu bytes (the loader's own survey hashes them)\n", href, ref);
}
return same;
}
static void
ufs_native_check(void)
{
uint8_t *h = kalloc_data(4096, Z_WAITOK);
uint32_t part;
bool going = true;
ufs_cmp_x = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);
ufs_cmp_y = kalloc_data(UFS_CMP_BUFFER, Z_WAITOK);
if (h == NULL || ufs_cmp_x == NULL || ufs_cmp_y == NULL) {
printf("ufs: native check: no memory\n");
/* what was got is given back: the early return would leak it */
if (h != NULL) {
kfree_data(h, 4096);
}
if (ufs_cmp_x != NULL) {
kfree_data(ufs_cmp_x, UFS_CMP_BUFFER);
ufs_cmp_x = NULL;
}
if (ufs_cmp_y != NULL) {
kfree_data(ufs_cmp_y, UFS_CMP_BUFFER);
ufs_cmp_y = NULL;
}
return;
}
printf("ufs: native check: through the loader and through this kernel's own driver, compared\n");
/* every unit's GPT: protective MBR, header, table; the backup table and header at the end */
for (uint32_t i = 0; going && i < ufs_parts; i++) {
volatile struct loader_block_part *e = &ufs_page->part[i];
uint64_t entries_lba, count, size, table, alt, alt_entries;
uint32_t want, got;
if ((e->flags & BLOCK_PART_UNIT) == 0 || ufs_lun[i] == 0xff) {
continue;
}
if (ufs_native_out("the loader reads, to compare") != 0 || ufs_read_mailbox(i, 4096, h, 4096) != 0) {
going = false;
break;
}
entries_lba = *(uint64_t *)(h + 72);
count = *(uint32_t *)(h + 80);
size = *(uint32_t *)(h + 84);
alt = *(uint64_t *)(h + 32);
table = (count * size + 4095) / 4096;
want = *(uint32_t *)(h + 16);
*(uint32_t *)(h + 16) = 0;
/* the header size comes from the flash: crc32 over more than the 4096 bytes read would run off the buffer */
if (*(uint32_t *)(h + 12) >= 92 && *(uint32_t *)(h + 12) <= 4096) {
got = crc32(0, h, *(uint32_t *)(h + 12));
} else {
got = ~want; /* (reported as WRONG below) */
}
printf("ufs: native check: %s (LUN %u): GPT header crc %s (%x), table at block %llu, %llu entries of %llu bytes, "
"backup header at block %llu\n", e->name, ufs_lun[i], got == want ? "right" : "WRONG", got, entries_lba, count,
size, alt);
going = ufs_compare(i, 0, (entries_lba + table) * 4096, 0, "GPT (MBR, header, table)");
if (!going || alt >= e->blocks || ufs_native_out("the loader reads, to compare") != 0 ||
ufs_read_mailbox(i, alt * 4096, h, 4096) != 0 || memcmp(h, "EFI PART", 8) != 0 ||
(alt_entries = *(uint64_t *)(h + 72)) > alt) {
printf("ufs: native check: %s: no backup GPT header at block %llu: STOP\n", e->name, alt);
going = false;
break;
}
going = ufs_compare(i, alt_entries * 4096, (alt + 1 - alt_entries) * 4096, 0, "backup GPT (table, header)");
}
/* the partitions there are images of, whole */
for (part = 0; going && part < ufs_parts; part++) {
if ((ufs_page->part[part].flags & BLOCK_PART_UNIT) || ufs_lun[part] == 0xff) {
continue;
}
for (unsigned r = 0; going && r < sizeof(ufs_references) / sizeof(ufs_references[0]); r++) {
if (ufs_is_reference((const char *)ufs_page->part[part].name, ufs_references[r].name)) {
going = ufs_compare(part, 0, ufs_part_bytes(part), ufs_references[r].bytes, "whole");
}
}
}
/* super's geometry and metadata */
if (going && (part = ufs_entry("super")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {
going = ufs_compare(part, 0, 8 << 20, 0, "first 8 MB (geometry, metadata)");
}
/* darwin: 32 spread megabytes, 256 scattered blocks, 32 pieces that begin and end inside a block */
if (going && (part = ufs_entry("darwin")) < BLOCK_MAX_PARTS && ufs_lun[part] != 0xff) {
struct ufs_span s[256];
uint64_t bytes = ufs_part_bytes(part), seed = 0x9e3779b97f4a7c15ULL, total = 0;
for (uint32_t k = 0; k < 32; k++) {
s[k] = (struct ufs_span){ part, (bytes - (1 << 20)) / 32 * k / 4096 * 4096, 1 << 20 };
}
going = ufs_compare_batch(s, 32, "spread megabytes", false);
for (uint32_t k = 0; k < 256; k++) {
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes / 4096) * 4096, 4096 };
}
going = going && ufs_compare_batch(s, 256, "scattered blocks", false);
for (uint32_t k = 0; k < 32; k++) {
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
s[k] = (struct ufs_span){ part, (seed >> 16) % (bytes - (2 << 20)), 1 + (seed >> 40) % (512 << 10) };
total += s[k].len;
}
going = going && ufs_compare_batch(s, 32, "unaligned pieces", false);
}
kfree_data(h, 4096);
printf("ufs: native check %s: %llu spans, %llu bytes, %llu NOT THE SAME, %llu failed reads; through the loader %llu ms, "
"through this kernel's own driver %llu ms (list switches included)\n", going ? "done" : "STOPPED", ufs_cmp_spans,
ufs_cmp_bytes, ufs_cmp_differ, ufs_cmp_failed, ufs_cmp_mailbox_ns / 1000000, ufs_cmp_native_ns / 1000000);
ufs_native_out("after the check");
ufshc_report("after the check");
}
/*
* ufs_native_bench=1 (with ufs_native=1): what the flash gives with N reads in flight -- N
* threads each reading at random places of darwin, a size at a time -- MB/s and microseconds a
* read, for 1, 2, 4, 8, 16 and 31 threads. Then back to the loader.
*/
struct ufs_nb {
uint32_t part, size, per_thread;
uint64_t span, seed;
volatile uint32_t left;
int error;
};
static void
ufs_nb_thread(void *arg, __unused wait_result_t wr)
{
struct ufs_nb *nb = arg;
uint8_t *buf = kalloc_data(nb->size, Z_WAITOK);
uint64_t seed = os_atomic_add(&nb->seed, 0x9e3779b97f4a7c15ULL, relaxed);
for (uint32_t i = 0; buf != NULL && i < nb->per_thread; i++) {
int e;
seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
e = ufs_read_native(nb->part, (seed >> 20) % (nb->span / nb->size) * nb->size, buf, nb->size);
if (e != 0) {
nb->error = e;
}
}
if (buf != NULL) {
kfree_data(buf, nb->size);
}
lck_mtx_lock(&ufs_lock);
nb->left--;
wakeup(nb);
lck_mtx_unlock(&ufs_lock);
}
static void
ufs_native_bench(void)
{
static const uint32_t sizes[] = { 4096, 32768, 131072, 524288 };
static const uint32_t threads[] = { 1, 2, 4, 8, 16, 31 };
uint32_t part = ufs_entry("darwin");
if (part >= BLOCK_MAX_PARTS || ufs_lun[part] == 0xff) {
printf("ufs: native bench: no darwin\n");
return;
}
for (unsigned z = 0; z < sizeof(sizes) / sizeof(sizes[0]); z++) {
for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) {
struct ufs_nb nb = {
.part = part, .size = sizes[z], .span = ufs_part_bytes(part),
.per_thread = (sizes[z] <= 32768 ? 8192 : 2048) / threads[t] + 1, .seed = 12345 + z * 100 + t,
};
uint64_t started = mach_absolute_time(), ns, bytes;
nb.left = threads[t];
for (uint32_t i = 0; i < threads[t]; i++) {
thread_t th;
if (kernel_thread_start(ufs_nb_thread, &nb, &th) == KERN_SUCCESS) {
thread_deallocate(th);
} else {
lck_mtx_lock(&ufs_lock);
nb.left--;
lck_mtx_unlock(&ufs_lock);
}
}
lck_mtx_lock(&ufs_lock);
while (nb.left > 0) {
msleep(&nb, &ufs_lock, PRIBIO, "ufsbench", NULL);
}
lck_mtx_unlock(&ufs_lock);
absolutetime_to_nanoseconds(mach_absolute_time() - started, &ns);
bytes = (uint64_t)nb.per_thread * threads[t] * sizes[z];
printf("ufs: native bench: %6u bytes, %2u in flight: %4llu MB/s, %6llu reads/s, %5llu us a read%s\n",
sizes[z], threads[t], ns ? bytes * 1000 / ns : 0, ns ? (bytes / sizes[z]) * 1000000000ULL / ns : 0,
ns / 1000 * threads[t] / (bytes / sizes[z]), nb.error ? " (ERRORS)" : "");
}
}
ufshc_report("after the bench");
}
/* kern.hakios.ufs_native: 1 while reads go through this kernel's own driver; 0 or 1 may be set */
static int
ufs_sysctl_native(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
{
int value = ufs_native_on, error, changed = 0;
error = sysctl_io_number(req, value, sizeof(value), &value, &changed);
if (error != 0 || !changed) {
return error;
}
if (value != 0) {
if (!(ufs_native_mapped && ufshc_available()) || ufs_native_in(true) != 0) {
return ENXIO;
}
ufs_native_on = true;
} else {
ufs_native_on = false;
ufs_native_out("kern.hakios.ufs_native=0");
}
return 0;
}
SYSCTL_PROC(_kern_hakios, OID_AUTO, ufs_native, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
ufs_sysctl_native, "I", "1: the phone's storage is read by this kernel's own UFS driver, 0: through the loader");
SYSCTL_QUAD(_kern_hakios, OID_AUTO, ufs_native_fallbacks, CTLFLAG_RD | CTLFLAG_LOCKED, &ufs_native_fallbacks,
"times a read of this kernel's own UFS driver failed and the loader's was used instead");
/* ---- bringing it up ---- */
void
hakios_ufs_init(void)
{
uint64_t pa = 0, buffer_pa;
uint32_t check = 0, made = 0;
vm_offset_t base;
if (!PE_parse_boot_argn("ufs_shared_pa", &pa, sizeof(pa)) || pa == 0) {
return; /* the loader was not asked for ufs=2: there is nothing to read through */
}
base = ml_io_map_unmappable((vm_offset_t)(pa & ~(UFS_PAGE - 1)),
((pa & (UFS_PAGE - 1)) + sizeof(struct loader_block) + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_WCOMB);
ufs_page = (volatile struct loader_block *)(base + (pa & (UFS_PAGE - 1)));
if (ufs_page->magic != BLOCK_MAGIC || ufs_page->version != BLOCK_VERSION ||
ufs_page->size != sizeof(struct loader_block) || ufs_page->parts > BLOCK_MAX_PARTS) {
printf("ufs: the loader's page at %llx is not one this kernel knows (magic %llx, version %u, size %u)\n",
pa, ufs_page->magic, ufs_page->version, ufs_page->size);
return;
}
ufs_parts = ufs_page->parts;
ufs_buffer_size = ufs_page->buffer_size;
buffer_pa = ufs_page->buffer_pa;
ufs_buffer = (const uint8_t *)(ml_io_map_unmappable((vm_offset_t)(buffer_pa & ~(UFS_PAGE - 1)),
((buffer_pa & (UFS_PAGE - 1)) + ufs_buffer_size + UFS_PAGE - 1) & ~(UFS_PAGE - 1), VM_WIMG_DEFAULT) +
(buffer_pa & (UFS_PAGE - 1)));
ufs_seq = ufs_page->answer.answered; /* a loader that served an earlier kernel counts on from there */
ufs_bmajor = bdevsw_add(-1, &ufs_bdevsw);
ufs_cmajor = ufs_bmajor < 0 ? -1 : cdevsw_add_with_bdev(-1, &ufs_cdevsw, ufs_bmajor);
if (ufs_cmajor < 0) {
printf("ufs: no device numbers to be had\n");
return;
}
for (uint32_t i = 0; i < ufs_parts; i++) {
char name[48];
ufs_bsize[i] = ufs_page->part[i].block_size;
if (ufs_page->part[i].flags & BLOCK_PART_UNIT) {
continue; /* the units are for this kernel's own driver to be checked against */
}
snprintf(name, sizeof(name), "ufs_%s", ufs_page->part[i].name);
if (devfs_make_node(makedev(ufs_bmajor, i), DEVFS_BLOCK, UID_ROOT, GID_OPERATOR, 0640, "%s", name) == NULL) {
printf("ufs: no node for %s (the same name twice?)\n", name);
continue;
}
snprintf(name, sizeof(name), "rufs_%s", ufs_page->part[i].name);
(void)devfs_make_node(makedev(ufs_cmajor, i), DEVFS_CHAR, UID_ROOT, GID_OPERATOR, 0640, "%s", name);
made++;
}
ufs_ready = true;
printf("ufs: %u partitions of the phone's storage as /dev/ufs_NAME and /dev/rufs_NAME, read only, "
"through the loader's page at %llx and its %u KB buffer\n", made, pa, ufs_buffer_size >> 10);
/*
* ufs_write=1: /dev/rufs_darwin takes writes, past the Darwin volume and the cryptexes. darwin
* has to be the partition the port was set up with, in everything the loader's table says of
* it; the loader looks at its GUIDs besides, and refuses the same bounds of its own accord.
*/
check = 0;
if (PE_parse_boot_argn("ufs_write", &check, sizeof(check)) && check == 1) {
uint32_t offered = ufs_page->write_part_plus_one;
if (offered == 0 || offered > ufs_parts || ufs_page->write_first != UFS_WRITE_FIRST) {
printf("ufs: the loader offers no partition to write into (ufs_write=1 on its command "
"line too, and a loader that knows about writes)\n");
}
for (uint32_t i = 0; offered != 0 && i < ufs_parts; i++) {
const char *why = NULL;
if (strncmp((const char *)ufs_page->part[i].name, "darwin", 7) != 0) {
continue;
}
if (i != offered - 1) {
why = "the loader offers another partition";
} else if (ufs_page->part[i].flags & BLOCK_PART_UNIT) {
why = "it is a whole logical unit, not the partition";
} else if (ufs_page->part[i].unit != 0) {
/* the first unit UEFI lists, which this kernel's own driver reads as LUN 0 and
* has compared with the loader's reads; the device path this firmware gives
* says no logical unit number at all, so part[].lun is 0xff for every unit */
why = "it is not on the first logical unit";
} else if (ufs_page->part[i].blocks < UFS_DARWIN_MIN) {
why = "it is smaller than the layout's 20 GiB";
} else if (ufs_page->part[i].block_size != 4096) {
why = "its blocks are not 4096 bytes";
}
if (why != NULL) {
printf("ufs: darwin: no write will be asked for: %s\n", why);
continue;
}
ufs_write_part = (int)i;
}
if (ufs_write_part >= 0) {
printf("ufs: /dev/rufs_darwin takes writes, whole blocks from %llu on (%llu MB of it); "
"everything else of the storage is read only\n", UFS_WRITE_FIRST,
(ufs_page->part[ufs_write_part].blocks - UFS_WRITE_FIRST) * 4096 >> 20);
if (devfs_make_node(makedev(ufs_bmajor, UFS_DATA_MINOR), DEVFS_BLOCK, UID_ROOT, GID_OPERATOR, 0640,
"ufs_darwin_data") != NULL) {
printf("ufs: /dev/ufs_darwin_data: the same %llu MB as a block device, for the data volume; "
"%s\n", ufs_data_bytes() >> 20, ufs_data_op() == BLOCK_OP_WRITE_FAST ?
"writes flushed when the file system asks" : "every write flushed and read back (an older loader)");
}
} else {
printf("ufs: nothing takes writes: darwin is not among the partitions\n");
}
}
/*
* ufs_native=1: through this kernel's own driver instead, its own transfer list in the
* controller for the run, if all is as expected; 2: only look; 3: a dry run -- the list in and
* out again, and in once more for the end of the run to take out, no command sent.
*/
check = 0;
memset(ufs_lun, 0xff, sizeof(ufs_lun));
if (PE_parse_boot_argn("ufs_native", &check, sizeof(check)) && check != 0) {
struct ufshc_loader offer = {
.hc_pa = ufs_page->native.hc_pa, .area_pa = ufs_page->native.area_pa,
.area_size = ufs_page->native.area_size, .ucd_offset = ufs_page->native.ucd_offset,
.ucd_size = ufs_page->native.ucd_size, .slots = ufs_page->native.slots,
.uefi_lo = ufs_page->native.uefi_utrlba, .uefi_hi = ufs_page->native.uefi_utrlbau,
.take_back = &ufs_page->loader_native.take_back, .list_in = &ufs_page->kernel_native.list_in,
.switched_in = (uint32_t *)&ufs_page->kernel_native.switched_in,
.switched_out = (uint32_t *)&ufs_page->kernel_native.switched_out,
.cnr_at_switch = &ufs_page->kernel_native.cnr_at_switch,
};
uint32_t mode = check, compare = 0;
PE_parse_boot_argn("ufs_native_check", &compare, sizeof(compare));
if (!ufshc_probe(&offer, mode == 2)) {
if (mode != 2) {
printf("ufs: this kernel's own UFS driver is not used: through the loader\n");
}
} else if (mode == 3) {
printf("ufs: native: dry run: this kernel's list in and out, no command\n");
if (ufs_native_in(false) == 0) {
IOSleep(1000);
ufs_native_out("dry run");
IOSleep(500);
ufs_native_in(false);
printf("ufs: native: dry run: the list stays in for the end of the run\n");
}
} else if (ufs_native_map(true)) {
ufs_native_mapped = true;
if (compare != 0) {
/* right here, in bsd_init's thread, and then back to the loader */
ufs_native_check();
hakios_ufs_report("after the check");
loader_return();
}
ufs_native_on = true;
compare = 0;
if (PE_parse_boot_argn("ufs_native_bench", &compare, sizeof(compare)) && compare != 0) {
ufs_native_bench();
hakios_ufs_report("after the bench");
loader_return();
}
printf("ufs: the partitions (LUN 0-%u) are read by this kernel's own UFS driver, its list in the "
"controller (kern.hakios.ufs_native=0: through the loader)\n", ufs_native_luns - 1);
} else {
printf("ufs: this kernel's own UFS driver is not used: through the loader\n");
}
}
if (PE_parse_boot_argn("ufs_bench", &check, sizeof(check)) && check != 0) {
/* right here, in bsd_init's thread, and then back to the loader: nothing else is wanted */
ufs_bench(NULL, THREAD_AWAKENED);
loader_return();
}
check = 0;
if (PE_parse_boot_argn("ufs_check", &check, sizeof(check)) && check == 2) {
/* right here, whichever way the partitions are read (ufs_native=), and back to the loader */
ufs_check((void *)1, THREAD_AWAKENED);
hakios_ufs_report("after the check");
loader_return();
}
if (check == 1) {
thread_t thread;
if (kernel_thread_start(ufs_check, NULL, &thread) == KERN_SUCCESS) {
thread_deallocate(thread);
}
}
}