Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
UFS: more than one request in flight is not to be had from Qualcomm's driver; reading ahead loses
vcvkk48485bfec952
diff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.cindex b159519..9f85b1f 100644--- a/xnu/files/bsd/dev/redwood_ufs.c+++ b/xnu/files/bsd/dev/redwood_ufs.c@@ -90,6 +90,9 @@ static bool ufs_ready, ufs_dead;static uint64_t ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us;static uint64_t ufs_copy_us; /* copying out of the loader's buffer */static uint64_t ufs_small, ufs_medium, ufs_large; /* requests of <= 4 KB, <= 64 KB, more */+static uint64_t ufs_sequential; /* requests that start where the one before ended */+static uint32_t ufs_last_part = ~0u;+static uint64_t ufs_last_end;static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);@@ -120,6 +123,11 @@ ufs_request(uint32_t part, uint64_t lba, uint64_t count, uint8_t *dst)if (ufs_dead) {return EIO;}+ if (part == ufs_last_part && lba == ufs_last_end) {+ ufs_sequential++;+ }+ ufs_last_part = part;+ ufs_last_end = lba + count;ufs_page->ask.part = part;ufs_page->ask.lba = lba;ufs_page->ask.count = count;@@ -211,10 +219,10 @@ redwood_ufs_report(const char *when)return;}absolutetime_to_nanoseconds(ufs_page->answer.ticks, &loader_ns);- printf("ufs: %s: %llu requests (%llu of 4 KB or less, %llu to 64 KB, %llu more), %llu errors, "- "%llu bytes; waiting %llu ms, of it in ReadBlocks %llu ms; copying %llu ms\n", when,- ufs_requests, ufs_small, ufs_medium, ufs_large, ufs_errors, ufs_bytes, ufs_wait_us / 1000,- loader_ns / 1000000, ufs_copy_us / 1000);+ printf("ufs: %s: %llu requests (%llu of 4 KB or less, %llu to 64 KB, %llu more; %llu right after "+ "the one before), %llu errors, %llu bytes; waiting %llu ms, of it in ReadBlocks %llu ms; "+ "copying %llu ms\n", when, ufs_requests, ufs_small, ufs_medium, ufs_large, ufs_sequential,+ ufs_errors, ufs_bytes, ufs_wait_us / 1000, loader_ns / 1000000, ufs_copy_us / 1000);}/*diff --git a/xnuloader/efi.h b/xnuloader/efi.hindex 670b134..5100529 100644--- a/xnuloader/efi.h+++ b/xnuloader/efi.h@@ -165,6 +165,20 @@ typedef struct efi_block_io {efi_status (EFIAPI *flush_blocks)(struct efi_block_io *self);} efi_block_io;+/* Block I/O 2: the same reads, asynchronous -- a token's event is signalled when one is done */+typedef struct {+ void *event;+ efi_status transaction_status;+} efi_block_io2_token;++typedef struct efi_block_io2 {+ efi_block_media *media;+ void *reset;+ efi_status (EFIAPI *read_blocks_ex)(struct efi_block_io2 *self, uint32_t media_id, uint64_t lba,+ efi_block_io2_token *token, uint64_t size, void *buf);+ void *write_blocks_ex, *flush_blocks_ex;+} efi_block_io2;+#pragma pack(push, 1)typedef struct {uint32_t revision; uint32_t type; uint8_t system;@@ -231,6 +245,7 @@ typedef struct {#define QCOM_USB_INIT_EVENT_GUID { 0x1c0cffce, 0xfc8d, 0x4e44, { 0x8c, 0x78, 0x9c, 0x9e, 0x5b, 0x53, 0x0d, 0x36 } }#define EFI_GOP_GUID { 0x9042a9de, 0x23dc, 0x4a38, { 0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a } }#define EFI_BLOCK_IO_GUID { 0x964e5b21, 0x6459, 0x11d2, { 0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b } }+#define EFI_BLOCK_IO2_GUID { 0xa77b2472, 0xe282, 0x4e9f, { 0xa2, 0x45, 0xc2, 0xc0, 0xe2, 0x7b, 0xbc, 0xc1 } }#define EFI_PARTITION_INFO_GUID { 0x8cf2f62c, 0xbc9b, 0x4821, { 0x80, 0x8d, 0xec, 0x9e, 0xc4, 0x21, 0xa1, 0xa0 } }#define QCOM_CHARGER_EX_GUID { 0x6edc8a6d, 0x2663, 0x43cd, { 0x90, 0xff, 0x46, 0x21, 0xff, 0xd1, 0x0d, 0xf5 } }#define QCOM_CHARGER_GUID { 0x21bfb78b, 0x02ea, 0x4ab7, { 0x99, 0x40, 0xb5, 0x8a, 0x2a, 0xd6, 0x58, 0x29 } }diff --git a/xnuloader/loader.c b/xnuloader/loader.cindex 7d4c30e..f6f4774 100644--- a/xnuloader/loader.c+++ b/xnuloader/loader.c@@ -1444,6 +1444,8 @@ struct loader_block {/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */static struct loader_block block_shared __attribute__((aligned(0x2000)));+static void block_io2_probe(efi_handle h, efi_block_io *bio);+static int block_probe_requested; /* ufs_probe=1: block_io2_probe() at block_start() */static uint8_t block_buffer[BLOCK_BUFFER_SIZE] __attribute__((aligned(0x2000)));static efi_block_io *block_bios[BLOCK_MAX_PARTS];static int block_enabled;@@ -1475,6 +1477,10 @@ block_start(void)bp->block_size = bio->media->block_size;bp->blocks = bio->media->last_block + 1;block_bios[block_shared.parts++] = bio;+ if (block_probe_requested && bp->name[0] == 'd' && bp->name[1] == 'a' && bp->name[2] == 'r' &&+ bp->name[3] == 'w' && bp->name[4] == 'i' && bp->name[5] == 'n' && bp->name[6] == 0) {+ block_io2_probe(handles[i], bio);+ }}block_shared.magic = BLOCK_MAGIC;block_shared.version = BLOCK_VERSION;@@ -1487,6 +1493,85 @@ block_start(void)(uint64_t)(uintptr_t)&block_shared, block_shared.buffer_pa, (uint64_t)(BLOCK_BUFFER_SIZE >> 10));}+/*+ * Whether the storage could have more than one request in flight (ufs_probe=1): Block I/O 2+ * on the partitions, and whether its reads come back before they are done. Eight 4 KB+ * reads at places 64 MB apart in "darwin", one after another through Block I/O and then all+ * eight at once through Block I/O 2, each into its own part of the buffer. Before the kernel+ * runs (the UEFI timer still ticks), and only to say what there is.+ *+ * What it says on this phone: Block I/O 2 is there and the calls come back at once (eight in+ * 160 us, none done), but the eight are done only after 61.5 ms, against 1.0 ms one at a time+ * through Block I/O -- Qualcomm's driver finishes them from the UEFI timer, which besides does+ * not tick on core 0 while the kernel runs. So one request at a time, synchronously.+ */+static void+block_io2_probe(efi_handle h, efi_block_io *bio)+{+ static const efi_guid blk2_guid = EFI_BLOCK_IO2_GUID;+ efi_status (EFIAPI *create_event)(uint32_t, uint64_t, void *, void *, void **) =+ (efi_status (EFIAPI *)(uint32_t, uint64_t, void *, void *, void **))BS->create_event;+ efi_status (EFIAPI *check_event)(void *) = (efi_status (EFIAPI *)(void *))BS->check_event;+ static efi_block_io2_token tokens[8];+ efi_block_io2 *bio2 = NULL;+ uint64_t bs = bio->media->block_size, t0, t_sync, t_issue = 0, t_all, freq = READ_SYSREG(CNTFRQ_EL0);+ uint64_t returned_done = 0, pending, polls = 0;+ efi_status s;++ if (EFI_ERROR(BS->handle_protocol(h, &blk2_guid, (void **)&bio2)) || bio2 == NULL) {+ print("ufs: darwin: no Block I/O 2: one request at a time it is\n");+ return;+ }+ t0 = READ_SYSREG(CNTVCT_EL0);+ for (int i = 0; i < 8; i++) {+ s = bio->read_blocks(bio, bio->media->media_id, (uint64_t)i * (64ULL << 20) / bs + 7, bs,+ block_buffer + i * 65536);+ if (EFI_ERROR(s)) {+ print("ufs: probe: Block I/O read failed %x\n", s);+ return;+ }+ }+ t_sync = READ_SYSREG(CNTVCT_EL0) - t0;+ for (int i = 0; i < 8; i++) {+ tokens[i].transaction_status = 0;+ if (EFI_ERROR(create_event(0, 8 /* TPL_CALLBACK */, NULL, NULL, &tokens[i].event))) {+ print("ufs: probe: no event\n");+ return;+ }+ }+ t0 = READ_SYSREG(CNTVCT_EL0);+ for (int i = 0; i < 8; i++) {+ /* other places than the synchronous reads, so that nothing the storage kept is read again */+ s = bio2->read_blocks_ex(bio2, bio2->media->media_id, (uint64_t)i * (64ULL << 20) / bs + 1031,+ &tokens[i], bs, block_buffer + 1048576 + i * 65536);+ if (EFI_ERROR(s)) {+ print("ufs: probe: ReadBlocksEx %d failed %x\n", (uint64_t)i, s);+ return;+ }+ if (check_event(tokens[i].event) == 0) {+ returned_done++; /* already signalled when the call came back */+ }+ }+ t_issue = READ_SYSREG(CNTVCT_EL0) - t0;+ pending = 8 - returned_done;+ while (pending > 0 && polls < 50000000) {+ pending = 0;+ for (int i = 0; i < 8; i++) {+ if (tokens[i].event != NULL && check_event(tokens[i].event) != 0) {+ pending++;+ } else if (tokens[i].event != NULL) {+ BS->close_event(tokens[i].event);+ tokens[i].event = NULL;+ }+ }+ polls++;+ }+ t_all = READ_SYSREG(CNTVCT_EL0) - t0;+ print("ufs: darwin: Block I/O 2 there. 8 random 4 KB reads: one at a time %d us; at once: the calls came back "+ "after %d us (%d of them done by then), all done after %d us, %d still pending\n",+ t_sync * 1000000 / freq, t_issue * 1000000 / freq, returned_done, t_all * 1000000 / freq, pending);+}+/* one request, if there is one: called from core 0's watch loop while the kernel runs */static voidblock_poll(void)@@ -2228,6 +2313,7 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)(c[5] == 0 || c[5] == ' ')) {ufs_requested = c[4] - '0';}+ block_probe_requested |= starts_with(c, "ufs_probe=1") && (c == pi.cmdline || c[-1] == ' ');}if (battery_requested) {charger_start();