Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

UFS: more than one request in flight is not to be had from Qualcomm's driver; reading ahead loses

vcvkk48485bfec952
diff --git a/xnu/files/bsd/dev/redwood_ufs.c b/xnu/files/bsd/dev/redwood_ufs.c
index b159519..9f85b1f 100644
--- a/xnu/files/bsd/dev/redwood_ufs.c
+++ b/xnu/files/bsd/dev/redwood_ufs.c
@@ -90,6 +90,9 @@ static bool ufs_ready, ufs_dead;
static uint64_t ufs_requests, ufs_errors, ufs_bytes, ufs_wait_us;
static uint64_t ufs_copy_us; /* copying out of the loader's buffer */
static uint64_t ufs_small, ufs_medium, ufs_large; /* requests of <= 4 KB, <= 64 KB, more */
+static uint64_t ufs_sequential; /* requests that start where the one before ended */
+static uint32_t ufs_last_part = ~0u;
+static uint64_t ufs_last_end;
static LCK_GRP_DECLARE(ufs_lck_grp, "redwood-ufs");
static LCK_MTX_DECLARE(ufs_lock, &ufs_lck_grp);
@@ -120,6 +123,11 @@ ufs_request(uint32_t part, uint64_t lba, uint64_t count, uint8_t *dst)
if (ufs_dead) {
return EIO;
}
+ if (part == ufs_last_part && lba == ufs_last_end) {
+ ufs_sequential++;
+ }
+ ufs_last_part = part;
+ ufs_last_end = lba + count;
ufs_page->ask.part = part;
ufs_page->ask.lba = lba;
ufs_page->ask.count = count;
@@ -211,10 +219,10 @@ redwood_ufs_report(const char *when)
return;
}
absolutetime_to_nanoseconds(ufs_page->answer.ticks, &loader_ns);
- printf("ufs: %s: %llu requests (%llu of 4 KB or less, %llu to 64 KB, %llu more), %llu errors, "
- "%llu bytes; waiting %llu ms, of it in ReadBlocks %llu ms; copying %llu ms\n", when,
- ufs_requests, ufs_small, ufs_medium, ufs_large, ufs_errors, ufs_bytes, ufs_wait_us / 1000,
- loader_ns / 1000000, ufs_copy_us / 1000);
+ printf("ufs: %s: %llu requests (%llu of 4 KB or less, %llu to 64 KB, %llu more; %llu right after "
+ "the one before), %llu errors, %llu bytes; waiting %llu ms, of it in ReadBlocks %llu ms; "
+ "copying %llu ms\n", when, ufs_requests, ufs_small, ufs_medium, ufs_large, ufs_sequential,
+ ufs_errors, ufs_bytes, ufs_wait_us / 1000, loader_ns / 1000000, ufs_copy_us / 1000);
}
/*
diff --git a/xnuloader/efi.h b/xnuloader/efi.h
index 670b134..5100529 100644
--- a/xnuloader/efi.h
+++ b/xnuloader/efi.h
@@ -165,6 +165,20 @@ typedef struct efi_block_io {
efi_status (EFIAPI *flush_blocks)(struct efi_block_io *self);
} efi_block_io;
+/* Block I/O 2: the same reads, asynchronous -- a token's event is signalled when one is done */
+typedef struct {
+ void *event;
+ efi_status transaction_status;
+} efi_block_io2_token;
+
+typedef struct efi_block_io2 {
+ efi_block_media *media;
+ void *reset;
+ efi_status (EFIAPI *read_blocks_ex)(struct efi_block_io2 *self, uint32_t media_id, uint64_t lba,
+ efi_block_io2_token *token, uint64_t size, void *buf);
+ void *write_blocks_ex, *flush_blocks_ex;
+} efi_block_io2;
+
#pragma pack(push, 1)
typedef struct {
uint32_t revision; uint32_t type; uint8_t system;
@@ -231,6 +245,7 @@ typedef struct {
#define QCOM_USB_INIT_EVENT_GUID { 0x1c0cffce, 0xfc8d, 0x4e44, { 0x8c, 0x78, 0x9c, 0x9e, 0x5b, 0x53, 0x0d, 0x36 } }
#define EFI_GOP_GUID { 0x9042a9de, 0x23dc, 0x4a38, { 0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a } }
#define EFI_BLOCK_IO_GUID { 0x964e5b21, 0x6459, 0x11d2, { 0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b } }
+#define EFI_BLOCK_IO2_GUID { 0xa77b2472, 0xe282, 0x4e9f, { 0xa2, 0x45, 0xc2, 0xc0, 0xe2, 0x7b, 0xbc, 0xc1 } }
#define EFI_PARTITION_INFO_GUID { 0x8cf2f62c, 0xbc9b, 0x4821, { 0x80, 0x8d, 0xec, 0x9e, 0xc4, 0x21, 0xa1, 0xa0 } }
#define QCOM_CHARGER_EX_GUID { 0x6edc8a6d, 0x2663, 0x43cd, { 0x90, 0xff, 0x46, 0x21, 0xff, 0xd1, 0x0d, 0xf5 } }
#define QCOM_CHARGER_GUID { 0x21bfb78b, 0x02ea, 0x4ab7, { 0x99, 0x40, 0xb5, 0x8a, 0x2a, 0xd6, 0x58, 0x29 } }
diff --git a/xnuloader/loader.c b/xnuloader/loader.c
index 7d4c30e..f6f4774 100644
--- a/xnuloader/loader.c
+++ b/xnuloader/loader.c
@@ -1444,6 +1444,8 @@ struct loader_block {
/* (8 KB is as far as this target aligns; the kernel maps whole 16 KB pages around both) */
static struct loader_block block_shared __attribute__((aligned(0x2000)));
+static void block_io2_probe(efi_handle h, efi_block_io *bio);
+static int block_probe_requested; /* ufs_probe=1: block_io2_probe() at block_start() */
static uint8_t block_buffer[BLOCK_BUFFER_SIZE] __attribute__((aligned(0x2000)));
static efi_block_io *block_bios[BLOCK_MAX_PARTS];
static int block_enabled;
@@ -1475,6 +1477,10 @@ block_start(void)
bp->block_size = bio->media->block_size;
bp->blocks = bio->media->last_block + 1;
block_bios[block_shared.parts++] = bio;
+ if (block_probe_requested && bp->name[0] == 'd' && bp->name[1] == 'a' && bp->name[2] == 'r' &&
+ bp->name[3] == 'w' && bp->name[4] == 'i' && bp->name[5] == 'n' && bp->name[6] == 0) {
+ block_io2_probe(handles[i], bio);
+ }
}
block_shared.magic = BLOCK_MAGIC;
block_shared.version = BLOCK_VERSION;
@@ -1487,6 +1493,85 @@ block_start(void)
(uint64_t)(uintptr_t)&block_shared, block_shared.buffer_pa, (uint64_t)(BLOCK_BUFFER_SIZE >> 10));
}
+/*
+ * Whether the storage could have more than one request in flight (ufs_probe=1): Block I/O 2
+ * on the partitions, and whether its reads come back before they are done. Eight 4 KB
+ * reads at places 64 MB apart in "darwin", one after another through Block I/O and then all
+ * eight at once through Block I/O 2, each into its own part of the buffer. Before the kernel
+ * runs (the UEFI timer still ticks), and only to say what there is.
+ *
+ * What it says on this phone: Block I/O 2 is there and the calls come back at once (eight in
+ * 160 us, none done), but the eight are done only after 61.5 ms, against 1.0 ms one at a time
+ * through Block I/O -- Qualcomm's driver finishes them from the UEFI timer, which besides does
+ * not tick on core 0 while the kernel runs. So one request at a time, synchronously.
+ */
+static void
+block_io2_probe(efi_handle h, efi_block_io *bio)
+{
+ static const efi_guid blk2_guid = EFI_BLOCK_IO2_GUID;
+ efi_status (EFIAPI *create_event)(uint32_t, uint64_t, void *, void *, void **) =
+ (efi_status (EFIAPI *)(uint32_t, uint64_t, void *, void *, void **))BS->create_event;
+ efi_status (EFIAPI *check_event)(void *) = (efi_status (EFIAPI *)(void *))BS->check_event;
+ static efi_block_io2_token tokens[8];
+ efi_block_io2 *bio2 = NULL;
+ uint64_t bs = bio->media->block_size, t0, t_sync, t_issue = 0, t_all, freq = READ_SYSREG(CNTFRQ_EL0);
+ uint64_t returned_done = 0, pending, polls = 0;
+ efi_status s;
+
+ if (EFI_ERROR(BS->handle_protocol(h, &blk2_guid, (void **)&bio2)) || bio2 == NULL) {
+ print("ufs: darwin: no Block I/O 2: one request at a time it is\n");
+ return;
+ }
+ t0 = READ_SYSREG(CNTVCT_EL0);
+ for (int i = 0; i < 8; i++) {
+ s = bio->read_blocks(bio, bio->media->media_id, (uint64_t)i * (64ULL << 20) / bs + 7, bs,
+ block_buffer + i * 65536);
+ if (EFI_ERROR(s)) {
+ print("ufs: probe: Block I/O read failed %x\n", s);
+ return;
+ }
+ }
+ t_sync = READ_SYSREG(CNTVCT_EL0) - t0;
+ for (int i = 0; i < 8; i++) {
+ tokens[i].transaction_status = 0;
+ if (EFI_ERROR(create_event(0, 8 /* TPL_CALLBACK */, NULL, NULL, &tokens[i].event))) {
+ print("ufs: probe: no event\n");
+ return;
+ }
+ }
+ t0 = READ_SYSREG(CNTVCT_EL0);
+ for (int i = 0; i < 8; i++) {
+ /* other places than the synchronous reads, so that nothing the storage kept is read again */
+ s = bio2->read_blocks_ex(bio2, bio2->media->media_id, (uint64_t)i * (64ULL << 20) / bs + 1031,
+ &tokens[i], bs, block_buffer + 1048576 + i * 65536);
+ if (EFI_ERROR(s)) {
+ print("ufs: probe: ReadBlocksEx %d failed %x\n", (uint64_t)i, s);
+ return;
+ }
+ if (check_event(tokens[i].event) == 0) {
+ returned_done++; /* already signalled when the call came back */
+ }
+ }
+ t_issue = READ_SYSREG(CNTVCT_EL0) - t0;
+ pending = 8 - returned_done;
+ while (pending > 0 && polls < 50000000) {
+ pending = 0;
+ for (int i = 0; i < 8; i++) {
+ if (tokens[i].event != NULL && check_event(tokens[i].event) != 0) {
+ pending++;
+ } else if (tokens[i].event != NULL) {
+ BS->close_event(tokens[i].event);
+ tokens[i].event = NULL;
+ }
+ }
+ polls++;
+ }
+ t_all = READ_SYSREG(CNTVCT_EL0) - t0;
+ print("ufs: darwin: Block I/O 2 there. 8 random 4 KB reads: one at a time %d us; at once: the calls came back "
+ "after %d us (%d of them done by then), all done after %d us, %d still pending\n",
+ t_sync * 1000000 / freq, t_issue * 1000000 / freq, returned_done, t_all * 1000000 / freq, pending);
+}
+
/* one request, if there is one: called from core 0's watch loop while the kernel runs */
static void
block_poll(void)
@@ -2228,6 +2313,7 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)
(c[5] == 0 || c[5] == ' ')) {
ufs_requested = c[4] - '0';
}
+ block_probe_requested |= starts_with(c, "ufs_probe=1") && (c == pi.cmdline || c[-1] == ' ');
}
if (battery_requested) {
charger_start();