Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: read-ahead for a reader going through a file in 64 KB reads
vcvkkf7504f82d8cc
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.cindex 7cdaaf1..3761936 100644--- a/tools/apfs/apfs-tool.c+++ b/tools/apfs/apfs-tool.c@@ -586,8 +586,9 @@ cmd_census(const char *path, const char *where, bool meta_only)" bytes out, %" PRIu64 " by zlib\n", nx.inflate_calls, nx.inflate_in, nx.inflate_out,nx.inflate_fallbacks);fprintf(stderr, "apfs-tool: %" PRIu64 " requests of whole pieces, %" PRIu64 " pieces in "- "them, on %u threads at most\n", nx.batch_calls, nx.batch_pieces,- nx.parallel != NULL ? nx.parallel_max : 1);+ "them, on %u threads at most; read ahead %" PRIu64 " times, %" PRIu64 " pieces, %"+ PRIu64 " of them used\n", nx.batch_calls, nx.batch_pieces,+ nx.parallel != NULL ? nx.parallel_max : 1, nx.ra_batches, nx.ra_pieces, nx.ra_hits);fprintf(stderr, "apfs-tool: highest block touched %" PRIu64 " (%.2f GB in), of which ""file contents reach %" PRIu64 " (%.2f GB in)\n", nx.max_addr,nx.max_addr * (double)nx.block_size / 1e9, nx.max_data_addr,diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex c6f3521..245566c 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -1766,6 +1766,9 @@ apfs_cfile_fini(struct apfs_cfile *cf)if (cf->held_data != NULL) {apfs_free(cf->held_data, cf->held_cap);}+ if (cf->ra_data != NULL) {+ apfs_free(cf->ra_data, (size_t)APFS_RA_PIECES * APFS_DECMPFS_BLOCK_SIZE);+ }memset(cf, 0, sizeof(*cf));cf->held = APFS_NO_PIECE;}@@ -2259,6 +2262,8 @@ struct apfs_piece_job {uint32_t plen; /* compressed */uint8_t *dst; /* the caller's, `want` bytes */size_t want;+ bool ra; /* read ahead, not asked for: its failure fails nothing */+ int error;};#define APFS_BATCH 32@@ -2390,7 +2395,8 @@ batch_one(void *arg, uint32_t item, uint32_t worker)if (error == 0 && got != j->want) {error = EIO;}- return error;+ j->error = error;+ return j->ra ? 0 : error;}/*@@ -2435,8 +2441,12 @@ batch_run(struct apfs_vol *vol, struct apfs_scratch *sc, uint32_t n)if (workers > 1) {error = nx->parallel(nx->parallel_ctx, n, workers, batch_one, &b);} else {- for (i = 0; i < n && error == 0; i++) {- error = batch_one(&b, i, 0);+ for (i = 0; i < n; i++) {+ int e = batch_one(&b, i, 0);++ if (e != 0 && error == 0) {+ error = e;+ }}}nx_relock(nx);@@ -2470,6 +2480,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,size_t big_cap = 0;uint32_t bs = vol->nx->block_size;uint32_t nbatch = 0; /* whole pieces gathered in sc->jobs, not yet done */+ uint32_t ra_at = 0, nra = 0; /* read-ahead jobs: sc->jobs[ra_at .. ra_at + nra) */uint64_t first, last, i;int error = 0;@@ -2499,6 +2510,20 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,}from_in_piece = (off > i * APFS_DECMPFS_BLOCK_SIZE)? (size_t)(off - i * APFS_DECMPFS_BLOCK_SIZE) : 0;+ if (cf->held != i && cf->ra_count > 0 && !cf->ra_filling &&+ i >= cf->ra_first && i - cf->ra_first < cf->ra_count &&+ cf->ra_len[i - cf->ra_first] == want) {+ /* read ahead by an earlier request of this reader's */+ n = want - from_in_piece;+ if (n > len - *out) {+ n = len - *out;+ }+ memcpy(buf + *out, cf->ra_data ++ (size_t)(i - cf->ra_first) * APFS_DECMPFS_BLOCK_SIZE + from_in_piece, n);+ *out += n;+ vol->nx->ra_hits++;+ continue;+ }if (cf->held != i) {uint32_t plen = cf->piece_len[i];uint64_t poff = cf->table_base + cf->piece_off[i];@@ -2583,6 +2608,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,j->plen = plen;j->dst = buf + *out;j->want = want;+ j->ra = false;*out += want;if (nbatch == APFS_BATCH) {error = batch_run(vol, sc, nbatch);@@ -2591,6 +2617,59 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,goto out;}}+ /*+ * The last piece of this request, the one right after where this+ * reader's last request ended, and threads to spare for it (the+ * share batch_run would give it is two or more): the pieces after it+ * go into this batch as well, into ra_data.+ */+ if (i == last && i >= 1 && i == cf->ra_expect && !cf->ra_filling &&+ vol->nx->parallel != NULL && i + 1 < cf->piece_count &&+ vol->nx->parallel_max / (vol->nx->requests > 1 ?+ vol->nx->requests : 1) >= 2) {+ uint64_t k;++ if (cf->ra_data == NULL) {+ cf->ra_data = apfs_alloc((size_t)APFS_RA_PIECES *+ APFS_DECMPFS_BLOCK_SIZE);+ }+ cf->ra_count = 0;+ ra_at = nbatch;+ for (k = i + 1; cf->ra_data != NULL && k < cf->piece_count &&+ k - i <= APFS_RA_PIECES && nbatch < APFS_BATCH; k++) {+ size_t kwant = APFS_DECMPFS_BLOCK_SIZE;+ uint32_t kplen = cf->piece_len[k];+ uint64_t kpoff = cf->table_base + cf->piece_off[k];+ struct apfs_run kruns[APFS_MAX_RUNS];+ uint32_t knruns = 0;+ uint64_t kcovered = 0;++ if ((k + 1) * APFS_DECMPFS_BLOCK_SIZE > cf->size) {+ kwant = (size_t)(cf->size - k * APFS_DECMPFS_BLOCK_SIZE);+ }+ if (kplen > APFS_PIECE_CAP || kpoff >= cf->rsrc_size ||+ kplen > cf->rsrc_size - kpoff ||+ stream_plan(vol, cf->rsrc_id, kpoff, kplen, kruns,+ APFS_MAX_RUNS, &knruns, &kcovered) != 0 ||+ kcovered != kplen || knruns != 1 || kruns[0].at != 0 ||+ kruns[0].len != kplen) {+ break; /* not one for a batch: the read-ahead stops here */+ }+ j = &sc->jobs[nbatch++];+ j->addr = kruns[0].addr;+ j->skew = kruns[0].skew;+ j->plen = kplen;+ j->dst = cf->ra_data + (size_t)(k - i - 1) * APFS_DECMPFS_BLOCK_SIZE;+ j->want = kwant;+ j->ra = true;+ cf->ra_len[k - i - 1] = (uint32_t)kwant;+ nra++;+ }+ if (nra > 0) {+ cf->ra_filling = true;+ cf->ra_first = i + 1;+ }+ }continue;}if (!raw_type && !direct && sc->mine == NULL) {@@ -2686,7 +2765,31 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,if (nbatch > 0) {error = batch_run(vol, sc, nbatch);}+ if (nra > 0) {+ uint32_t k;++ cf->ra_count = nra;+ for (k = 0; k < nra; k++) {+ if (sc->jobs[ra_at + k].error != 0) {+ cf->ra_count = k; /* what came before the first that failed */+ break;+ }+ }+ vol->nx->ra_batches++;+ vol->nx->ra_pieces += cf->ra_count;+ cf->ra_filling = false;+ nra = 0;+ }+ if (error == 0) {+ cf->ra_expect = ((off + len) % APFS_DECMPFS_BLOCK_SIZE == 0)+ ? (off + len) / APFS_DECMPFS_BLOCK_SIZE : APFS_NO_PIECE;+ }out:+ if (nra > 0) {+ /* given up on before the batch ran: nothing in ra_data is good */+ cf->ra_count = 0;+ cf->ra_filling = false;+ }if (big != NULL) {apfs_free(big, big_cap);}diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex 34e918b..dce5f70 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -75,6 +75,9 @@ apfs_ticks(void)/* Room for a symbolic link's target (Darwin's own limit is 1024 bytes; it is kept in an xattr) */#define APFS_LINK_MAX 4096+/* How many pieces a sequential reader's read-ahead decompresses at a time */+#define APFS_RA_PIECES 7+/** The device. read() must fill the whole buffer or return an error; offsets and lengths* are always whole blocks of the container.@@ -189,6 +192,7 @@ struct apfs_nx {*/uint32_t requests;uint64_t batch_calls, batch_pieces; /* how many requests used it, for how much */+ uint64_t ra_batches, ra_pieces, ra_hits; /* read-ahead: filled, and used */};/* A volume inside the container. */@@ -322,6 +326,21 @@ struct apfs_cfile {uint8_t *held_data;size_t held_len;size_t held_cap; /* 0 when this file is not worth holding */++ /*+ * Read-ahead. A reader with threads to spare (alone on the volume, or one of two)+ * that asks for the piece right after the one its last read ended on is taken to be+ * reading the file through: the pieces+ * after it are decompressed in the same batch, on the other threads, into ra_data,+ * and its next reads are copies out of that. All of it happens inside the request+ * that asked; ra_filling keeps everyone else off ra_data while the lock is let go of.+ */+ uint64_t ra_expect; /* the piece the last whole-piece read ended before */+ uint64_t ra_first; /* the first piece in ra_data */+ uint32_t ra_count; /* how many are there; 0: none */+ bool ra_filling;+ uint8_t *ra_data; /* APFS_RA_PIECES pieces, made when first wanted */+ uint32_t ra_len[APFS_RA_PIECES];};/* ---- container and volumes ---- */diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.cindex 4439bba..fc7b0d2 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c@@ -252,6 +252,9 @@ apfs_stats_print(void)printf("apfs: batch %llu requests %llu pieces %u threads at most\n",(unsigned long long)nx->batch_calls, (unsigned long long)nx->batch_pieces,nx->parallel != NULL ? nx->parallel_max : 1);+ printf("apfs: ahead %llu times %llu pieces %llu of them used\n",+ (unsigned long long)nx->ra_batches, (unsigned long long)nx->ra_pieces,+ (unsigned long long)nx->ra_hits);printf("apfs: blocks %llu read %llu cache hits highest block %llu\n",(unsigned long long)nx->reads, (unsigned long long)nx->hits,(unsigned long long)nx->max_addr);