Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: read-ahead for a reader going through a file in 64 KB reads

vcvkkf7504f82d8cc
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index 7cdaaf1..3761936 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -586,8 +586,9 @@ cmd_census(const char *path, const char *where, bool meta_only)
" bytes out, %" PRIu64 " by zlib\n", nx.inflate_calls, nx.inflate_in, nx.inflate_out,
nx.inflate_fallbacks);
fprintf(stderr, "apfs-tool: %" PRIu64 " requests of whole pieces, %" PRIu64 " pieces in "
- "them, on %u threads at most\n", nx.batch_calls, nx.batch_pieces,
- nx.parallel != NULL ? nx.parallel_max : 1);
+ "them, on %u threads at most; read ahead %" PRIu64 " times, %" PRIu64 " pieces, %"
+ PRIu64 " of them used\n", nx.batch_calls, nx.batch_pieces,
+ nx.parallel != NULL ? nx.parallel_max : 1, nx.ra_batches, nx.ra_pieces, nx.ra_hits);
fprintf(stderr, "apfs-tool: highest block touched %" PRIu64 " (%.2f GB in), of which "
"file contents reach %" PRIu64 " (%.2f GB in)\n", nx.max_addr,
nx.max_addr * (double)nx.block_size / 1e9, nx.max_data_addr,
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index c6f3521..245566c 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -1766,6 +1766,9 @@ apfs_cfile_fini(struct apfs_cfile *cf)
if (cf->held_data != NULL) {
apfs_free(cf->held_data, cf->held_cap);
}
+ if (cf->ra_data != NULL) {
+ apfs_free(cf->ra_data, (size_t)APFS_RA_PIECES * APFS_DECMPFS_BLOCK_SIZE);
+ }
memset(cf, 0, sizeof(*cf));
cf->held = APFS_NO_PIECE;
}
@@ -2259,6 +2262,8 @@ struct apfs_piece_job {
uint32_t plen; /* compressed */
uint8_t *dst; /* the caller's, `want` bytes */
size_t want;
+ bool ra; /* read ahead, not asked for: its failure fails nothing */
+ int error;
};
#define APFS_BATCH 32
@@ -2390,7 +2395,8 @@ batch_one(void *arg, uint32_t item, uint32_t worker)
if (error == 0 && got != j->want) {
error = EIO;
}
- return error;
+ j->error = error;
+ return j->ra ? 0 : error;
}
/*
@@ -2435,8 +2441,12 @@ batch_run(struct apfs_vol *vol, struct apfs_scratch *sc, uint32_t n)
if (workers > 1) {
error = nx->parallel(nx->parallel_ctx, n, workers, batch_one, &b);
} else {
- for (i = 0; i < n && error == 0; i++) {
- error = batch_one(&b, i, 0);
+ for (i = 0; i < n; i++) {
+ int e = batch_one(&b, i, 0);
+
+ if (e != 0 && error == 0) {
+ error = e;
+ }
}
}
nx_relock(nx);
@@ -2470,6 +2480,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
size_t big_cap = 0;
uint32_t bs = vol->nx->block_size;
uint32_t nbatch = 0; /* whole pieces gathered in sc->jobs, not yet done */
+ uint32_t ra_at = 0, nra = 0; /* read-ahead jobs: sc->jobs[ra_at .. ra_at + nra) */
uint64_t first, last, i;
int error = 0;
@@ -2499,6 +2510,20 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
}
from_in_piece = (off > i * APFS_DECMPFS_BLOCK_SIZE)
? (size_t)(off - i * APFS_DECMPFS_BLOCK_SIZE) : 0;
+ if (cf->held != i && cf->ra_count > 0 && !cf->ra_filling &&
+ i >= cf->ra_first && i - cf->ra_first < cf->ra_count &&
+ cf->ra_len[i - cf->ra_first] == want) {
+ /* read ahead by an earlier request of this reader's */
+ n = want - from_in_piece;
+ if (n > len - *out) {
+ n = len - *out;
+ }
+ memcpy(buf + *out, cf->ra_data +
+ (size_t)(i - cf->ra_first) * APFS_DECMPFS_BLOCK_SIZE + from_in_piece, n);
+ *out += n;
+ vol->nx->ra_hits++;
+ continue;
+ }
if (cf->held != i) {
uint32_t plen = cf->piece_len[i];
uint64_t poff = cf->table_base + cf->piece_off[i];
@@ -2583,6 +2608,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
j->plen = plen;
j->dst = buf + *out;
j->want = want;
+ j->ra = false;
*out += want;
if (nbatch == APFS_BATCH) {
error = batch_run(vol, sc, nbatch);
@@ -2591,6 +2617,59 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
goto out;
}
}
+ /*
+ * The last piece of this request, the one right after where this
+ * reader's last request ended, and threads to spare for it (the
+ * share batch_run would give it is two or more): the pieces after it
+ * go into this batch as well, into ra_data.
+ */
+ if (i == last && i >= 1 && i == cf->ra_expect && !cf->ra_filling &&
+ vol->nx->parallel != NULL && i + 1 < cf->piece_count &&
+ vol->nx->parallel_max / (vol->nx->requests > 1 ?
+ vol->nx->requests : 1) >= 2) {
+ uint64_t k;
+
+ if (cf->ra_data == NULL) {
+ cf->ra_data = apfs_alloc((size_t)APFS_RA_PIECES *
+ APFS_DECMPFS_BLOCK_SIZE);
+ }
+ cf->ra_count = 0;
+ ra_at = nbatch;
+ for (k = i + 1; cf->ra_data != NULL && k < cf->piece_count &&
+ k - i <= APFS_RA_PIECES && nbatch < APFS_BATCH; k++) {
+ size_t kwant = APFS_DECMPFS_BLOCK_SIZE;
+ uint32_t kplen = cf->piece_len[k];
+ uint64_t kpoff = cf->table_base + cf->piece_off[k];
+ struct apfs_run kruns[APFS_MAX_RUNS];
+ uint32_t knruns = 0;
+ uint64_t kcovered = 0;
+
+ if ((k + 1) * APFS_DECMPFS_BLOCK_SIZE > cf->size) {
+ kwant = (size_t)(cf->size - k * APFS_DECMPFS_BLOCK_SIZE);
+ }
+ if (kplen > APFS_PIECE_CAP || kpoff >= cf->rsrc_size ||
+ kplen > cf->rsrc_size - kpoff ||
+ stream_plan(vol, cf->rsrc_id, kpoff, kplen, kruns,
+ APFS_MAX_RUNS, &knruns, &kcovered) != 0 ||
+ kcovered != kplen || knruns != 1 || kruns[0].at != 0 ||
+ kruns[0].len != kplen) {
+ break; /* not one for a batch: the read-ahead stops here */
+ }
+ j = &sc->jobs[nbatch++];
+ j->addr = kruns[0].addr;
+ j->skew = kruns[0].skew;
+ j->plen = kplen;
+ j->dst = cf->ra_data + (size_t)(k - i - 1) * APFS_DECMPFS_BLOCK_SIZE;
+ j->want = kwant;
+ j->ra = true;
+ cf->ra_len[k - i - 1] = (uint32_t)kwant;
+ nra++;
+ }
+ if (nra > 0) {
+ cf->ra_filling = true;
+ cf->ra_first = i + 1;
+ }
+ }
continue;
}
if (!raw_type && !direct && sc->mine == NULL) {
@@ -2686,7 +2765,31 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
if (nbatch > 0) {
error = batch_run(vol, sc, nbatch);
}
+ if (nra > 0) {
+ uint32_t k;
+
+ cf->ra_count = nra;
+ for (k = 0; k < nra; k++) {
+ if (sc->jobs[ra_at + k].error != 0) {
+ cf->ra_count = k; /* what came before the first that failed */
+ break;
+ }
+ }
+ vol->nx->ra_batches++;
+ vol->nx->ra_pieces += cf->ra_count;
+ cf->ra_filling = false;
+ nra = 0;
+ }
+ if (error == 0) {
+ cf->ra_expect = ((off + len) % APFS_DECMPFS_BLOCK_SIZE == 0)
+ ? (off + len) / APFS_DECMPFS_BLOCK_SIZE : APFS_NO_PIECE;
+ }
out:
+ if (nra > 0) {
+ /* given up on before the batch ran: nothing in ra_data is good */
+ cf->ra_count = 0;
+ cf->ra_filling = false;
+ }
if (big != NULL) {
apfs_free(big, big_cap);
}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index 34e918b..dce5f70 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -75,6 +75,9 @@ apfs_ticks(void)
/* Room for a symbolic link's target (Darwin's own limit is 1024 bytes; it is kept in an xattr) */
#define APFS_LINK_MAX 4096
+/* How many pieces a sequential reader's read-ahead decompresses at a time */
+#define APFS_RA_PIECES 7
+
/*
* The device. read() must fill the whole buffer or return an error; offsets and lengths
* are always whole blocks of the container.
@@ -189,6 +192,7 @@ struct apfs_nx {
*/
uint32_t requests;
uint64_t batch_calls, batch_pieces; /* how many requests used it, for how much */
+ uint64_t ra_batches, ra_pieces, ra_hits; /* read-ahead: filled, and used */
};
/* A volume inside the container. */
@@ -322,6 +326,21 @@ struct apfs_cfile {
uint8_t *held_data;
size_t held_len;
size_t held_cap; /* 0 when this file is not worth holding */
+
+ /*
+ * Read-ahead. A reader with threads to spare (alone on the volume, or one of two)
+ * that asks for the piece right after the one its last read ended on is taken to be
+ * reading the file through: the pieces
+ * after it are decompressed in the same batch, on the other threads, into ra_data,
+ * and its next reads are copies out of that. All of it happens inside the request
+ * that asked; ra_filling keeps everyone else off ra_data while the lock is let go of.
+ */
+ uint64_t ra_expect; /* the piece the last whole-piece read ended before */
+ uint64_t ra_first; /* the first piece in ra_data */
+ uint32_t ra_count; /* how many are there; 0: none */
+ bool ra_filling;
+ uint8_t *ra_data; /* APFS_RA_PIECES pieces, made when first wanted */
+ uint32_t ra_len[APFS_RA_PIECES];
};
/* ---- container and volumes ---- */
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
index 4439bba..fc7b0d2 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
@@ -252,6 +252,9 @@ apfs_stats_print(void)
printf("apfs: batch %llu requests %llu pieces %u threads at most\n",
(unsigned long long)nx->batch_calls, (unsigned long long)nx->batch_pieces,
nx->parallel != NULL ? nx->parallel_max : 1);
+ printf("apfs: ahead %llu times %llu pieces %llu of them used\n",
+ (unsigned long long)nx->ra_batches, (unsigned long long)nx->ra_pieces,
+ (unsigned long long)nx->ra_hits);
printf("apfs: blocks %llu read %llu cache hits highest block %llu\n",
(unsigned long long)nx->reads, (unsigned long long)nx->hits,
(unsigned long long)nx->max_addr);