Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: decompress outside the volume's lock

vcvkk25bf6e114a20
diff --git a/tools/apfs/bench-phone.sh b/tools/apfs/bench-phone.sh
new file mode 100755
index 0000000..8325017
--- /dev/null
+++ b/tools/apfs/bench-phone.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Readers at once over an APFS volume on the phone (boot-arg apfs_check=5), one line per run.
+# The volume has to be the phone's resident image already (`./xl ramdisk IMAGE`); it stays
+# there across runs as long as none of them hangs.
+#
+# usage: tools/apfs/bench-phone.sh "1 2 4 8" [extra boot-args]
+# env: BENCH_KERNEL a kernel to put in place first (default: whatever is built)
+# BENCH_CPUS cpus=N for the kernel (default 8)
+set -u
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT=$(cd "$HERE/../.." && pwd)
+COUNTS=${1:-"1 2 4 8"}
+EXTRA=${2:-}
+CPUS=${BENCH_CPUS:-8}
+cd "$ROOT" || exit 1
+K=xnu-src/xnu-8796.121.2/BUILD/obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837
+[ -n "${BENCH_KERNEL:-}" ] && cp -p "$BENCH_KERNEL" "$K"
+
+printf "%-7s %9s %8s %12s %12s %10s %4s %s\n" readers wall_ms MB/s lock_wait_ms lock_held_ms inflate_ms bad sums
+for n in $COUNTS; do
+ out=$(XL_CMDLINE="cpus=$CPUS apfs_check=5 apfs_threads=$n $EXTRA" XL_SECS=300 \
+ timeout 500 ./xl run 2>&1)
+ log=$(echo "$out" | grep -o 'logs/[0-9]*-run.log' | tail -1)
+ if [ -z "$log" ] || ! grep -aq "readers done in" "$log" 2>/dev/null; then
+ printf "%-7s run did not come back: %s\n" "$n" "$(echo "$out" | tail -1)"
+ break
+ fi
+ wall=$(grep -ao "$n readers done in [0-9]*" "$log" | head -1 | awk '{print $5}')
+ kbs=$(grep -ao "bytes [0-9]* KB/s" "$log" | head -1 | awk '{print $2}')
+ lock=$(grep -a "apfs: lock" "$log" | head -1 | sed 's/^\[[^]]*\]: //')
+ infl=$(grep -a "apfs: inflate" "$log" | head -1 | sed 's/^\[[^]]*\]: //' | awk '{print $5}')
+ bad=$(grep -ao "readers that disagreed or failed: [0-9]*" "$log" | head -1 | awk '{print $6}')
+ sums=$(grep -ao "sum [0-9a-f]\{16\}" "$log" | sort -u | awk '{print $2}' | tr '\n' ' ')
+ printf "%-7s %9s %8s %12s %12s %10s %4s %s [%s]\n" "$n" "$wall" \
+ "$(echo "scale=1; $kbs / 1024" | bc)" "$(echo "$lock" | awk '{print $5}')" \
+ "$(echo "$lock" | awk '{print $8}')" "$infl" "$bad" "$sums" "$log"
+done
diff --git a/tools/apfs/bench-qemu.sh b/tools/apfs/bench-qemu.sh
new file mode 100755
index 0000000..4f6d06b
--- /dev/null
+++ b/tools/apfs/bench-qemu.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Readers at once over an APFS volume in QEMU, the way they are measured on the phone
+# (boot-arg apfs_check=5): one line per run, so two kernels can be put side by side.
+#
+# The volume is the root device -- IMAGE becomes md0 -- and every reader walks all of it.
+#
+# usage: tools/apfs/bench-qemu.sh IMAGE "1 2 4 8" [extra boot-args]
+# env: QEMU_SMP (default 8) processors of the machine, and cpus=N to the kernel to match
+# BENCH_LOGS directory for the logs (default apfs-work)
+#
+# Runs only this working copy's QEMU instance (qemu/instance), and waits for its ports.
+set -u
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT=$(cd "$HERE/../.." && pwd)
+IMAGE=$1
+COUNTS=${2:-"1 2 4 8"}
+EXTRA=${3:-}
+SMP=${QEMU_SMP:-8}
+LOGS=${BENCH_LOGS:-$ROOT/apfs-work}
+INSTANCE=$(cat "$ROOT/qemu/instance" 2>/dev/null || echo 0)
+PORT=$((4444 + 10 * INSTANCE))
+cd "$ROOT" || exit 1
+touch "$IMAGE" # run-ios.sh rebuilds its RAM image only from a newer file
+
+printf "%-7s %9s %11s %12s %12s %8s %s\n" readers wall_ms MB/s lock_wait_ms lock_held_ms "bad" sums
+for n in $COUNTS; do
+ for i in $(seq 1 60); do
+ avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)
+ ss -ltn 2>/dev/null | grep -q ":$PORT " || { [ "$avail" -ge 3300 ] && break; }
+ sleep 5
+ done
+ log="$LOGS/bench-$(date +%H%M%S)-$n.log"
+ QEMU_SMP=$SMP IMG="$IMAGE" ROOT=userland/root.packfs \
+ CMDLINE="cpus=$SMP apfs_check=5 apfs_threads=$n $EXTRA" \
+ timeout 900 qemu/run-ios.sh 600 > "$log" 2>&1
+ wall=$(grep -ao "$n readers done in [0-9]*" "$log" | head -1 | awk '{print $5}')
+ kbs=$(grep -ao "bytes [0-9]* KB/s" "$log" | head -1 | awk '{print $2}')
+ lock=$(grep -a "apfs: lock" "$log" | head -1 | sed 's/^\[[^]]*\]: //')
+ wait=$(echo "$lock" | awk '{print $5}')
+ held=$(echo "$lock" | awk '{print $8}')
+ bad=$(grep -ao "readers that disagreed or failed: [0-9]*" "$log" | head -1 | awk '{print $6}')
+ sums=$(grep -ao "sum [0-9a-f]\{16\}" "$log" | sort -u | awk '{print $2}' | tr '\n' ' ')
+ printf "%-7s %9s %11s %12s %12s %8s %s\n" "$n" "${wall:-?}" \
+ "$( [ -n "${kbs:-}" ] && echo "scale=1; $kbs / 1024" | bc || echo ?)" \
+ "${wait:-?}" "${held:-?}" "${bad:-?}" "${sums:-?}"
+done
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index 98c15b3..9dde810 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -171,6 +171,22 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)
return 0;
}
+static inline void
+nx_unlock(struct apfs_nx *nx)
+{
+ if (nx->unlock != NULL) {
+ nx->unlock(nx->lock_ctx);
+ }
+}
+
+static inline void
+nx_relock(struct apfs_nx *nx)
+{
+ if (nx->relock != NULL) {
+ nx->relock(nx->lock_ctx);
+ }
+}
+
static uint32_t
obj_type(const uint8_t *blk)
{
@@ -1964,6 +1980,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)
{
uint8_t *piece = NULL;
+ uint8_t *mine = NULL; /* a piece's worth, decompressed into without the lock */
size_t piece_cap = 0;
uint64_t first, last, i;
int error = 0;
@@ -2015,18 +2032,57 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
error = EIO;
goto out;
}
- cf->held = APFS_NO_PIECE; /* what is in there now is unknown */
if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {
size_t m = plen < want ? (size_t)plen : want;
+ cf->held = APFS_NO_PIECE;
memcpy(cf->held_data, piece, m);
cf->held_len = m;
} else {
- error = apfs_inflate_counted(vol->nx, piece, plen,
- cf->held_data, want, &cf->held_len);
+ /*
+ * Decompress without the caller's lock, into a buffer of our
+ * own: zlib touches nothing anyone else can see, and it is most
+ * of the time spent here -- with the lock held through it, four
+ * readers on eight processors kept the lock busy 94% of the
+ * time. The piece only goes into the file's cache once the
+ * lock is back, by swapping buffers. Two readers of the same
+ * file may decompress the same piece once each; that is the
+ * price, and the cache stays right either way.
+ */
+ size_t got_len = 0;
+ uint64_t t0;
+
+ if (mine == NULL) {
+ mine = apfs_alloc(APFS_DECMPFS_BLOCK_SIZE);
+ if (mine == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
+ }
+ t0 = apfs_ticks();
+ nx_unlock(vol->nx);
+ error = apfs_inflate_counted(NULL, piece, plen, mine, want,
+ &got_len);
+ nx_relock(vol->nx);
+ vol->nx->inflate_ticks += apfs_ticks() - t0;
+ vol->nx->inflate_calls++;
+ vol->nx->inflate_in += plen;
+ vol->nx->inflate_out += got_len;
if (error) {
goto out;
}
+ if (got_len != want) {
+ error = EIO;
+ goto out;
+ }
+ /* in it goes: the old piece's buffer becomes ours */
+ {
+ uint8_t *t = cf->held_data;
+
+ cf->held_data = mine;
+ mine = t;
+ }
+ cf->held_len = got_len;
}
if (cf->held_len != want) {
error = EIO;
@@ -2047,6 +2103,9 @@ out:
if (piece != NULL) {
apfs_free(piece, piece_cap);
}
+ if (mine != NULL) {
+ apfs_free(mine, APFS_DECMPFS_BLOCK_SIZE);
+ }
return error;
}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index db95c7b..0acb63a 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -120,6 +120,16 @@ struct apfs_nx {
uint32_t fs_count;
uint8_t uuid[16];
struct apfs_omap omap; /* resolves volume superblocks */
+ /*
+ * The caller's lock round this container, which apfs_lib may let go of while it
+ * does work that touches nothing shared -- decompressing a piece into a buffer of
+ * its own -- and take back afterwards. Everything else here (the block cache, the
+ * cursors, a file's held piece) is only touched with it held. NULL for a caller
+ * with no lock, such as the single-threaded program on the build machine.
+ */
+ void (*unlock)(void *ctx);
+ void (*relock)(void *ctx);
+ void *lock_ctx;
struct apfs_buf cache[APFS_CACHE_BLOCKS];
uint32_t clock;
uint64_t reads;
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
index 643afab..e5cfb96 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
@@ -318,6 +318,19 @@ apfs_node_detach(struct apfs_node *an)
lck_mtx_unlock(&am->am_lock);
}
+/* apfs_lib lets go of the mount's lock round decompression, through these */
+static void
+apfs_lib_unlock(void *ctx)
+{
+ apfs_unlock((struct apfs_mount *)ctx);
+}
+
+static void
+apfs_lib_relock(void *ctx)
+{
+ apfs_lock((struct apfs_mount *)ctx);
+}
+
/* ------------------------------------------------------------------ mount */
static void
@@ -382,6 +395,11 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)
return error;
}
+ /* apfs_nx_open() starts from a clean slate, so the lock is handed over only now */
+ am->am_nx.unlock = apfs_lib_unlock;
+ am->am_nx.relock = apfs_lib_relock;
+ am->am_nx.lock_ctx = am;
+
/* the first volume that has a root directory */
{
uint32_t i;