Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: decompress outside the volume's lock
vcvkk25bf6e114a20
diff --git a/tools/apfs/bench-phone.sh b/tools/apfs/bench-phone.shnew file mode 100755index 0000000..8325017--- /dev/null+++ b/tools/apfs/bench-phone.sh@@ -0,0 +1,37 @@+#!/bin/bash+# Readers at once over an APFS volume on the phone (boot-arg apfs_check=5), one line per run.+# The volume has to be the phone's resident image already (`./xl ramdisk IMAGE`); it stays+# there across runs as long as none of them hangs.+#+# usage: tools/apfs/bench-phone.sh "1 2 4 8" [extra boot-args]+# env: BENCH_KERNEL a kernel to put in place first (default: whatever is built)+# BENCH_CPUS cpus=N for the kernel (default 8)+set -u+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)+ROOT=$(cd "$HERE/../.." && pwd)+COUNTS=${1:-"1 2 4 8"}+EXTRA=${2:-}+CPUS=${BENCH_CPUS:-8}+cd "$ROOT" || exit 1+K=xnu-src/xnu-8796.121.2/BUILD/obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837+[ -n "${BENCH_KERNEL:-}" ] && cp -p "$BENCH_KERNEL" "$K"++printf "%-7s %9s %8s %12s %12s %10s %4s %s\n" readers wall_ms MB/s lock_wait_ms lock_held_ms inflate_ms bad sums+for n in $COUNTS; do+ out=$(XL_CMDLINE="cpus=$CPUS apfs_check=5 apfs_threads=$n $EXTRA" XL_SECS=300 \+ timeout 500 ./xl run 2>&1)+ log=$(echo "$out" | grep -o 'logs/[0-9]*-run.log' | tail -1)+ if [ -z "$log" ] || ! grep -aq "readers done in" "$log" 2>/dev/null; then+ printf "%-7s run did not come back: %s\n" "$n" "$(echo "$out" | tail -1)"+ break+ fi+ wall=$(grep -ao "$n readers done in [0-9]*" "$log" | head -1 | awk '{print $5}')+ kbs=$(grep -ao "bytes [0-9]* KB/s" "$log" | head -1 | awk '{print $2}')+ lock=$(grep -a "apfs: lock" "$log" | head -1 | sed 's/^\[[^]]*\]: //')+ infl=$(grep -a "apfs: inflate" "$log" | head -1 | sed 's/^\[[^]]*\]: //' | awk '{print $5}')+ bad=$(grep -ao "readers that disagreed or failed: [0-9]*" "$log" | head -1 | awk '{print $6}')+ sums=$(grep -ao "sum [0-9a-f]\{16\}" "$log" | sort -u | awk '{print $2}' | tr '\n' ' ')+ printf "%-7s %9s %8s %12s %12s %10s %4s %s [%s]\n" "$n" "$wall" \+ "$(echo "scale=1; $kbs / 1024" | bc)" "$(echo "$lock" | awk '{print $5}')" \+ "$(echo "$lock" | awk '{print $8}')" "$infl" "$bad" "$sums" "$log"+donediff --git a/tools/apfs/bench-qemu.sh b/tools/apfs/bench-qemu.shnew file mode 100755index 0000000..4f6d06b--- /dev/null+++ b/tools/apfs/bench-qemu.sh@@ -0,0 +1,46 @@+#!/bin/bash+# Readers at once over an APFS volume in QEMU, the way they are measured on the phone+# (boot-arg apfs_check=5): one line per run, so two kernels can be put side by side.+#+# The volume is the root device -- IMAGE becomes md0 -- and every reader walks all of it.+#+# usage: tools/apfs/bench-qemu.sh IMAGE "1 2 4 8" [extra boot-args]+# env: QEMU_SMP (default 8) processors of the machine, and cpus=N to the kernel to match+# BENCH_LOGS directory for the logs (default apfs-work)+#+# Runs only this working copy's QEMU instance (qemu/instance), and waits for its ports.+set -u+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)+ROOT=$(cd "$HERE/../.." && pwd)+IMAGE=$1+COUNTS=${2:-"1 2 4 8"}+EXTRA=${3:-}+SMP=${QEMU_SMP:-8}+LOGS=${BENCH_LOGS:-$ROOT/apfs-work}+INSTANCE=$(cat "$ROOT/qemu/instance" 2>/dev/null || echo 0)+PORT=$((4444 + 10 * INSTANCE))+cd "$ROOT" || exit 1+touch "$IMAGE" # run-ios.sh rebuilds its RAM image only from a newer file++printf "%-7s %9s %11s %12s %12s %8s %s\n" readers wall_ms MB/s lock_wait_ms lock_held_ms "bad" sums+for n in $COUNTS; do+ for i in $(seq 1 60); do+ avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)+ ss -ltn 2>/dev/null | grep -q ":$PORT " || { [ "$avail" -ge 3300 ] && break; }+ sleep 5+ done+ log="$LOGS/bench-$(date +%H%M%S)-$n.log"+ QEMU_SMP=$SMP IMG="$IMAGE" ROOT=userland/root.packfs \+ CMDLINE="cpus=$SMP apfs_check=5 apfs_threads=$n $EXTRA" \+ timeout 900 qemu/run-ios.sh 600 > "$log" 2>&1+ wall=$(grep -ao "$n readers done in [0-9]*" "$log" | head -1 | awk '{print $5}')+ kbs=$(grep -ao "bytes [0-9]* KB/s" "$log" | head -1 | awk '{print $2}')+ lock=$(grep -a "apfs: lock" "$log" | head -1 | sed 's/^\[[^]]*\]: //')+ wait=$(echo "$lock" | awk '{print $5}')+ held=$(echo "$lock" | awk '{print $8}')+ bad=$(grep -ao "readers that disagreed or failed: [0-9]*" "$log" | head -1 | awk '{print $6}')+ sums=$(grep -ao "sum [0-9a-f]\{16\}" "$log" | sort -u | awk '{print $2}' | tr '\n' ' ')+ printf "%-7s %9s %11s %12s %12s %8s %s\n" "$n" "${wall:-?}" \+ "$( [ -n "${kbs:-}" ] && echo "scale=1; $kbs / 1024" | bc || echo ?)" \+ "${wait:-?}" "${held:-?}" "${bad:-?}" "${sums:-?}"+donediff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex 98c15b3..9dde810 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -171,6 +171,22 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)return 0;}+static inline void+nx_unlock(struct apfs_nx *nx)+{+ if (nx->unlock != NULL) {+ nx->unlock(nx->lock_ctx);+ }+}++static inline void+nx_relock(struct apfs_nx *nx)+{+ if (nx->relock != NULL) {+ nx->relock(nx->lock_ctx);+ }+}+static uint32_tobj_type(const uint8_t *blk){@@ -1964,6 +1980,7 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out){uint8_t *piece = NULL;+ uint8_t *mine = NULL; /* a piece's worth, decompressed into without the lock */size_t piece_cap = 0;uint64_t first, last, i;int error = 0;@@ -2015,18 +2032,57 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,error = EIO;goto out;}- cf->held = APFS_NO_PIECE; /* what is in there now is unknown */if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {size_t m = plen < want ? (size_t)plen : want;+ cf->held = APFS_NO_PIECE;memcpy(cf->held_data, piece, m);cf->held_len = m;} else {- error = apfs_inflate_counted(vol->nx, piece, plen,- cf->held_data, want, &cf->held_len);+ /*+ * Decompress without the caller's lock, into a buffer of our+ * own: zlib touches nothing anyone else can see, and it is most+ * of the time spent here -- with the lock held through it, four+ * readers on eight processors kept the lock busy 94% of the+ * time. The piece only goes into the file's cache once the+ * lock is back, by swapping buffers. Two readers of the same+ * file may decompress the same piece once each; that is the+ * price, and the cache stays right either way.+ */+ size_t got_len = 0;+ uint64_t t0;++ if (mine == NULL) {+ mine = apfs_alloc(APFS_DECMPFS_BLOCK_SIZE);+ if (mine == NULL) {+ error = ENOMEM;+ goto out;+ }+ }+ t0 = apfs_ticks();+ nx_unlock(vol->nx);+ error = apfs_inflate_counted(NULL, piece, plen, mine, want,+ &got_len);+ nx_relock(vol->nx);+ vol->nx->inflate_ticks += apfs_ticks() - t0;+ vol->nx->inflate_calls++;+ vol->nx->inflate_in += plen;+ vol->nx->inflate_out += got_len;if (error) {goto out;}+ if (got_len != want) {+ error = EIO;+ goto out;+ }+ /* in it goes: the old piece's buffer becomes ours */+ {+ uint8_t *t = cf->held_data;++ cf->held_data = mine;+ mine = t;+ }+ cf->held_len = got_len;}if (cf->held_len != want) {error = EIO;@@ -2047,6 +2103,9 @@ out:if (piece != NULL) {apfs_free(piece, piece_cap);}+ if (mine != NULL) {+ apfs_free(mine, APFS_DECMPFS_BLOCK_SIZE);+ }return error;}diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex db95c7b..0acb63a 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -120,6 +120,16 @@ struct apfs_nx {uint32_t fs_count;uint8_t uuid[16];struct apfs_omap omap; /* resolves volume superblocks */+ /*+ * The caller's lock round this container, which apfs_lib may let go of while it+ * does work that touches nothing shared -- decompressing a piece into a buffer of+ * its own -- and take back afterwards. Everything else here (the block cache, the+ * cursors, a file's held piece) is only touched with it held. NULL for a caller+ * with no lock, such as the single-threaded program on the build machine.+ */+ void (*unlock)(void *ctx);+ void (*relock)(void *ctx);+ void *lock_ctx;struct apfs_buf cache[APFS_CACHE_BLOCKS];uint32_t clock;uint64_t reads;diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.cindex 643afab..e5cfb96 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c@@ -318,6 +318,19 @@ apfs_node_detach(struct apfs_node *an)lck_mtx_unlock(&am->am_lock);}+/* apfs_lib lets go of the mount's lock round decompression, through these */+static void+apfs_lib_unlock(void *ctx)+{+ apfs_unlock((struct apfs_mount *)ctx);+}++static void+apfs_lib_relock(void *ctx)+{+ apfs_lock((struct apfs_mount *)ctx);+}+/* ------------------------------------------------------------------ mount */static void@@ -382,6 +395,11 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)return error;}+ /* apfs_nx_open() starts from a clean slate, so the lock is handed over only now */+ am->am_nx.unlock = apfs_lib_unlock;+ am->am_nx.relock = apfs_lib_relock;+ am->am_nx.lock_ctx = am;+/* the first volume that has a root directory */{uint32_t i;