Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: counters inside the driver say where the time goes -- waiting for the device
vcvkke3e5efb953e9
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.cindex f95ad03..bd12985 100644--- a/tools/apfs/apfs-tool.c+++ b/tools/apfs/apfs-tool.c@@ -16,9 +16,7 @@* apfs-tool census IMAGE [PATH]*/-/* pread(2) */-#define _DEFAULT_SOURCE-+/* pread(2) and clock_gettime(2) come with -D_DEFAULT_SOURCE, set by build-tool.sh */#include <fcntl.h>#include <inttypes.h>#include <stdio.h>diff --git a/tools/apfs/build-tool.sh b/tools/apfs/build-tool.shindex ff0aad3..25478e5 100755--- a/tools/apfs/build-tool.sh+++ b/tools/apfs/build-tool.sh@@ -17,6 +17,6 @@ done[ -n "$SRC" ] || { echo "build-tool.sh: apfs_lib.c is in neither the kernel tree nor xnu/files" >&2; exit 1; }mkdir -p "$(dirname "$OUT")"-cc -std=c11 -O2 -Wall -Wextra -Wno-unused-parameter -I"$SRC" -I"$HERE" \+cc -std=c11 -D_DEFAULT_SOURCE -O2 -Wall -Wextra -Wno-unused-parameter -I"$SRC" -I"$HERE" \-o "$OUT" "$HERE/apfs-tool.c" "$SRC/apfs_lib.c" -lzecho "build-tool.sh: $OUT (from $SRC)"diff --git a/userland/src/apfs_mmap_test.c b/userland/src/apfs_mmap_test.cindex ac051bb..1ab7d77 100644--- a/userland/src/apfs_mmap_test.c+++ b/userland/src/apfs_mmap_test.c@@ -29,6 +29,7 @@#include <string.h>#include <sys/mman.h>#include <sys/stat.h>+#include <sys/sysctl.h>#include <time.h>#include <unistd.h>@@ -57,6 +58,44 @@ fnv1a(const unsigned char *p, size_t n, unsigned long long h)return h;}+/*+ * The driver's own breakdown of where the time went (kern.apfs_stats): waiting for the+ * device, decompressing, and the rest. Zeroed before a road is walked and printed after,+ * so the numbers belong to that road alone.+ */+static void+stats_reset(void)+{+ char zero = 0;++ if (sysctlbyname("kern.apfs_stats", NULL, NULL, &zero, 1) != 0) {+ printf("%s cannot zero kern.apfs_stats: %s\n", tag, strerror(errno));+ }+}++static void+stats_print(void)+{+ char buf[1024];+ size_t n = sizeof(buf);+ char *line, *next;++ if (sysctlbyname("kern.apfs_stats", buf, &n, NULL, 0) != 0) {+ printf("%s cannot read kern.apfs_stats: %s\n", tag, strerror(errno));+ return;+ }+ buf[sizeof(buf) - 1] = '\0';+ for (line = buf; line != NULL && *line != '\0'; line = next) {+ next = strchr(line, '\n');+ if (next != NULL) {+ *next++ = '\0';+ }+ if (*line != '\0') {+ printf("%s kernel: %s\n", tag, line);+ }+ }+}+static doublenow_ms(void){@@ -224,6 +263,7 @@ main(int argc, char **argv)printf("%s pieces of %d bytes\n", tag, chunk_size);}snprintf(path, sizeof(path), "%s", where);+ stats_reset();t0 = now_ms();walk(path, strlen(path));t1 = now_ms();@@ -233,5 +273,6 @@ main(int argc, char **argv)printf("%s line-hash sum %016llx\n", tag, total_sum);printf("%s done in %.0f ms (%.1f MB/s)\n", tag, t1 - t0,(t1 > t0) ? (double)total_bytes / 1048576.0 / ((t1 - t0) / 1000.0) : 0.0);+ stats_print();return total_errors ? 1 : 0;}diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.hindex 43f7420..4583ec1 100644--- a/xnu/files/bsd/miscfs/apfs/apfs.h+++ b/xnu/files/bsd/miscfs/apfs/apfs.h@@ -38,6 +38,12 @@ struct apfs_mount {bool am_owns_devvp; /* we opened it, so we close it */struct apfs_node *am_hash[APFS_HASH_BUCKETS];uint64_t am_root_id;+ /*+ * Where the time goes at the vnode operations, to set beside the container's own+ * counters (waiting for the device, decompressing). Ticks; see apfs_ticks().+ */+ uint64_t am_pagein_calls, am_pagein_ticks, am_pagein_bytes;+ uint64_t am_read_calls, am_read_ticks, am_read_bytes;};/* The private part of a vnode: which object it is. */@@ -77,6 +83,9 @@ int apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,struct componentname *cnp, vnode_t *vpp);void apfs_node_detach(struct apfs_node *an);+void apfs_lock(struct apfs_mount *am);+void apfs_unlock(struct apfs_mount *am);+/* apfs_vnops.c */extern const struct vnodeopv_desc apfs_vnodeop_opv_desc;extern int (**apfs_vnodeop_p)(void *);diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex 63daed6..fa51816 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -145,8 +145,15 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)return ENOMEM;}victim->addr = APFS_BUF_EMPTY;- error = nx->dev.read(nx->dev.ctx, addr * (uint64_t)nx->block_size, victim->data,- nx->block_size);+ {+ uint64_t t0 = apfs_ticks();++ error = nx->dev.read(nx->dev.ctx, addr * (uint64_t)nx->block_size,+ victim->data, nx->block_size);+ nx->dev_ticks += apfs_ticks() - t0;+ nx->dev_calls++;+ nx->dev_bytes += nx->block_size;+ }if (error) {return error;}@@ -1762,11 +1769,12 @@ apfs_zfree(voidpf opaque, voidpf address)/* One zlib stream. A leading 0xff means the piece was stored rather than compressed. */static int-apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,- size_t *out)+apfs_inflate_counted(struct apfs_nx *nx, const uint8_t *src, size_t src_len, uint8_t *dst,+ size_t dst_len, size_t *out){z_stream zs;int zerr;+ uint64_t t0;if (src_len == 0) {return EIO;@@ -1781,6 +1789,7 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,*out = n;return 0;}+ t0 = apfs_ticks();memset(&zs, 0, sizeof(zs));zs.zalloc = apfs_zalloc;zs.zfree = apfs_zfree;@@ -1795,6 +1804,12 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,zerr = inflate(&zs, Z_FINISH);*out = (size_t)zs.total_out;inflateEnd(&zs);+ if (nx != NULL) {+ nx->inflate_ticks += apfs_ticks() - t0;+ nx->inflate_calls++;+ nx->inflate_in += src_len;+ nx->inflate_out += *out;+ }if (zerr != Z_STREAM_END && zerr != Z_OK && zerr != Z_BUF_ERROR) {return EIO;}@@ -1811,8 +1826,8 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,* exactly the right length is taken whole.*/static int-decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,- uint8_t *dst, size_t *out)+decmpfs_attr(struct apfs_nx *nx, uint32_t type, const uint8_t *src, size_t src_len,+ size_t want, uint8_t *dst, size_t *out){switch (type) {case APFS_DECMPFS_TYPE_RAW_ATTR: {@@ -1834,7 +1849,7 @@ decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,return 0;}case APFS_DECMPFS_TYPE_ZLIB_ATTR:- return apfs_inflate(src, src_len, dst, want, out);+ return apfs_inflate_counted(nx, src, src_len, dst, want, out);default:return ENOTSUP;}@@ -1902,7 +1917,7 @@ read_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,goto out;}}- error = decmpfs_attr(cf->type, attr + 16, attr_len - 16, (size_t)cf->size,+ error = decmpfs_attr(vol->nx, cf->type, attr + 16, attr_len - 16, (size_t)cf->size,plain, &plain_len);if (error) {goto out;@@ -2001,8 +2016,8 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,memcpy(cf->held_data, piece, m);cf->held_len = m;} else {- error = apfs_inflate(piece, plen, cf->held_data, want,- &cf->held_len);+ error = apfs_inflate_counted(vol->nx, piece, plen,+ cf->held_data, want, &cf->held_len);if (error) {goto out;}diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex 5d29e00..ff909f1 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -28,6 +28,26 @@#include "apfs_format.h"+/*+ * A clock for the breakdown counters below. Raw ticks: the kernel's are mach absolute+ * time and are turned into nanoseconds where they are printed, the host's already are+ * nanoseconds.+ */+#ifdef KERNEL+#include <kern/clock.h>+#define apfs_ticks() mach_absolute_time()+#else+#include <time.h>+static inline uint64_t+apfs_ticks(void)+{+ struct timespec ts;++ clock_gettime(CLOCK_MONOTONIC, &ts);+ return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec;+}+#endif+#ifdef KERNEL#include <sys/param.h>#include <sys/errno.h>@@ -104,6 +124,12 @@ struct apfs_nx {uint32_t clock;uint64_t reads;uint64_t hits;+ /*+ * Where the time goes, to tell apart waiting for the device from decompressing.+ * Ticks, not nanoseconds -- see apfs_ticks().+ */+ uint64_t dev_calls, dev_ticks, dev_bytes;+ uint64_t inflate_calls, inflate_ticks, inflate_in, inflate_out;};/* A volume inside the container. */diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.cindex 1975c12..632cff6 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c@@ -16,6 +16,7 @@#include <sys/systm.h>#include <sys/time.h>#include <uuid/uuid.h>+#include <sys/sysctl.h>#include <sys/vnode_internal.h>#include <sys/fcntl.h>@@ -70,6 +71,85 @@ apfs_dev_read(void *ctx, uint64_t off, void *buf, size_t len)return 0;}+/* ------------------------------------------------------------- where the time goes */++/*+ * One mounted volume, for the sysctl below to look at. This port mounts one APFS volume+ * and the counters are a bring-up aid, so a single pointer is enough; it is set when the+ * volume is mounted and cleared when it goes away.+ */+static struct apfs_mount *apfs_the_mount;++static uint64_t+apfs_ticks_ms(uint64_t ticks)+{+ uint64_t ns = 0;++ absolutetime_to_nanoseconds(ticks, &ns);+ return ns / 1000000;+}++/*+ * kern.apfs_stats: reading it says where the time went, writing anything zeroes the+ * counters. The point is to tell apart, on one road through the driver, the time spent+ * waiting for the device, the time spent decompressing, and the rest -- which is the+ * driver's own work and the paging machinery above it.+ */+static int+sysctl_apfs_stats SYSCTL_HANDLER_ARGS+{+#pragma unused(oidp, arg1, arg2)+ struct apfs_mount *am = apfs_the_mount;+ char text[640];+ int error;++ if (am == NULL) {+ strlcpy(text, "apfs: nothing mounted\n", sizeof(text));+ } else {+ struct apfs_nx *nx = &am->am_nx;++ snprintf(text, sizeof(text),+ "pagein %llu calls %llu ms %llu bytes\n"+ "read %llu calls %llu ms %llu bytes\n"+ "device %llu calls %llu ms %llu bytes\n"+ "inflate %llu calls %llu ms %llu bytes in %llu bytes out\n"+ "blocks %llu read %llu cache hits\n",+ (unsigned long long)am->am_pagein_calls,+ (unsigned long long)apfs_ticks_ms(am->am_pagein_ticks),+ (unsigned long long)am->am_pagein_bytes,+ (unsigned long long)am->am_read_calls,+ (unsigned long long)apfs_ticks_ms(am->am_read_ticks),+ (unsigned long long)am->am_read_bytes,+ (unsigned long long)nx->dev_calls,+ (unsigned long long)apfs_ticks_ms(nx->dev_ticks),+ (unsigned long long)nx->dev_bytes,+ (unsigned long long)nx->inflate_calls,+ (unsigned long long)apfs_ticks_ms(nx->inflate_ticks),+ (unsigned long long)nx->inflate_in,+ (unsigned long long)nx->inflate_out,+ (unsigned long long)nx->reads,+ (unsigned long long)nx->hits);+ }+ error = SYSCTL_OUT(req, text, strlen(text) + 1);+ if (error || req->newptr == USER_ADDR_NULL) {+ return error;+ }+ if (am != NULL) {+ apfs_lock(am);+ am->am_pagein_calls = am->am_pagein_ticks = am->am_pagein_bytes = 0;+ am->am_read_calls = am->am_read_ticks = am->am_read_bytes = 0;+ am->am_nx.dev_calls = am->am_nx.dev_ticks = am->am_nx.dev_bytes = 0;+ am->am_nx.inflate_calls = am->am_nx.inflate_ticks = 0;+ am->am_nx.inflate_in = am->am_nx.inflate_out = 0;+ am->am_nx.reads = am->am_nx.hits = 0;+ apfs_unlock(am);+ }+ return 0;+}++SYSCTL_PROC(_kern, OID_AUTO, apfs_stats, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,+ 0, 0, sysctl_apfs_stats, "A", "APFS: where the time goes; write to zero it");+/* ------------------------------------------------------------------ vnodes */static uint32_t@@ -305,6 +385,7 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)sp->f_fsid.val[1] = vfs_typenum(mp);strlcpy(sp->f_fstypename, "apfs", sizeof(sp->f_fstypename));+ apfs_the_mount = am;printf("apfs: volume \"%s\": %llu files, %llu directories, %llu blocks of %u used\n",am->am_vol.name, (unsigned long long)am->am_vol.num_files,(unsigned long long)am->am_vol.num_dirs,@@ -394,6 +475,9 @@ apfs_unmount(mount_t mp, int mntflags, vfs_context_t ctx)if (error) {return error;}+ if (apfs_the_mount == am) {+ apfs_the_mount = NULL;+ }vfs_setfsprivate(mp, NULL);if (am->am_owns_devvp && am->am_devvp != NULLVP) {(void)VNOP_CLOSE(am->am_devvp, FREAD, ctx);diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.cindex 710db07..04f296c 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c@@ -25,13 +25,13 @@/* How much is read out of a file at a time when copying into a uio. */#define APFS_IOBUF 65536-static void+voidapfs_lock(struct apfs_mount *am){lck_mtx_lock(&am->am_lock);}-static void+voidapfs_unlock(struct apfs_mount *am){lck_mtx_unlock(&am->am_lock);@@ -159,6 +159,7 @@ apfs_vnop_read(struct vnop_read_args *ap)struct apfs_inode ino;uint8_t *buf;uint64_t size = 0;+ uint64_t t0, read_bytes = 0;int error = 0;if (vnode_isdir(vp)) {@@ -176,6 +177,7 @@ apfs_vnop_read(struct vnop_read_args *ap)if (buf == NULL) {return ENOMEM;}+ t0 = apfs_ticks();while (uio_resid(uio) > 0 && (uint64_t)uio_offset(uio) < size) {size_t want = (size_t)MIN((user_ssize_t)APFS_IOBUF, uio_resid(uio));size_t got = 0;@@ -194,8 +196,14 @@ apfs_vnop_read(struct vnop_read_args *ap)if (error) {break;}+ read_bytes += got;}kfree_data(buf, APFS_IOBUF);+ apfs_lock(am);+ am->am_read_ticks += apfs_ticks() - t0;+ am->am_read_calls++;+ am->am_read_bytes += read_bytes;+ apfs_unlock(am);return error;}@@ -215,6 +223,7 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)struct apfs_inode ino;vm_offset_t address = 0;uint64_t size = 0;+ uint64_t t0 = apfs_ticks();off_t at, end;int error = 0;@@ -284,6 +293,11 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)}ubc_upl_unmap(upl);+ apfs_lock(am);+ am->am_pagein_ticks += apfs_ticks() - t0;+ am->am_pagein_calls++;+ am->am_pagein_bytes += ap->a_size;+ apfs_unlock(am);if (!(ap->a_flags & UPL_NOCOMMIT)) {if (error) {ubc_upl_abort_range(upl, upl_offset, (upl_size_t)ap->a_size,