Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: counters inside the driver say where the time goes -- waiting for the device

vcvkke3e5efb953e9
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index f95ad03..bd12985 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -16,9 +16,7 @@
* apfs-tool census IMAGE [PATH]
*/
-/* pread(2) */
-#define _DEFAULT_SOURCE
-
+/* pread(2) and clock_gettime(2) come with -D_DEFAULT_SOURCE, set by build-tool.sh */
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
diff --git a/tools/apfs/build-tool.sh b/tools/apfs/build-tool.sh
index ff0aad3..25478e5 100755
--- a/tools/apfs/build-tool.sh
+++ b/tools/apfs/build-tool.sh
@@ -17,6 +17,6 @@ done
[ -n "$SRC" ] || { echo "build-tool.sh: apfs_lib.c is in neither the kernel tree nor xnu/files" >&2; exit 1; }
mkdir -p "$(dirname "$OUT")"
-cc -std=c11 -O2 -Wall -Wextra -Wno-unused-parameter -I"$SRC" -I"$HERE" \
+cc -std=c11 -D_DEFAULT_SOURCE -O2 -Wall -Wextra -Wno-unused-parameter -I"$SRC" -I"$HERE" \
-o "$OUT" "$HERE/apfs-tool.c" "$SRC/apfs_lib.c" -lz
echo "build-tool.sh: $OUT (from $SRC)"
diff --git a/userland/src/apfs_mmap_test.c b/userland/src/apfs_mmap_test.c
index ac051bb..1ab7d77 100644
--- a/userland/src/apfs_mmap_test.c
+++ b/userland/src/apfs_mmap_test.c
@@ -29,6 +29,7 @@
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
+#include <sys/sysctl.h>
#include <time.h>
#include <unistd.h>
@@ -57,6 +58,44 @@ fnv1a(const unsigned char *p, size_t n, unsigned long long h)
return h;
}
+/*
+ * The driver's own breakdown of where the time went (kern.apfs_stats): waiting for the
+ * device, decompressing, and the rest. Zeroed before a road is walked and printed after,
+ * so the numbers belong to that road alone.
+ */
+static void
+stats_reset(void)
+{
+ char zero = 0;
+
+ if (sysctlbyname("kern.apfs_stats", NULL, NULL, &zero, 1) != 0) {
+ printf("%s cannot zero kern.apfs_stats: %s\n", tag, strerror(errno));
+ }
+}
+
+static void
+stats_print(void)
+{
+ char buf[1024];
+ size_t n = sizeof(buf);
+ char *line, *next;
+
+ if (sysctlbyname("kern.apfs_stats", buf, &n, NULL, 0) != 0) {
+ printf("%s cannot read kern.apfs_stats: %s\n", tag, strerror(errno));
+ return;
+ }
+ buf[sizeof(buf) - 1] = '\0';
+ for (line = buf; line != NULL && *line != '\0'; line = next) {
+ next = strchr(line, '\n');
+ if (next != NULL) {
+ *next++ = '\0';
+ }
+ if (*line != '\0') {
+ printf("%s kernel: %s\n", tag, line);
+ }
+ }
+}
+
static double
now_ms(void)
{
@@ -224,6 +263,7 @@ main(int argc, char **argv)
printf("%s pieces of %d bytes\n", tag, chunk_size);
}
snprintf(path, sizeof(path), "%s", where);
+ stats_reset();
t0 = now_ms();
walk(path, strlen(path));
t1 = now_ms();
@@ -233,5 +273,6 @@ main(int argc, char **argv)
printf("%s line-hash sum %016llx\n", tag, total_sum);
printf("%s done in %.0f ms (%.1f MB/s)\n", tag, t1 - t0,
(t1 > t0) ? (double)total_bytes / 1048576.0 / ((t1 - t0) / 1000.0) : 0.0);
+ stats_print();
return total_errors ? 1 : 0;
}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.h
index 43f7420..4583ec1 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs.h
@@ -38,6 +38,12 @@ struct apfs_mount {
bool am_owns_devvp; /* we opened it, so we close it */
struct apfs_node *am_hash[APFS_HASH_BUCKETS];
uint64_t am_root_id;
+ /*
+ * Where the time goes at the vnode operations, to set beside the container's own
+ * counters (waiting for the device, decompressing). Ticks; see apfs_ticks().
+ */
+ uint64_t am_pagein_calls, am_pagein_ticks, am_pagein_bytes;
+ uint64_t am_read_calls, am_read_ticks, am_read_bytes;
};
/* The private part of a vnode: which object it is. */
@@ -77,6 +83,9 @@ int apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,
struct componentname *cnp, vnode_t *vpp);
void apfs_node_detach(struct apfs_node *an);
+void apfs_lock(struct apfs_mount *am);
+void apfs_unlock(struct apfs_mount *am);
+
/* apfs_vnops.c */
extern const struct vnodeopv_desc apfs_vnodeop_opv_desc;
extern int (**apfs_vnodeop_p)(void *);
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index 63daed6..fa51816 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -145,8 +145,15 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)
return ENOMEM;
}
victim->addr = APFS_BUF_EMPTY;
- error = nx->dev.read(nx->dev.ctx, addr * (uint64_t)nx->block_size, victim->data,
- nx->block_size);
+ {
+ uint64_t t0 = apfs_ticks();
+
+ error = nx->dev.read(nx->dev.ctx, addr * (uint64_t)nx->block_size,
+ victim->data, nx->block_size);
+ nx->dev_ticks += apfs_ticks() - t0;
+ nx->dev_calls++;
+ nx->dev_bytes += nx->block_size;
+ }
if (error) {
return error;
}
@@ -1762,11 +1769,12 @@ apfs_zfree(voidpf opaque, voidpf address)
/* One zlib stream. A leading 0xff means the piece was stored rather than compressed. */
static int
-apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
- size_t *out)
+apfs_inflate_counted(struct apfs_nx *nx, const uint8_t *src, size_t src_len, uint8_t *dst,
+ size_t dst_len, size_t *out)
{
z_stream zs;
int zerr;
+ uint64_t t0;
if (src_len == 0) {
return EIO;
@@ -1781,6 +1789,7 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
*out = n;
return 0;
}
+ t0 = apfs_ticks();
memset(&zs, 0, sizeof(zs));
zs.zalloc = apfs_zalloc;
zs.zfree = apfs_zfree;
@@ -1795,6 +1804,12 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
zerr = inflate(&zs, Z_FINISH);
*out = (size_t)zs.total_out;
inflateEnd(&zs);
+ if (nx != NULL) {
+ nx->inflate_ticks += apfs_ticks() - t0;
+ nx->inflate_calls++;
+ nx->inflate_in += src_len;
+ nx->inflate_out += *out;
+ }
if (zerr != Z_STREAM_END && zerr != Z_OK && zerr != Z_BUF_ERROR) {
return EIO;
}
@@ -1811,8 +1826,8 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
* exactly the right length is taken whole.
*/
static int
-decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,
- uint8_t *dst, size_t *out)
+decmpfs_attr(struct apfs_nx *nx, uint32_t type, const uint8_t *src, size_t src_len,
+ size_t want, uint8_t *dst, size_t *out)
{
switch (type) {
case APFS_DECMPFS_TYPE_RAW_ATTR: {
@@ -1834,7 +1849,7 @@ decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,
return 0;
}
case APFS_DECMPFS_TYPE_ZLIB_ATTR:
- return apfs_inflate(src, src_len, dst, want, out);
+ return apfs_inflate_counted(nx, src, src_len, dst, want, out);
default:
return ENOTSUP;
}
@@ -1902,7 +1917,7 @@ read_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,
goto out;
}
}
- error = decmpfs_attr(cf->type, attr + 16, attr_len - 16, (size_t)cf->size,
+ error = decmpfs_attr(vol->nx, cf->type, attr + 16, attr_len - 16, (size_t)cf->size,
plain, &plain_len);
if (error) {
goto out;
@@ -2001,8 +2016,8 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
memcpy(cf->held_data, piece, m);
cf->held_len = m;
} else {
- error = apfs_inflate(piece, plen, cf->held_data, want,
- &cf->held_len);
+ error = apfs_inflate_counted(vol->nx, piece, plen,
+ cf->held_data, want, &cf->held_len);
if (error) {
goto out;
}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index 5d29e00..ff909f1 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -28,6 +28,26 @@
#include "apfs_format.h"
+/*
+ * A clock for the breakdown counters below. Raw ticks: the kernel's are mach absolute
+ * time and are turned into nanoseconds where they are printed, the host's already are
+ * nanoseconds.
+ */
+#ifdef KERNEL
+#include <kern/clock.h>
+#define apfs_ticks() mach_absolute_time()
+#else
+#include <time.h>
+static inline uint64_t
+apfs_ticks(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec;
+}
+#endif
+
#ifdef KERNEL
#include <sys/param.h>
#include <sys/errno.h>
@@ -104,6 +124,12 @@ struct apfs_nx {
uint32_t clock;
uint64_t reads;
uint64_t hits;
+ /*
+ * Where the time goes, to tell apart waiting for the device from decompressing.
+ * Ticks, not nanoseconds -- see apfs_ticks().
+ */
+ uint64_t dev_calls, dev_ticks, dev_bytes;
+ uint64_t inflate_calls, inflate_ticks, inflate_in, inflate_out;
};
/* A volume inside the container. */
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
index 1975c12..632cff6 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
@@ -16,6 +16,7 @@
#include <sys/systm.h>
#include <sys/time.h>
#include <uuid/uuid.h>
+#include <sys/sysctl.h>
#include <sys/vnode_internal.h>
#include <sys/fcntl.h>
@@ -70,6 +71,85 @@ apfs_dev_read(void *ctx, uint64_t off, void *buf, size_t len)
return 0;
}
+/* ------------------------------------------------------------- where the time goes */
+
+/*
+ * One mounted volume, for the sysctl below to look at. This port mounts one APFS volume
+ * and the counters are a bring-up aid, so a single pointer is enough; it is set when the
+ * volume is mounted and cleared when it goes away.
+ */
+static struct apfs_mount *apfs_the_mount;
+
+static uint64_t
+apfs_ticks_ms(uint64_t ticks)
+{
+ uint64_t ns = 0;
+
+ absolutetime_to_nanoseconds(ticks, &ns);
+ return ns / 1000000;
+}
+
+/*
+ * kern.apfs_stats: reading it says where the time went, writing anything zeroes the
+ * counters. The point is to tell apart, on one road through the driver, the time spent
+ * waiting for the device, the time spent decompressing, and the rest -- which is the
+ * driver's own work and the paging machinery above it.
+ */
+static int
+sysctl_apfs_stats SYSCTL_HANDLER_ARGS
+{
+#pragma unused(oidp, arg1, arg2)
+ struct apfs_mount *am = apfs_the_mount;
+ char text[640];
+ int error;
+
+ if (am == NULL) {
+ strlcpy(text, "apfs: nothing mounted\n", sizeof(text));
+ } else {
+ struct apfs_nx *nx = &am->am_nx;
+
+ snprintf(text, sizeof(text),
+ "pagein %llu calls %llu ms %llu bytes\n"
+ "read %llu calls %llu ms %llu bytes\n"
+ "device %llu calls %llu ms %llu bytes\n"
+ "inflate %llu calls %llu ms %llu bytes in %llu bytes out\n"
+ "blocks %llu read %llu cache hits\n",
+ (unsigned long long)am->am_pagein_calls,
+ (unsigned long long)apfs_ticks_ms(am->am_pagein_ticks),
+ (unsigned long long)am->am_pagein_bytes,
+ (unsigned long long)am->am_read_calls,
+ (unsigned long long)apfs_ticks_ms(am->am_read_ticks),
+ (unsigned long long)am->am_read_bytes,
+ (unsigned long long)nx->dev_calls,
+ (unsigned long long)apfs_ticks_ms(nx->dev_ticks),
+ (unsigned long long)nx->dev_bytes,
+ (unsigned long long)nx->inflate_calls,
+ (unsigned long long)apfs_ticks_ms(nx->inflate_ticks),
+ (unsigned long long)nx->inflate_in,
+ (unsigned long long)nx->inflate_out,
+ (unsigned long long)nx->reads,
+ (unsigned long long)nx->hits);
+ }
+ error = SYSCTL_OUT(req, text, strlen(text) + 1);
+ if (error || req->newptr == USER_ADDR_NULL) {
+ return error;
+ }
+ if (am != NULL) {
+ apfs_lock(am);
+ am->am_pagein_calls = am->am_pagein_ticks = am->am_pagein_bytes = 0;
+ am->am_read_calls = am->am_read_ticks = am->am_read_bytes = 0;
+ am->am_nx.dev_calls = am->am_nx.dev_ticks = am->am_nx.dev_bytes = 0;
+ am->am_nx.inflate_calls = am->am_nx.inflate_ticks = 0;
+ am->am_nx.inflate_in = am->am_nx.inflate_out = 0;
+ am->am_nx.reads = am->am_nx.hits = 0;
+ apfs_unlock(am);
+ }
+ return 0;
+}
+
+SYSCTL_PROC(_kern, OID_AUTO, apfs_stats, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
+ 0, 0, sysctl_apfs_stats, "A", "APFS: where the time goes; write to zero it");
+
/* ------------------------------------------------------------------ vnodes */
static uint32_t
@@ -305,6 +385,7 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)
sp->f_fsid.val[1] = vfs_typenum(mp);
strlcpy(sp->f_fstypename, "apfs", sizeof(sp->f_fstypename));
+ apfs_the_mount = am;
printf("apfs: volume \"%s\": %llu files, %llu directories, %llu blocks of %u used\n",
am->am_vol.name, (unsigned long long)am->am_vol.num_files,
(unsigned long long)am->am_vol.num_dirs,
@@ -394,6 +475,9 @@ apfs_unmount(mount_t mp, int mntflags, vfs_context_t ctx)
if (error) {
return error;
}
+ if (apfs_the_mount == am) {
+ apfs_the_mount = NULL;
+ }
vfs_setfsprivate(mp, NULL);
if (am->am_owns_devvp && am->am_devvp != NULLVP) {
(void)VNOP_CLOSE(am->am_devvp, FREAD, ctx);
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
index 710db07..04f296c 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
@@ -25,13 +25,13 @@
/* How much is read out of a file at a time when copying into a uio. */
#define APFS_IOBUF 65536
-static void
+void
apfs_lock(struct apfs_mount *am)
{
lck_mtx_lock(&am->am_lock);
}
-static void
+void
apfs_unlock(struct apfs_mount *am)
{
lck_mtx_unlock(&am->am_lock);
@@ -159,6 +159,7 @@ apfs_vnop_read(struct vnop_read_args *ap)
struct apfs_inode ino;
uint8_t *buf;
uint64_t size = 0;
+ uint64_t t0, read_bytes = 0;
int error = 0;
if (vnode_isdir(vp)) {
@@ -176,6 +177,7 @@ apfs_vnop_read(struct vnop_read_args *ap)
if (buf == NULL) {
return ENOMEM;
}
+ t0 = apfs_ticks();
while (uio_resid(uio) > 0 && (uint64_t)uio_offset(uio) < size) {
size_t want = (size_t)MIN((user_ssize_t)APFS_IOBUF, uio_resid(uio));
size_t got = 0;
@@ -194,8 +196,14 @@ apfs_vnop_read(struct vnop_read_args *ap)
if (error) {
break;
}
+ read_bytes += got;
}
kfree_data(buf, APFS_IOBUF);
+ apfs_lock(am);
+ am->am_read_ticks += apfs_ticks() - t0;
+ am->am_read_calls++;
+ am->am_read_bytes += read_bytes;
+ apfs_unlock(am);
return error;
}
@@ -215,6 +223,7 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)
struct apfs_inode ino;
vm_offset_t address = 0;
uint64_t size = 0;
+ uint64_t t0 = apfs_ticks();
off_t at, end;
int error = 0;
@@ -284,6 +293,11 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)
}
ubc_upl_unmap(upl);
+ apfs_lock(am);
+ am->am_pagein_ticks += apfs_ticks() - t0;
+ am->am_pagein_calls++;
+ am->am_pagein_bytes += ap->a_size;
+ apfs_unlock(am);
if (!(ap->a_flags & UPL_NOCOMMIT)) {
if (error) {
ubc_upl_abort_range(upl, upl_offset, (upl_size_t)ap->a_size,